add .dockerignore

close #3

.dockerignore

@@ -0,0 +1,5 @@
+*.csv.gz
+*.csv
+*.db
+env/
+authfile.example

Dockerfile

@@ -0,0 +1,8 @@
+FROM python:3.10-alpine
+RUN apk add sqlite
+ADD . ./e621_api_cloner
+RUN pip3 install -Ur ./e621_api_cloner/requirements.txt
+RUN touch ./e621_api_cloner/e621.db
+EXPOSE 1337
+WORKDIR /e621_api_cloner
+CMD ["hypercorn", "--access-log", "-", "-b", "0.0.0.0:1337", "e621_api_cloner:app"]

authfile.example

@@ -0,0 +1,2 @@
+5a8649fb426e13c080e118332f44b3adb70cc9c677fba58c3d018313c4b0ad67 my silly tool account 1
+504054a7c35af520ab0ddd14e1ad257633fe75ba8601915541abba6cec1a81f7 my silly tool account 2

build_database.py

@@ -196,6 +196,9 @@ async def main_with_ctx(ctx, wanted_date):
             await ctx.db.commit()
 
     log.info("going to process posts")
+    post_count_rows = await ctx.db.execute_fetchall("select count(*) from posts")
+    post_count = post_count_rows[0][0]
+    log.info("already have %d posts", post_count)
 
     with output_uncompressed_paths["posts"].open(
         mode="r", encoding="utf-8"
@@ -207,6 +210,9 @@ async def main_with_ctx(ctx, wanted_date):
         line_count -= 1  # remove header
 
         log.info("%d posts to import", line_count)
+        if line_count == post_count:
+            log.info("already imported everything, skipping")
+        else:
             posts_csv_fd.seek(0)
             posts_reader = csv.DictReader(posts_csv_fd)
 
@@ -282,6 +288,10 @@ async def main_with_ctx(ctx, wanted_date):
 
             await ctx.db.commit()
 
+    log.info("vacuuming db...")
+    await ctx.db.execute("vacuum")
+    log.info("database built")
+
 
 async def main():
     wanted_date = sys.argv[1]

e621_api_cloner.py

@@ -7,6 +7,7 @@ import sqlite3
 import sys
 import os
 import enum
+from pathlib import Path
 from datetime import datetime
 from dataclasses import dataclass, asdict
 from hypercorn.asyncio import serve, Config
@@ -19,13 +20,32 @@ app = Quart(__name__)
 
 @app.before_serving
 async def app_before_serving():
+    logging.basicConfig(
+        level=logging.DEBUG if os.environ.get("DEBUG") else logging.INFO
+    )
+
     db_path = os.environ["DBPATH"]
     app.db = await aiosqlite.connect(db_path)
     app.db.row_factory = sqlite3.Row
 
+    # contains api keys
+    maybe_authfile_path = os.environ.get("AUTHFILE")
+    app.maybe_authfile = Path(maybe_authfile_path) if maybe_authfile_path else None
+    if app.maybe_authfile:
+        app.apikeys = {}
+        log.info("loading auth with api keys")
+        with app.maybe_authfile.open(mode="r") as fd:
+            for line in fd:
+                api_key, *user_name = line.split(" ")
+                app.apikeys[api_key] = (" ".join(user_name)).strip()
+
 
 @app.after_serving
 async def app_after_serving():
+    log.info("possibly optimizing database")
+    await app.db.execute("PRAGMA analysis_limit=400")
+    await app.db.execute("PRAGMA optimize")
+    log.info("closing connection")
     await app.db.close()
 
 
@@ -96,8 +116,34 @@ class TagCategory(enum.IntEnum):
         }[self]
 
 
+async def maybe_do_authentication():
+    if not app.maybe_authfile:
+        return None
+
+    auth_line = request.headers.get("authorization")
+    if not auth_line:
+        return "authorization header required", 400
+
+    auth_type, auth_data = auth_line.split(" ")
+    if auth_type != "Bearer":
+        log.warn("invalid auth type")
+        return "invalid auth token type (must be Bearer)", 400
+
+    auth_name = app.apikeys.get(auth_data)
+    if auth_name is None:
+        log.warn("invalid auth value")
+        return "invalid auth token (unknown api key)", 400
+
+    log.info("logged in as %r", auth_name)
+    return None
+
+
 @app.route("/posts.json")
 async def posts_json():
+    res = await maybe_do_authentication()
+    if res:
+        return res
+
     tag_str = request.args["tags"]
     tags = tag_str.split(" ")
     if len(tags) != 1:
@@ -130,12 +176,3 @@ async def posts_json():
         post_json["tags"][category_str].append(tag)
 
     return {"posts": [post_json]}
-
-
-if __name__ == "__main__":
-    logging.basicConfig(
-        level=logging.DEBUG if os.environ.get("DEBUG") else logging.INFO
-    )
-    config = Config()
-    config.bind = ["0.0.0.0:1334"]
-    asyncio.run(serve(app, config))