luna
/
e621_api_cloner
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

don't reimport posts when not needed

This commit is contained in:
Luna 2022-08-29 21:56:57 -03:00
parent 21e05ba546
commit c2576d271c
1 changed files with 78 additions and 68 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
build_database.py
View File

@ -196,6 +196,9 @@ async def main_with_ctx(ctx, wanted_date):
await ctx.db.commit()
log.info("going to process posts")
post_count_rows = await ctx.db.execute_fetchall("select count(*) from posts")
post_count = post_count_rows[0][0]
log.info("already have %d posts", post_count)
with output_uncompressed_paths["posts"].open(
mode="r", encoding="utf-8"
@ -207,6 +210,9 @@ async def main_with_ctx(ctx, wanted_date):
line_count -= 1 # remove header
log.info("%d posts to import", line_count)
if line_count == post_count:
log.info("already imported everything, skipping")
else:
posts_csv_fd.seek(0)
posts_reader = csv.DictReader(posts_csv_fd)
@ -282,6 +288,10 @@ async def main_with_ctx(ctx, wanted_date):
await ctx.db.commit()
log.info("vacuuming db...")
await ctx.db.execute("vacuum")
log.info("database built")
async def main():
wanted_date = sys.argv[1]