scrape: Fix for saving last id

This commit is contained in:
2021-01-16 22:07:44 +03:00
parent 3b15ef41f4
commit 7d8df2761d

View File

@@ -51,6 +51,7 @@ def get_last_id() -> int:
def save_last_id(val: int):
logging.info(f"Saving last_id={val}")
with open(SCRAPING_LAST_ID_PATH, "w") as f:
f.write(str(val))
@@ -71,7 +72,7 @@ def main():
send_repository(rabbit_channel, json.dumps(repository_data).encode("utf-8"))
last_item_id = repository_data["id"]
if processed_count % SAVE_ID_EVERY:
if processed_count % SAVE_ID_EVERY == 0:
save_last_id(last_item_id)
except RateLimitExceededException:
save_last_id(last_item_id)