scrape: Fix for saving last id
This commit is contained in:
@@ -51,6 +51,7 @@ def get_last_id() -> int:
|
||||
|
||||
|
||||
def save_last_id(val: int):
|
||||
logging.info(f"Saving last_id={val}")
|
||||
with open(SCRAPING_LAST_ID_PATH, "w") as f:
|
||||
f.write(str(val))
|
||||
|
||||
@@ -71,7 +72,7 @@ def main():
|
||||
send_repository(rabbit_channel, json.dumps(repository_data).encode("utf-8"))
|
||||
|
||||
last_item_id = repository_data["id"]
|
||||
if processed_count % SAVE_ID_EVERY:
|
||||
if processed_count % SAVE_ID_EVERY == 0:
|
||||
save_last_id(last_item_id)
|
||||
except RateLimitExceededException:
|
||||
save_last_id(last_item_id)
|
||||
|
||||
Reference in New Issue
Block a user