alex/assets-bot
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

scrape: Fix for saving last id

Browse Source
This commit is contained in:
Aleksey Lobanov
2021-01-16 22:07:44 +03:00
parent 3b15ef41f4
commit 7d8df2761d
1 changed files with 2 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
scrape_repos.py
View File

@@ -51,6 +51,7 @@ def get_last_id() -> int:
def save_last_id(val: int): def save_last_id(val: int):
logging.info(f"Saving last_id={val}")
with open(SCRAPING_LAST_ID_PATH, "w") as f: with open(SCRAPING_LAST_ID_PATH, "w") as f:
f.write(str(val)) f.write(str(val))
@@ -71,7 +72,7 @@ def main():
send_repository(rabbit_channel, json.dumps(repository_data).encode("utf-8")) send_repository(rabbit_channel, json.dumps(repository_data).encode("utf-8"))
last_item_id = repository_data["id"] last_item_id = repository_data["id"]
if processed_count % SAVE_ID_EVERY: if processed_count % SAVE_ID_EVERY == 0:
save_last_id(last_item_id) save_last_id(last_item_id)
except RateLimitExceededException: except RateLimitExceededException:
save_last_id(last_item_id) save_last_id(last_item_id)