MirrorHub/atlas
0
0
Fork 0
mirror of https://github.com/placeAtlas/atlas.git synced 2024-12-27 23:44:40 +01:00
Code Issues Projects Releases Wiki Activity

Add forgotten parts on the port, move some parts for better git diff

Browse source
This commit is contained in:
Hans5958 2022-04-08 11:19:46 +07:00
parent e9cd995dbd
commit cc497d5178
1 changed files with 9 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
tools/redditcrawl.py
View file

@ -1,11 +1,10 @@
import praw import praw
import json import json
import time import time
import os
from formatter import format_all
import traceback
import re import re
import os
import traceback
from formatter import format_all
outfile = open('temp_atlas.json', 'w', encoding='utf-8') outfile = open('temp_atlas.json', 'w', encoding='utf-8')
failfile = open('manual_atlas.json', 'w', encoding='utf-8') failfile = open('manual_atlas.json', 'w', encoding='utf-8')
@ -30,10 +29,6 @@
print("Warning: No write access. Post flairs will not be updated.") print("Warning: No write access. Post flairs will not be updated.")
time.sleep(5) time.sleep(5)
failcount = 0
successcount = 0
totalcount = 0
jsonfile = open("../web/atlas.json", "r", encoding='utf-8') jsonfile = open("../web/atlas.json", "r", encoding='utf-8')
existing = json.load(jsonfile) existing = json.load(jsonfile)
@ -50,6 +45,10 @@ def set_flair(submission, flair):
total_all_flairs = 0 total_all_flairs = 0
duplicate_count = 0 duplicate_count = 0
failcount = 0
successcount = 0
totalcount = 0
outfile.write("[\n") outfile.write("[\n")
for submission in reddit.subreddit('placeAtlas2').new(limit=2000): for submission in reddit.subreddit('placeAtlas2').new(limit=2000):
""" """
@ -78,9 +77,10 @@ def set_flair(submission, flair):
total_all_flairs += 1 total_all_flairs += 1
if (submission.id in existing_ids): if (submission.id in existing_ids):
set_flair(submission, "Processed Entry")
print("Found first duplicate!") print("Found first duplicate!")
duplicate_count += 1 duplicate_count += 1
if (duplicate_count > 10): if (duplicate_count > 0):
break break
else: else:
continue continue