MirrorHub/atlas
0
0
Fork 0
mirror of https://github.com/placeAtlas/atlas.git synced 2024-10-15 05:49:12 +02:00
Code Issues Projects Releases Wiki Activity

Merge pull request #1183 from Hans5958/fix-recursive-adding

Browse source 
List ids that already been read in a seperate file
This commit is contained in:
Stefano 2022-04-10 16:40:57 +02:00 committed by GitHub
commit 4db8e85f23
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 7165 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7160
data/read-ids.txt Normal file
View file

File diff suppressed because it is too large Load diff

10
tools/redditcrawl.py
View file

@ -7,6 +7,7 @@
from formatter import format_all from formatter import format_all
outfile = open('temp_atlas.json', 'w', encoding='utf-8') outfile = open('temp_atlas.json', 'w', encoding='utf-8')
editidsfile = open('read-ids-temp.txt', 'w')
failfile = open('manual_atlas.json', 'w', encoding='utf-8') failfile = open('manual_atlas.json', 'w', encoding='utf-8')
with open('credentials', 'r') as file: with open('credentials', 'r') as file:
@ -29,13 +30,11 @@
print("Warning: No write access. Post flairs will not be updated.") print("Warning: No write access. Post flairs will not be updated.")
time.sleep(5) time.sleep(5)
jsonfile = open("../web/atlas.json", "r", encoding='utf-8')
existing = json.load(jsonfile)
existing_ids = [] existing_ids = []
for item in existing: with open('../data/edit-ids.txt', 'r') as edit_ids_file:
existing_ids.append(item['id']) for id in [x.strip() for x in edit_ids_file.readlines()]:
existing_ids.append(id)
def set_flair(submission, flair): def set_flair(submission, flair):
if has_write_access and submission.link_flair_text != flair: if has_write_access and submission.link_flair_text != flair:
@ -116,6 +115,7 @@ def set_flair(submission, flair):
"Submission invalid after validation. This may be caused by not enough points on the path." "Submission invalid after validation. This may be caused by not enough points on the path."
outfile.write(json.dumps(submission_json, ensure_ascii=False) + ",\n") outfile.write(json.dumps(submission_json, ensure_ascii=False) + ",\n")
editidsfile.write(submission.id + '\n')
successcount += 1 successcount += 1
set_flair(submission, "Processed Entry") set_flair(submission, "Processed Entry")