MirrorHub/atlas
0
0
Fork 0
mirror of https://github.com/placeAtlas/atlas.git synced 2024-09-27 12:39:18 +02:00
Code Issues Projects Releases Wiki Activity

update crawler to reject already submitted

Browse source
This commit is contained in:
ash 2022-04-05 11:25:12 +01:00
parent 6f9fd822c9
commit 3e2f998116
No known key found for this signature in database
GPG key ID: 9680D0BE1469C427
1 changed files with 10 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
tools/redditcrawl.py
View file

@ -14,9 +14,16 @@
failcount = 0
successcount = 0
latestID = int(input("Latest ID: "))
jsonfile = open("../web/atlas.json", "r")
existing = json.load(jsonfile)
for submission in reddit.subreddit('placeAtlas2').new(limit=1100):
existing_ids = []
for item in existing:
existing_ids.append(item['id'])
for submission in reddit.subreddit('placeAtlas2').new(limit=1500):
"""
Auth setup
1. Head to https://www.reddit.com/prefs/apps
@ -38,7 +45,7 @@
"""
#print(dir(submission))
if(submission.link_flair_text == "New Entry"):
if(submission.link_flair_text == "New Entry" and submission.id not in existing_ids):
print(submission.id)
text = submission.selftext
text = text.replace("\\", "")
@ -53,7 +60,6 @@
for i, line in enumerate(lines):
if("\"id\": 0" in line):
lines[i] = line.replace("\"id\": 0", "\"id\": "+"\""+str(submission.id)+"\"")
latestID = latestID + 1
text = "\n".join(lines)
try:
outfile.write(json.dumps(json.loads(text))+",\n")