MirrorHub/atlas
0
0
Fork 0
mirror of https://github.com/placeAtlas/atlas.git synced 2024-09-16 07:09:02 +02:00
Code Issues Projects Releases Wiki Activity

Merge pull request #829 from nico-abram/valid_temp_json

Browse source 
Make redditcrawl create a valid json for atlas_temp
This commit is contained in:
ash 2022-04-06 22:12:19 +01:00 committed by GitHub
commit 3b50b0950f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 18 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
tools/redditcrawl.py
View file

@ -3,6 +3,7 @@
import json import json
import time import time
import re import re
import os
outfile = open('temp_atlas.json', 'w', encoding='utf-8') outfile = open('temp_atlas.json', 'w', encoding='utf-8')
failfile = open('manual_atlas.json', 'w', encoding='utf-8') failfile = open('manual_atlas.json', 'w', encoding='utf-8')
@ -25,7 +26,9 @@
for item in existing: for item in existing:
existing_ids.append(item['id']) existing_ids.append(item['id'])
total_all_flairs = 0
duplicate_count = 0
outfile.write("[\n")
for submission in reddit.subreddit('placeAtlas2').new(limit=2000): for submission in reddit.subreddit('placeAtlas2').new(limit=2000):
""" """
Auth setup Auth setup
@ -47,10 +50,14 @@
4. Pull Request 4. Pull Request
""" """
#print(dir(submission)) total_all_flairs += 1
if (submission.id in existing_ids): if (submission.id in existing_ids):
print("Found first duplicate!") print("Found first duplicate!")
duplicate_count += 1
if (duplicate_count > 50):
break break
else:
continue
if(submission.link_flair_text == "New Entry"): if(submission.link_flair_text == "New Entry"):
text = submission.selftext text = submission.selftext
#Old backslash filter: #Old backslash filter:
@ -81,4 +88,10 @@
print("written "+submission.id+" submitted "+str(round(time.time()-submission.created_utc))+" seconds ago") print("written "+submission.id+" submitted "+str(round(time.time()-submission.created_utc))+" seconds ago")
totalcount += 1 totalcount += 1
print(f"\n\nSuccess: {successcount}/{totalcount}\nFail: {failcount}/{totalcount}\nPlease check manual_atlas.txt for failed entries to manually resolve.") # Remove ,\n
outfile.seek(outfile.tell()-4, os.SEEK_SET)
outfile.truncate()
outfile.write("\n]")
print(f"\n\nTotal all flairs:{total_all_flairs}\nSuccess: {successcount}/{totalcount}\nFail: {failcount}/{totalcount}\nPlease check manual_atlas.txt for failed entries to manually resolve.")