MirrorHub/atlas
0
0
Fork 0
mirror of https://github.com/placeAtlas/atlas.git synced 2024-06-17 10:48:38 +02:00
Code Issues Projects Releases Wiki Activity

Take only the first object, remove escapes, add more info on errors

Browse source
This commit is contained in:
Hans5958 2022-04-08 06:52:00 +07:00
parent 713ebec844
commit 1da79b806b
1 changed files with 14 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
tools/redditcrawl.py
View file

@ -1,11 +1,11 @@
from array import array
import praw
import json
import time
import re
import os
from formatter import format_all
import traceback
import re
outfile = open('temp_atlas.json', 'w', encoding='utf-8')
failfile = open('manual_atlas.json', 'w', encoding='utf-8')
@ -67,6 +67,11 @@
try:
text = submission.selftext
rawtext = text
text = text.replace("\\", "")
text = re.compile(r".*(\{.+\}).*", re.DOTALL).search(text).group(1)
submission_json = json.loads(text)
if submission_json:
@ -88,10 +93,15 @@
successcount += 1
except Exception as e:
failfile.write(text+"\n\n" + "="*40 + "\n\n")
failfile.write(
"\n\n" + "="*40 + "\n\n" +
submission.id + "\n\n" +
traceback.format_exc() + "\n\n" +
rawtext
)
failcount += 1
print("written "+submission.id+" submitted "+str(round(time.time()-submission.created_utc))+" seconds ago")
print("written "+submission.id+", submitted "+str(round(time.time()-submission.created_utc))+" seconds ago")
totalcount += 1
# Remove ,\n