atlas/tools/redditcrawl.py

152 lines
5.5 KiB
Python
Raw Normal View History

2022-04-18 16:25:36 +02:00
"""
2023-03-22 15:18:11 +01:00
Setting up authentication:
2022-04-18 16:25:36 +02:00
1. Head to https://www.reddit.com/prefs/apps
2023-03-22 15:18:11 +01:00
2. Click "are you a developer? create an app..." on the button
3. Enter the name and description
4. Select "script" for the type
5. Enter "redirect uri" as "http://localhost:8080"
2022-04-18 16:25:36 +02:00
6. Create file "credentials" with the format below
2023-03-22 15:18:11 +01:00
[id] <- Under "personal use script"
[secret]
[username] <- For flair access, must be a mod, Don't do this... │
[password] <- ...if you don't know what you are doing. │
2022-04-18 16:25:36 +02:00
2023-03-22 15:18:11 +01:00
Running:
1. Run the script
2. Input the next ID to use
3. Manually resolve errors in temp-atlas-manual.json
4 a. Use merge_out.py, or...
2023-03-21 13:29:18 +01:00
b. a. Copy temp-atlas.json entries into web/_js/atlas.js (mind the edits!)
b. Copy temp-read-ids.txt IDs into data/read-ids.txt
2022-04-18 16:25:36 +02:00
5. Create a pull request
"""
2017-04-04 20:12:45 +02:00
import praw
2022-04-04 18:49:57 +02:00
import json
import time
import re
import traceback
2023-03-22 15:18:11 +01:00
from aformatter import format_all, validate
2017-04-04 20:12:45 +02:00
2023-03-21 13:29:18 +01:00
with open('temp-atlas.json', 'w', encoding='utf-8') as OUT_FILE, open('temp-read-ids.txt', 'w') as READ_IDS_FILE, open('temp-atlas-manual.txt', 'w', encoding='utf-8') as FAIL_FILE:
2022-06-11 18:39:10 +02:00
OUT_FILE_LINES = ['[\n', ']\n']
2017-04-04 20:12:45 +02:00
2022-06-11 18:39:10 +02:00
with open('credentials', 'r') as file:
credentials = file.readlines()
client_id = credentials[0].strip()
client_secret = credentials[1].strip()
username = credentials[2].strip() if len(credentials) > 3 else ""
password = credentials[3].strip() if len(credentials) > 3 else ""
reddit = praw.Reddit(
client_id=client_id,
client_secret=client_secret,
username=username,
password=password,
user_agent='atlas_bot'
)
has_write_access = not reddit.read_only
if not has_write_access:
print("Warning: No write access. Post flairs will not be updated.")
time.sleep(5)
existing_ids = []
with open('../data/read-ids.txt', 'r') as edit_ids_file:
for id in [x.strip() for x in edit_ids_file.readlines()]:
existing_ids.append(id)
def set_flair(submission, flair):
if has_write_access and submission.link_flair_text != flair:
flair_choices = submission.flair.choices()
flair = next(x for x in flair_choices if x["flair_text_editable"] and flair == x["flair_text"])
submission.flair.select(flair["flair_template_id"])
total_all_flairs = 0
duplicate_count = 0
failcount = 0
successcount = 0
totalcount = 0
for submission in reddit.subreddit('placeAtlas2').new(limit=2000):
total_all_flairs += 1
if (submission.id in existing_ids):
set_flair(submission, "Processed Entry")
print("Found first duplicate!")
duplicate_count += 1
if (duplicate_count > 0):
break
else:
continue
if submission.link_flair_text == "New Entry" or submission.link_flair_text == "Edit Entry":
try:
text = submission.selftext
rawtext = text
text = text.replace('\u200c', '')
text = re.compile(r"(\{.+\})", re.DOTALL).search(text).group(0)
# Test if it needs to escape the escape character. Usually happens on fancy mode.
try: json.loads(text)
except json.JSONDecodeError: text = re.sub(r"\\(.)", r"\1", text)
submission_json = json.loads(text)
if submission_json:
if submission.link_flair_text == "Edit Entry":
assert submission_json["id"] != 0, "Edit invalid because ID is tampered, it must not be 0!"
submission_json_dummy = {"id": submission_json["id"], "edit": submission.id}
else:
assert submission_json["id"] == 0, "Edit invalid because ID is tampered, it must be 0!"
2023-03-22 15:16:52 +01:00
submission_json_dummy = {"id": submission.id}
2022-06-11 18:39:10 +02:00
for key in submission_json:
if not key in submission_json_dummy:
2022-12-31 21:13:56 +01:00
submission_json_dummy[key] = submission_json[key]
2022-06-11 18:39:10 +02:00
submission_json = format_all(submission_json_dummy, True)
validation_status = validate(submission_json)
assert validation_status < 3, \
"Submission invalid after validation. This may be caused by not enough points on the path."
add_comma_line = len(OUT_FILE_LINES) - 2
if len(OUT_FILE_LINES[add_comma_line]) > 2:
OUT_FILE_LINES[add_comma_line] = OUT_FILE_LINES[add_comma_line].replace('\n', ',\n')
OUT_FILE_LINES.insert(len(OUT_FILE_LINES) - 1, json.dumps(submission_json, ensure_ascii=False) + '\n')
READ_IDS_FILE.write(submission.id + '\n')
successcount += 1
set_flair(submission, "Processed Entry")
except Exception as e:
FAIL_FILE.write(
"\n\n" + "="*40 + "\n\nSubmission ID: " +
submission.id + "\n\n" +
traceback.format_exc() + "\n\n" +
"==== RAW ====" + "\n\n" +
rawtext + "\n\n"
"==== CLEAN ====" + "\n\n" +
text + "\n\n"
)
failcount += 1
set_flair(submission, "Rejected Entry")
print("Wrote " + submission.id + ", submitted " + str(round(time.time()-submission.created_utc)) + " seconds ago")
totalcount += 1
2022-04-04 18:49:57 +02:00
2022-06-11 18:39:10 +02:00
OUT_FILE.writelines(OUT_FILE_LINES)
2023-03-21 13:29:18 +01:00
print(f"\n\nTotal all flairs: {total_all_flairs}\nSuccess: {successcount}/{totalcount}\nFail: {failcount}/{totalcount}\nPlease check temp-atlas-manual.txt for failed entries to manually resolve.")