atlas/tools/redditcrawl.py

"""
Setting up authentication:
1. Head to https://www.reddit.com/prefs/apps
2. Click "are you a developer? create an app..." on the button
3. Enter the name and description
4. Select "script" for the type
5. Enter "redirect uri" as "http://localhost:8080"
6. Create file "credentials" with the format below
┌──────────────────────────────────────────────────────────────────────────────┐
│ [id]        <-  Under "personal use script"                                  │
│ [secret]                                                                     │
│ [username]  <-  For flair access, must be a mod, Don't do this...            │
│ [password]  <-  ...if you don't know what you are doing.                     │
└──────────────────────────────────────────────────────────────────────────────┘

Running:
1. Run the script
2. Input the next ID to use
3. Manually resolve errors in temp-atlas-manual.json
4. a. Use merge_out.py, or...
   b. a. Copy temp-atlas.json entries into web/_js/atlas.js (mind the edits!)
      b. Copy temp-read-ids.txt IDs into data/read-ids.txt
5. Create a pull request
"""

from praw import Reddit
from praw.models import Submission
import json
import re
import traceback
from aformatter import format_all, validate
from pathlib import Path
import humanize
from datetime import datetime
import os

while not os.path.exists('README.md'):
	os.chdir('..')

patches_dir = "data/patches/"
Path(patches_dir).mkdir(parents=True, exist_ok=True)

def set_flair(submission, flair):
	if has_write_access and submission.link_flair_text != flair:
		flair_choices = submission.flair.choices()
		flair = next(x for x in flair_choices if x["flair_text_editable"] and flair == x["flair_text"])
		submission.flair.select(flair["flair_template_id"])


with open('credentials', 'r') as file:
	credentials = file.readlines()
	client_id = credentials[0].strip()
	client_secret = credentials[1].strip()
	username = credentials[2].strip() if len(credentials) > 3 else ""
	password = credentials[3].strip() if len(credentials) > 3 else ""

reddit = Reddit(
	client_id=client_id,
	client_secret=client_secret,
	username=username,
	password=password,
	user_agent='atlas_bot'
)

has_write_access = not reddit.read_only
if not has_write_access:
	print("Warning: No write access. Post flairs will not be updated. Waiting 5 seconds...")
	# time.sleep(5)

print("Running...")

existing_ids = []

with open('data/read-ids.txt', 'r') as edit_ids_file:
	for id in [x.strip() for x in edit_ids_file.readlines()]:
		existing_ids.append(id)

total_all_flairs = 0
count_dupe = 0
count_fail = 0
count_success = 0
count_total = 0

with open('temp-atlas-manual.txt', 'w', encoding='utf-8') as FAIL_FILE:

	submission: Submission
	for submission in reddit.subreddit('placeAtlas2').new(limit=1000):
		total_all_flairs += 1

		print(f"{submission.id}: Submitted {humanize.naturaltime(datetime.utcnow() - datetime.utcfromtimestamp(submission.created_utc))}.")

		# print(patches_dir + 'reddit-' + submission.id + '.json')
		if submission.id in existing_ids or Path(patches_dir + 'reddit-' + submission.id + '.json').is_file():
			set_flair(submission, "Processed Entry")
			print(f"{submission.id}: Submission is a duplicate! Skipped.")
			if (count_dupe == 1):
				print(f"{submission.id}: Second duplicate. Stopped!")
				break
			print(f"{submission.id}: First duplicate. Continue running.")
			count_dupe += 1
			continue

		print(f"{submission.id}: Processing...")

		if submission.link_flair_text == "New Entry" or submission.link_flair_text == "Edit Entry":

			try:

				text = submission.selftext
				rawtext = text

				text = text.replace('\u200c', '')
				text = re.compile(r"(\{.+\})", re.DOTALL).search(text).group(0)
				# Test if it needs to escape the escape character. Usually happens on fancy mode.
				try: json.loads(text)
				except json.JSONDecodeError: text = re.sub(r"\\(.)", r"\1", text)

				submission_json = json.loads(text)

				if submission_json:

					if submission.link_flair_text == "Edit Entry":
						assert submission_json["id"] != -1, "Edit invalid because ID is tampered, it must not be -1!"
					else:
						assert submission_json["id"] == -1, "Addition invalid because ID is tampered, it must be -1!"
						
					submission_json_dummy = {"id": submission_json["id"], "_reddit_id": submission.id, "_author": submission.author.name}

					for key in submission_json:
						if not key in submission_json_dummy:
							submission_json_dummy[key] = submission_json[key]
					submission_json = format_all(submission_json_dummy, True)
					validation_status = validate(submission_json)

					assert validation_status < 3, \
						"Submission invalid after validation. This may be caused by not enough points on the path."
					
					with open(f'{patches_dir}reddit-{submission.id}-{"-".join(submission.name.split()).lower()}.json', 'w', encoding='utf-8') as out_file:
						out_file.write(json.dumps(submission_json, ensure_ascii=False))

					count_success += 1
					set_flair(submission, "Processed Entry")

			except Exception as e:
				FAIL_FILE.write(
					"\n\n" + "="*40 + "\n\nSubmission ID: " +
					submission.id + "\n\n" +
					traceback.format_exc() + "\n\n" +
					"==== RAW ====" + "\n\n" +
					rawtext + "\n\n"
					"==== CLEAN ====" + "\n\n" +
					text + "\n\n"
				)
				count_fail += 1
				set_flair(submission, "Rejected Entry")
				print(f"{submission.id}: Something went wrong! Rejected.")

			count_total += 1
			print(f"{submission.id}: Processed!")

print(f"\n\nTotal all flairs: {total_all_flairs}\nSuccess: {count_success}/{count_total}\nFail: {count_fail}/{count_total}\nPlease check temp-atlas-manual.txt for failed entries to manually resolve.")