From 93e80dc7f99e8e75de246f6903290f681d216214 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 7 Apr 2022 17:00:30 -0300 Subject: [PATCH] New submissions, flair editing script --- tools/redditcrawl.py | 29 +++++++--- tools/redditflairset.py | 102 +++++++++++++++++++++++++++++++++++ web/atlas.json | 116 +++++++++++++++++++++++++++++++++++++++- 3 files changed, 238 insertions(+), 9 deletions(-) create mode 100644 tools/redditflairset.py diff --git a/tools/redditcrawl.py b/tools/redditcrawl.py index 1f7b2f9b..7ab6e398 100755 --- a/tools/redditcrawl.py +++ b/tools/redditcrawl.py @@ -11,12 +11,14 @@ credentials = open('credentials', 'r') client_id = credentials.readline().strip(' \t\n\r') client_secret = credentials.readline().strip(' \t\n\r') +user = credentials.readline().strip(' \t\n\r') +pw = credentials.readline().strip(' \t\n\r') -reddit = praw.Reddit(client_id=client_id, client_secret=client_secret, user_agent='atlas_bot') - -failcount = 0 -successcount = 0 -totalcount = 0 +reddit = praw.Reddit(client_id=client_id, client_secret=client_secret, user_agent='atlas_bot',username=user,password=pw) +has_write_access = not reddit.read_only +if not has_write_access: + print("Warning: No write access. Post flairs will not be updated") + sleep(5) jsonfile = open("../web/atlas.json", "r", encoding='utf-8') existing = json.load(jsonfile) @@ -26,8 +28,17 @@ for item in existing: existing_ids.append(item['id']) +def set_flair(submission, flair): + if has_write_access and submission.link_flair_text != flair: + flair_choices = submission.flair.choices() + flair = next(x for x in flair_choices if x["flair_text_editable"] and flair == x["flair_text"]) + submission.flair.select(flair["flair_template_id"]) + total_all_flairs = 0 duplicate_count = 0 +failcount = 0 +successcount = 0 +totalcount = 0 outfile.write("[\n") for submission in reddit.subreddit('placeAtlas2').new(limit=2000): """ @@ -41,7 +52,8 @@ 7. Append to file called "credentials" 8. Copy Secret 9. Append on newline to "credentials" file - 10. Run Script + 10. If you want flair write access append 2 newlines with username and password (Must be a mod, don't do this if you don't know what you're doing) + 11. Run Script Running Script 1. Input the next ID to use @@ -52,9 +64,10 @@ """ total_all_flairs += 1 if (submission.id in existing_ids): + set_flair(submission, "Processed Entry") print("Found first duplicate!") duplicate_count += 1 - if (duplicate_count > 10): + if (duplicate_count > 0): break else: continue @@ -82,9 +95,11 @@ try: outfile.write(json.dumps(json.loads(text))+" ,\n") successcount += 1 + set_flair(submission, "Processed Entry") except json.JSONDecodeError: failfile.write(text+",\n") failcount += 1 + set_flair(submission, "Rejected Entry") print("written "+submission.id+" submitted "+str(round(time.time()-submission.created_utc))+" seconds ago") totalcount += 1 diff --git a/tools/redditflairset.py b/tools/redditflairset.py new file mode 100644 index 00000000..ea913356 --- /dev/null +++ b/tools/redditflairset.py @@ -0,0 +1,102 @@ +# Script to retroactively fix flairs +# Only touches things flaired "New entry" that either fail JSON parsing or are already in the atlas +# Otherwise, it leaves them untouched + +import praw +import json +import time +import re +import os + +credentials = open('credentials', 'r') +client_id = credentials.readline().strip(' \t\n\r') +client_secret = credentials.readline().strip(' \t\n\r') +user = credentials.readline().strip(' \t\n\r') +pw = credentials.readline().strip(' \t\n\r') + +reddit = praw.Reddit(client_id=client_id, client_secret=client_secret, user_agent='atlas_bot',username=user,password=pw) +has_write_access = not reddit.read_only +if not has_write_access: + print("No write access, exiting without doing anything") + quit() + +jsonfile = open("../web/atlas.json", "r", encoding='utf-8') +existing = json.load(jsonfile) + +existing_ids = [] + +for item in existing: + existing_ids.append(item['id']) + +def set_flair(submission, flair): + if has_write_access and submission.link_flair_text != flair: + print(submission.link_flair_text) + print(submission.id) + print(flair) + flair_choices = submission.flair.choices() + flair = next(x for x in flair_choices if x["flair_text_editable"] and flair == x["flair_text"]) + submission.flair.select(flair["flair_template_id"]) + return 1 + return 0 + +total_all_flairs = 0 +rejected_count = 0 +processed_count = 0 +#for submission in reddit.subreddit('placeAtlas2').new(limit=1000): +#for submission in reddit.subreddit('placeAtlas2').search('flair:"New Entry"',limit=1000,syntax='lucene', sort="top"): +#for submission in reddit.subreddit('placeAtlas2').search('flair:"New Entry"',limit=1000,syntax='lucene', sort="comments"): +#for submission in reddit.subreddit('placeAtlas2').search('flair:"New Entry"',limit=1000,syntax='lucene', sort="hot"): +#for submission in reddit.subreddit('placeAtlas2').search('flair:"New Entry"',limit=1000,syntax='lucene', sort="new"): +#for submission in reddit.subreddit('placeAtlas2').search('flair:"New Entry"',limit=1000,syntax='lucene', sort="relevance"): + """ + Auth setup + 1. Head to https://www.reddit.com/prefs/apps + 2. Click "create another app" + 3. Give it a name and description + 4. Select "script" + 5. Redirect to http://localhost:8080 + 6. Copy ID (under Personal Use Script) + 7. Append to file called "credentials" + 8. Copy Secret + 9. Append on newline to "credentials" file + 10-. Append 2 newlines with username and password if you want flair write access + 11. Run Script + + Running Script + 1. Input the next ID to use + 2. Manually resolve errors in manual_atlas.json + 3. Copy temp_atlas.json entries into web/_js/atlas.js + 4. Pull Request + + """ + total_all_flairs += 1 + if (submission.id in existing_ids): + processed_count += set_flair(submission, "Processed Entry") + continue + if(submission.link_flair_text == "New Entry"): + text = submission.selftext + #Old backslash filter: + #text = text.replace("\\", "") + #New one: One \\ escapes a backslash in python's parser + # Two escape it again in the regex parser, so \\\\ is \ + # Then anything but " or n is replaced with the first capture group (anything but " or n) + # Test in repl: re.sub("\\\\([^\"n])", "\\1", "\\t < removed slash, t stays and > stays \\n \\\"") + text = re.sub("\\\\([^\"n])", "\\1", text) + try: + text = text.replace("\"id\": 0,", "\"id\": 0,\n\t\t\"submitted_by\": \""+submission.author.name+"\",") + except AttributeError: + text = text.replace("\"id\": 0,", "\"id\": 0,\n\t\t\"submitted_by\": \""+"unknown"+"\",") + + lines = text.split("\n") + + for i, line in enumerate(lines): + if("\"id\": 0" in line): + lines[i] = line.replace("\"id\": 0", "\"id\": "+"\""+str(submission.id)+"\"") + text = "\n".join(lines) + try: + json.dumps(json.loads(text)) + # Do not set processed, we're only updating old entries in atlas or invalid submission flairs and not processing + except json.JSONDecodeError: + rejected_count += set_flair(submission, "Rejected Entry") + +print(f"\n\nTotal all flairs:{total_all_flairs}\nUpdated as rejected: {rejected_count}/{total_all_flairs}\nUpdated as processed: {processed_count}/{total_all_flairs}\n") diff --git a/web/atlas.json b/web/atlas.json index 36cb6368..59229f34 100644 --- a/web/atlas.json +++ b/web/atlas.json @@ -6885,7 +6885,6 @@ {"id": "txuuvt", "submitted_by": "K1t_Cat", "name": "Remnants of the old Greek flag", "description": "", "website": "", "subreddit": "", "center": [358.5, 165.5], "path": [[348.5, 161.5], [367.5, 161.5], [367.5, 168.5], [348.5, 168.5], [348.5, 161.5]]}, {"id": "txuuo2", "submitted_by": "yotsubashi19", "name": "Pomu Rainpuff", "description": "A member of LazuLight, the first wave of VTubers from NIJISANJI's English branch. She's Pomu!", "website": "https://www.youtube.com/channel/UCP4nMSTdwU1KqYWu3UH5DHQ", "subreddit": "/r/Nijisanji", "center": [1892.5, 739.5], "path": [[1893.5, 743.5], [1886.5, 743.5], [1885.5, 741.5], [1888.5, 738.5], [1887.5, 735.5], [1890.5, 734.5], [1892.5, 738.5], [1894.5, 735.5], [1894.5, 734.5], [1896.5, 734.5], [1897.5, 735.5], [1895.5, 737.5], [1900.5, 741.5], [1899.5, 743.5]]}, {"id": "txuudl", "submitted_by": "Difficult_Basil2254", "name": "canary island flag", "description": "The Canary Islands are an archipelago located in the Atlantic Ocean that forms a Spanish autonomous community in Northwest Africa, with the status of historical nationality. It is also one of the outermost regions of the European Union.", "website": "https://es.wikipedia.org/wiki/Canarias", "subreddit": "", "center": [1070.5, 1376.5], "path": [[1066.5, 1368.5], [1065.5, 1386.5], [1076.5, 1368.5], [1075.5, 1369.5], [1065.5, 1368.5], [1065.5, 1387.5], [1076.5, 1386.5], [1076.5, 1368.5], [1076.5, 1368.5], [1070.5, 1368.5]]}, -{"id": "txutus", "submitted_by": "ShawermaBox", "name": "Vandalism", "description": "Previously, this space was occupied by an Israeli flag, but changed to a Palestinian flag, then back to an Israeli flag, and then finally to a Palestinian flag. Pro-Israel & pro-Palestine leaders came to an agreement to not attack one another afterwards. Yet, one blue dot remains - which was placed in a portion of the Palestinian flag which is all white (which is also all white on an Israeli flag), in an effort to vandalize it. ", "website": "", "subreddit": "", "center": [56.5, 661.5], "path": [[55.5, 660.5], [57.5, 660.5], [57.5, 662.5], [55.5, 662.5], [55.5, 660.5], [55.5, 660.5]]}, {"id": "txusul", "submitted_by": "AccordingClassroom30", "name": "Slovenian Flag", "description": "A shorter version was placed in the early stages of Place and was expanded to it's final lenght towards the end of the first day. Altho tryes to expand the white part of the flag were halted multiple times by Germany, an agrerement was proposed and accepted to have Bernd das Brot's beer foam spill into the white part of the flag. It was briefly invaded by the Cool S pattern after the second expansion, but quickly reconstructed and additional pixel art was added to the flag.", "website": "", "subreddit": "/r/slovenia", "center": [288.5, 879.5], "path": [[216.5, 868.5], [216.5, 888.5], [353.5, 888.5], [353.5, 886.5], [354.5, 885.5], [355.5, 884.5], [356.5, 883.5], [357.5, 883.5], [358.5, 884.5], [358.5, 868.5], [297.5, 868.5], [297.5, 874.5], [296.5, 875.5], [295.5, 875.5], [294.5, 876.5], [293.5, 877.5], [291.5, 879.5], [289.5, 879.5], [288.5, 880.5], [287.5, 880.5], [286.5, 881.5], [282.5, 881.5], [281.5, 882.5], [280.5, 882.5], [279.5, 883.5], [278.5, 883.5], [277.5, 884.5], [273.5, 884.5], [272.5, 883.5], [271.5, 883.5], [270.5, 882.5], [269.5, 882.5], [268.5, 881.5], [264.5, 881.5], [263.5, 880.5], [262.5, 880.5], [261.5, 879.5], [259.5, 879.5], [258.5, 878.5], [255.5, 875.5], [255.5, 875.5], [254.5, 875.5], [253.5, 874.5], [253.5, 865.5], [238.5, 865.5], [237.5, 866.5], [237.5, 867.5], [236.5, 868.5], [236.5, 869.5], [235.5, 870.5], [229.5, 870.5], [228.5, 869.5], [227.5, 869.5], [226.5, 868.5], [225.5, 867.5], [218.5, 867.5], [218.5, 868.5], [216.5, 868.5]]}, {"id": "txusjm", "submitted_by": "baga1706", "name": "TTC", "description": "The most powerful school in the world\nSCI