Migrate contributors, move contributors to the end, simplify script

Forgot that there won't be submitted_by on the submissions

tools/migrate_atlas_format.py

@@ -6,6 +6,7 @@
 Migrator script from old atlas format to remastered atlas format.
 - center and path: single -> time-specific
 - website and subreddit: single strings -> links object
+- submitted_by -> contributors
 '''
 # 
 
@@ -40,6 +41,7 @@ def per_line_entries(entries: list):
     "links": {},
     "center": {},
     "path": {},
+    "contributors": []
   }
 
   center = entry['center']
@@ -79,7 +81,11 @@ def per_line_entries(entries: list):
     if isinstance(entry["subreddit"], str) and entry["subreddit"]:
       new_entry['links']['subreddit'] = list(map(lambda x: FS_REGEX.sub(r"\1", x), COMMATIZATION.split(entry['subreddit'])))
     del entry['subreddit']
- 
+
+  if "submitted_by" in entry:
+    new_entry['contributors'].append(entry['submitted_by'])
+    del entry['submitted_by']
+  
   entries[index] = {
     **new_entry,
     **entry

tools/redditcrawl.py

@@ -2,13 +2,14 @@
 import json
 import time
 import re
-import os
 import traceback
 from formatter import format_all
 
-outfile = open('temp_atlas.json', 'w', encoding='utf-8')
-editidsfile = open('read-ids-temp.txt', 'w')
-failfile = open('manual_atlas.json', 'w', encoding='utf-8')
+OUT_FILE = open('temp_atlas.json', 'w', encoding='utf-8')
+READ_IDS_FILE = open('read-ids-temp.txt', 'w')
+FAIL_FILE = open('manual_atlas.json', 'w', encoding='utf-8')
+
+OUT_FILE_LINES = ['[\n', ']\n']
 
 with open('credentials', 'r') as file:
 	credentials = file.readlines()
@@ -32,14 +33,20 @@
 
 existing_ids = []
 
-with open('../data/edit-ids.txt', 'r') as edit_ids_file:
-	for id in [x.strip() for x in edit_ids_file.readlines()]:
-		existing_ids.append(id)
-
-with open('../data/edit-ids.txt', 'r') as edit_ids_file:
+with open('../data/read-ids.txt', 'r') as edit_ids_file:
 	for id in [x.strip() for x in edit_ids_file.readlines()]:
 		existing_ids.append(id)
 
+# with open('../web/atlas.json', 'r') as atlas_file:
+# 	atlas_raw: list = json.loads(atlas_file)
+# 	atlas = {}
+# 	atlas_ids = []
+# 	for index in atlas_raw:
+# 		entry = atlas_raw[index]
+# 		id = entry['id']
+# 		del entry['id']
+# 		atlas[id] = entry
+# 		atlas_ids.append(id)
 
 def set_flair(submission, flair):
 	if has_write_access and submission.link_flair_text != flair:
@@ -53,7 +60,6 @@ def set_flair(submission, flair):
 successcount = 0
 totalcount = 0
 
-outfile.write("[\n")
 for submission in reddit.subreddit('placeAtlas2').new(limit=2000):
 	"""
 	Auth setup
@@ -108,38 +114,29 @@ def set_flair(submission, flair):
 
 				if submission.link_flair_text == "Edit Entry":
 
-					assert submission_json["id"] != 0, "ID is tampered, it must not be 0!"
-					submission_json_dummy = {"id": submission_json["id"], "edit": True, "contributors": []}
+					assert submission_json["id"] != 0, "Edit invalid because ID is tampered, it must not be 0!"
 
-					if "submitted_by" in submission_json:
-						submission_json_dummy["contributors"].append(submission_json['submitted_by'])
-						del submission_json['submitted_by']
-					elif "contributors" in submission_json:
-						submission_json_dummy["contributors"] = submission_json["contributors"]
+					submission_json_dummy = {"id": submission_json["id"], "edit": True}
+					submission_json["contributors"] = []
 
 					try:
-						if not submission.author.name in submission_json_dummy:
-							submission_json_dummy["contributors"].append(submission.author.name)
+						if not submission.author.name in submission_json:
+							submission_json["contributors"].append(submission.author.name)
 					except AttributeError:
-						submission_json_dummy["contributors"].append("unknown")
+						pass
 
 				else:
 
-					assert submission_json["id"] == 0, "ID is tampered, it must be 0!"
-					submission_json_dummy = {"id": submission.id, "contributors": []}
-
-					if "submitted_by" in submission_json:
-						submission_json_dummy["contributors"].append(submission_json['submitted_by'])
-						del submission_json['submitted_by']
-					elif "contributors" in submission_json:
-						submission_json_dummy["contributors"] = submission_json["contributors"]
+					assert submission_json["id"] == 0, "Edit invalid because ID is tampered, it must be 0!"
+					
+					submission_json_dummy = {"id": submission.id}
+					submission_json["contributors"] = []
 
 					try:
-						if not submission.author.name in submission_json_dummy:
-							submission_json_dummy["contributors"].append(submission.author.name)
+						if not submission.author.name in submission_json:
+							submission_json["contributors"].append(submission.author.name)
 					except AttributeError:
-						submission_json_dummy["contributors"].append("unknown")
-
+						pass
 
 				for key in submission_json:
 					if not key in submission_json_dummy:
@@ -149,13 +146,13 @@ def set_flair(submission, flair):
 				assert validation_status < 3, \
 					"Submission invalid after validation. This may be caused by not enough points on the path."
 					
-				outfile.write(json.dumps(submission_json, ensure_ascii=False) + ",\n")
-				editidsfile.write(submission.id + '\n')
+				OUT_FILE_LINES.insert(len(OUT_FILE_LINES) - 1, json.dumps(submission_json, ensure_ascii=False) + '\n')
+				READ_IDS_FILE.write(submission.id + '\n')
 				successcount += 1
 				set_flair(submission, "Processed Entry")
 
 		except Exception as e:
-			failfile.write(
+			FAIL_FILE.write(
 				"\n\n" + "="*40 + "\n\n" +
 				submission.id + "\n\n" +
 				traceback.format_exc() + "\n\n" +
@@ -170,10 +167,6 @@ def set_flair(submission, flair):
 		print("Wrote "+submission.id+", submitted "+str(round(time.time()-submission.created_utc))+" seconds ago")
 		totalcount += 1
 
-# Remove last trailing comma
-outfile.seek(outfile.tell()-3, os.SEEK_SET)
-outfile.truncate()
-
-outfile.write("\n]")
+OUT_FILE.writelines(OUT_FILE_LINES)
 
 print(f"\n\nTotal all flairs:{total_all_flairs}\nSuccess: {successcount}/{totalcount}\nFail: {failcount}/{totalcount}\nPlease check manual_atlas.txt for failed entries to manually resolve.")