Merge branch 'remaster' into new-center-algorithm

_headers

@@ -1,2 +0,0 @@
-/*
-  Access-Control-Allow-Origin: *

data/read-ids.txt

@@ -8513,3 +8513,341 @@ u0ny8w
 u0nx4x
 u0nvr4
 u0nu1q
+u1i035
+u1huft
+u1hkm0
+u1hi3b
+u1h5qv
+u1eptq
+u1dxtp
+u1dl6n
+u1cstp
+u1c1d4
+u1aovf
+u1ajro
+u1aibg
+u1agdn
+u1afna
+u1acfb
+u1abm9
+u1aara
+u1a9u3
+u1a8dm
+u1a7qf
+u1a31o
+u1a0xi
+u1a0rt
+u19sp2
+u16zq6
+u16onh
+u15lrh
+u13gcr
+u139b1
+u12oy9
+u12m8r
+u127qi
+u117oa
+u115md
+u2e2d7
+u2e1hr
+u2dy4y
+u2dujr
+u2dtb7
+u2dpox
+u2dp8n
+u2doq2
+u2dmza
+u2dm46
+u2djd0
+u2dhau
+u2dg9w
+u2dfsi
+u2df59
+u2dekn
+u2d62x
+u2cq4k
+u2cp4v
+u2cogz
+u2cnvv
+u2cn67
+u2camx
+u2c9me
+u2c8m8
+u2buo2
+u2bu9k
+u2bu60
+u2bdk8
+u2b99z
+u2b627
+u2axzg
+u2axil
+u2arkb
+u2aqvz
+u2aopl
+u2a3ui
+u290gp
+u27syy
+u27rx8
+u27p92
+u27ld9
+u26hqz
+u26hic
+u25i81
+u25bkn
+u253rh
+u24xi0
+u24svk
+u24f09
+u24d19
+u24a2b
+u2486i
+u246v6
+u23xjv
+u236bp
+u2227k
+u20hgo
+u20gqu
+u20fxh
+u20f2a
+u20ben
+u1zym3
+u1wqwo
+u1tlgc
+u1svlz
+u1srpt
+u1rvlg
+u1ruj9
+u1rtir
+u1rsj9
+u1rlsl
+u1r97p
+u1r8tf
+u1r86b
+u1qagw
+u1q68v
+u1q5ym
+u1q0qk
+u1q086
+u1pxv3
+u1ptws
+u1psuv
+u1plds
+u1pkde
+u1pjcr
+u1pimz
+u1pibv
+u1phc8
+u1pga3
+u1olw3
+u1odif
+u1o1qg
+u1o12z
+u1o0c6
+u1nzqx
+u1nz1l
+u1nyer
+u1nxoh
+u1nwyr
+u1nw8o
+u1nv98
+u1nqhu
+u1nkb6
+u1nhvc
+u1ng54
+u1nfcm
+u1nc6b
+u1nb69
+u1n93d
+u1n10d
+u1mj4t
+u1miou
+u1mgnt
+u1ma6t
+u1lzqc
+u1ljm9
+u1l8dx
+u1kqsq
+u1k5l6
+u1jcms
+u1j7kb
+u1j2if
+u1in4u
+u1icud
+u2fp8n
+u314dc
+u2ziyc
+u2zhof
+u2zeqw
+u2zbxw
+u2zau5
+u2z9u9
+u2z8vq
+u2z7yd
+u2z5w0
+u2z3h7
+u2z1fk
+u2z0i7
+u2yyuq
+u2yj7i
+u2y7tn
+u2wno2
+u2w68c
+u2w5d9
+u2vtk6
+u2vsc5
+u2vqdl
+u2veh4
+u2vccn
+u2upjt
+u2teqf
+u2ta3t
+u2scps
+u2s6ko
+u2s2hq
+u2rp04
+u2oe4z
+u2nm4h
+u2mglu
+u2mes6
+u2kgt0
+u2kgkx
+u2kgam
+u2jx5f
+u2jwm8
+u2jw9y
+u2jv33
+u2jpqe
+u2j9e2
+u2j34d
+u2isv1
+u2iljx
+u2ikti
+u2id8k
+u2iar2
+u2iaaj
+u2i8dt
+u2i77s
+u2i6pj
+u2i52l
+u2i4c8
+u2i3n3
+u2i2s5
+u2i2ay
+u2i18l
+u2hx8q
+u2hw35
+u2hq4n
+u2hoxw
+u2hndh
+u2hmv9
+u2hjn8
+u2hfb1
+u2heem
+u2h3d5
+u2h2uj
+u2h2ba
+u2gxgd
+u2gmz0
+u2gkze
+u2gjfb
+u2ggwy
+u2gdu6
+u2g3ho
+u39w0p
+u3956r
+u38yox
+u38y8w
+u38xtp
+u38x97
+u38wn0
+u38w7l
+u38siz
+u38rko
+u38glf
+u38fnb
+u38eza
+u38b71
+u38ang
+u38a1v
+u389gh
+u388kv
+u385vc
+u3856f
+u384h1
+u383k2
+u382nl
+u381h1
+u37bs2
+u37bal
+u37b2x
+u37abx
+u37196
+u3703q
+u36yic
+u36w1e
+u36k9o
+u360ch
+u35z7z
+u35y99
+u352sv
+u35255
+u3505v
+u34wnr
+u343uv
+u342ys
+u3426y
+u341c5
+u33yqz
+u33y72
+u33tjn
+u33syf
+u33n2r
+u40u61
+u40diy
+u402pe
+u3zfyg
+u3zbgo
+u3z8s5
+u3z0ym
+u3yzud
+u3yvxk
+u3yuzv
+u3ytml
+u3ysji
+u3yepe
+u3xupi
+u3xn44
+u3xlz8
+u3wnsl
+u3wmdt
+u3wlmr
+u3wkqe
+u3wigu
+u3wf0o
+u3wcn9
+u3wbpf
+u3umji
+u3uftd
+u3ueag
+u3udwc
+u3u038
+u3sxkx
+u3sq78
+u3qf4m
+u3pe7k
+u3pc5u
+u3o3ls
+u3nhfo
+u3lnul
+u3llih
+u3kmx3
+u3kftg
+u3ir6q
+u3grqq
+u3ghal
+u3gei4
+u3gcsl
+u3ga5g
+u3g3d6
+u3bdkp
+u39z7g

netlify.toml

@@ -1,4 +0,0 @@
-[[headers]]
-  for = "/*"
-  [headers.values]
-    Access-Control-Allow-Origin = "*"

tools/formatter.py

@@ -28,6 +28,7 @@
 	"pattern1user": r'\/*(?:u|user)\/([A-Za-z0-9][A-Za-z0-9_]{1,20})(?:\/$)?',
 	"pattern2user": r'^\/*(?:u|user)(?!\/)([A-Za-z0-9][A-Za-z0-9_]{1,20})(?:\/$)?',
 	"pattern3user": r'(?:(?:https?:\/\/)?(?:(?:www|old|new|np)\.)?)?reddit\.com\/(?:u|user)\/([A-Za-z0-9][A-Za-z0-9_]{1,20})(?:\/[^" ]*)*',
+	"pattern1new": r'(?:(?:(?:(?:https?:\/\/)?(?:(?:www|old|new|np)\.)?)?reddit\.com)?\/)?[rR]\/([A-Za-z0-9][A-Za-z0-9_]{1,20})(?:\/[^" ]*)*'
 	# "pattern4": r'(?:https?:\/\/)?(?!^www\.)(.+)\.reddit\.com(?:\/[^"]*)*',
 	# "pattern5": r'\[(?:https?:\/\/)?(?!^www\.)(.+)\.reddit\.com(?:\/[^"]*)*\]\((?:https:\/\/)?(?!^www\.)(.+)\.reddit\.com(?:\/[^"]*)*\)"',
 }
@@ -55,10 +56,11 @@ def format_subreddit(entry: dict):
 	"""
 	Fix formatting of the value on "subreddit".
 	"""
-	if not "subreddit" in entry or not entry['subreddit']:
-		return entry
+
+	if "subreddit" in entry and entry["subreddit"]:
 
 		subredditLink = entry["subreddit"]
+
 		subredditLink = re.sub(FS_REGEX["commatization"], ', ', subredditLink)
 		subredditLink = re.sub(FS_REGEX["pattern3"], SUBREDDIT_TEMPLATE, subredditLink)
 		subredditLink = re.sub(FS_REGEX["pattern1"], SUBREDDIT_TEMPLATE, subredditLink)
@@ -67,15 +69,30 @@ def format_subreddit(entry: dict):
 		subredditLink = re.sub(FS_REGEX["pattern1user"], USER_TEMPLATE, subredditLink)
 		subredditLink = re.sub(FS_REGEX["pattern2user"], USER_TEMPLATE, subredditLink)
 
-	if not subredditLink:
-		return entry
-	
 		entry["subreddit"] = subredditLink
+
+	if "links" in entry and "subreddit" in entry["links"]:
+
+		for i in range(len(entry["links"]["subreddit"])):
+
+			subredditLink = entry["links"]["subreddit"][i]
+
+			subredditLink = re.sub(FS_REGEX["pattern3"], r"\1", subredditLink)
+			subredditLink = re.sub(FS_REGEX["pattern1new"], r"\1", subredditLink)
+
+			entry["links"]["subreddit"][i] = subredditLink
+
 	return entry
 
 def collapse_links(entry: dict):
+	"""
+	Collapses Markdown links.
+	"""
+
 	if "website" in entry and entry['website']:
-		website = entry["website"];
+
+		website = entry["website"]
+
 		if re.search(CL_REGEX, website):
 			match = re.search(CL_REGEX, website)
 			if match.group(1) == match.group(2):
@@ -83,8 +100,23 @@ def collapse_links(entry: dict):
 
 		entry["website"] = website
 
+	elif "links" in entry and "website" in entry["links"]:
+
+		for i in range(len(entry["links"]["website"])):
+
+			website = entry["links"]["website"][i]
+
+			if re.search(CL_REGEX, website):
+				match = re.search(CL_REGEX, website)
+				if match.group(1) == match.group(2):
+					website = match.group(2)
+
+			entry["links"]["website"][i] = website
+
 	if "subreddit" in entry and entry['subreddit']:
-		subreddit = entry["subreddit"];
+
+		subreddit = entry["subreddit"]
+
 		if re.search(CL_REGEX, subreddit):
 			match = re.search(CL_REGEX, subreddit)
 			if match.group(1) == match.group(2):
@@ -92,12 +124,27 @@ def collapse_links(entry: dict):
 
 		entry["subreddit"] = subreddit
 
+	elif "links" in entry and "subreddit" in entry["links"]:
+
+		for i in range(len(entry["links"]["subreddit"])):
+
+			subreddit = entry["links"]["subreddit"][i]
+
+			if re.search(CL_REGEX, subreddit):
+				match = re.search(CL_REGEX, subreddit)
+				if match.group(1) == match.group(2):
+					subreddit = match.group(2)
+
+			entry["links"]["subreddit"][i] = subreddit
+	
+
 	return entry
 
 def remove_extras(entry: dict):
 	"""
 	Removing unnecessary extra characters and converts select characters.
 	"""
+
 	if "subreddit" in entry and entry["subreddit"]:
 		# if not entry["subreddit"].startswith('/r/'):
 		# 	entry["subreddit"] = re.sub(r'^(.*)(?=\/r\/)', r'', entry["subreddit"])
@@ -127,6 +174,11 @@ def remove_duplicate_points(entry: dict):
 	"""
 	Removes points from paths that occur twice after each other
 	"""
+
+	if not "path" in entry:
+		return entry
+
+	if isinstance(entry['path'], list):
 		path: list = entry['path']
 		previous: list = path[0]
 		for i in range(len(path)-1, -1, -1):
@@ -134,6 +186,15 @@ def remove_duplicate_points(entry: dict):
 			if current == previous:
 				path.pop(i)
 			previous = current
+	else:
+		for key in entry['path']:
+			path: list = entry['path'][key]
+			previous: list = path[0]
+			for i in range(len(path)-1, -1, -1):
+				current: list = path[i]
+				if current == previous:
+					path.pop(i)
+				previous = current
 
 	return entry
 
@@ -141,6 +202,7 @@ def fix_r_caps(entry: dict):
 	"""
 	Fixes capitalization of /r/. (/R/place -> /r/place)
 	"""
+
 	if not "description" in entry or not entry['description']:
 		return entry
 	
@@ -153,9 +215,12 @@ def fix_no_protocol_urls(entry: dict):
 	"""
 	Fixes URLs with no protocol by adding "https://" protocol.
 	"""
-	if not "website" in entry or not entry['website']:
-		return entry
 
+	if "links" in entry and "website" in entry['links']:
+		for i in range(len(entry["links"]["website"])):
+			if entry["links"]["website"][i] and not entry["links"]["website"][i].startswith("http"):
+				entry["links"]["website"][i] = "https://" + entry["website"]
+	elif "website" in entry and not entry['website']:
 		if not entry["website"].startswith("http"):
 			entry["website"] = "https://" + entry["website"]
 
@@ -165,9 +230,29 @@ def convert_website_to_subreddit(entry: dict):
 	"""
 	Converts the subreddit link on "website" to "subreddit" if possible.
 	"""
-	if not "website" in entry or not entry['website']:
-		return entry
 
+	if "links" in entry and "website" in entry["links"]:
+		for i in range(len(entry["links"]["website"])):
+			if re.match(CWTS_REGEX["url"], entry["links"]["website"][i]):
+				new_subreddit = re.sub(CWTS_REGEX["url"], r"\1", entry["links"]["website"][i])
+				if new_subreddit in entry["links"]["subreddit"]:
+					entry["links"]["website"][i] = ""
+				elif not "subreddit" in entry["links"] or len(entry["subreddit"]) == 0:
+					if not "subreddit" in entry["links"]:
+						entry["links"]["subreddit"] = []
+					entry["links"]["subreddit"].append(new_subreddit)
+					entry["links"]["website"][i] = ""
+			elif re.match(CWTS_REGEX["subreddit"], entry["links"]["website"][i]):
+				new_subreddit = re.sub(CWTS_REGEX["subreddit"], r"\1", entry["links"]["website"][i])
+				if new_subreddit in entry["links"]["subreddit"]:
+					entry["links"]["website"][i] = ""
+				elif not "subreddit" in entry["links"] or len(entry["subreddit"]) == 0:
+					if not "subreddit" in entry["links"]:
+						entry["links"]["subreddit"] = []
+					entry["links"]["subreddit"].append(new_subreddit)
+					entry["links"]["website"][i] = ""
+
+	elif "website" in entry and entry['website']:
 		if re.match(CWTS_REGEX["url"], entry["website"]):
 			new_subreddit = re.sub(CWTS_REGEX["url"], SUBREDDIT_TEMPLATE, entry["website"])
 			if (new_subreddit.lower() == entry["subreddit"].lower()):
@@ -189,9 +274,26 @@ def convert_subreddit_to_website(entry: dict):
 	"""
 	Converts the links on "subreddit" to a "website" if needed. This also supports Reddit users (/u/reddit). 
 	"""
-	if not "subreddit" in entry or not entry['subreddit']:
-		return entry
 
+	if "links" in entry and "subreddit" in entry["links"]:
+		for i in range(len(entry["links"]["subreddit"])):
+			if re.match(CSTW_REGEX["website"], entry["links"]["subreddit"][i]):
+				if "website" in entry["links"] and entry["links"]["subreddit"][i] in entry["links"]["website"]:
+					entry["links"]["subreddit"][i] = ""
+				elif not "website" in entry["links"] or len(entry["website"]) == 0:
+					if not "website" in entry["links"]:
+						entry["links"]["website"] = []
+					entry["website"].append(entry["links"]["subreddit"][i])
+					entry["links"]["subreddit"][i] = ""
+			elif re.match(CSTW_REGEX["user"], entry["links"]["subreddit"][i]):
+				if not "website" in entry["links"] or len(entry["website"]) == 0:
+					username = re.match(CSTW_REGEX["user"], entry["links"]["subreddit"][i]).group(1)
+					if not "website" in entry["links"]:
+						entry["links"]["website"] = []
+					entry["website"].append("https://www.reddit.com/user/" + username)
+					entry["links"]["subreddit"][i] = ""
+
+	elif "subreddit" in entry and entry['subreddit']:
 		if re.match(CSTW_REGEX["website"], entry["subreddit"]):
 			if (entry["website"].lower() == entry["subreddit"].lower()):
 				entry["subreddit"] = ""
@@ -215,17 +317,40 @@ def calculate_center(path: list):
 
 def update_center(entry: dict):
 	"""
-	checks if the center of a entry is up to date, and updates it if it's either missing or outdated
+	checks if the center of a entry is up to date, and updates it if it's either missing or outdated.
 	"""
+	
 	if 'path' not in entry:
 		return entry
+
+	if isinstance(entry['path'], list):
 		path = entry['path']
 		if len(path) > 1:
 			calculated_center = calculate_center(path)
 			if 'center' not in entry or entry['center'] != calculated_center:
 				entry['center'] = calculated_center
+	else:
+		for key in entry['path']:
+			path = entry['path'][key]
+			if len(path) > 1:
+				calculated_center = calculate_center(path)
+				if 'center' not in entry or key not in entry['center'] or entry['center'][key] != calculated_center:
+					entry['center'][key] = calculated_center
+	
 	return entry
 
+def remove_empty_and_similar(entry: dict):
+	"""
+	Removes empty items on lists, usually from the past formattings.
+	"""
+
+	for key in entry["links"]:
+		small = list(map(lambda x: x.lower(), entry["links"][key]))
+		entry["links"][key] = [x for x in entry["links"][key] if x and x.lower() in small]
+
+	return entry
+
+
 def validate(entry: dict):
 	"""
 	Validates the entry. Catch errors and tell warnings related to the entry.
@@ -236,17 +361,34 @@ def validate(entry: dict):
 	2: Warnings that may effect user experience when interacting with the entry
 	3: Errors that make the entry inaccessible or broken.
 	"""
+	
 	return_status = 0
 	if (not "id" in entry or (not entry['id'] and not entry['id'] == 0)):
 		print(f"Wait, no id here! How did this happened? {entry}")
 		return_status = 3
 		entry['id'] = '[MISSING_ID]'
-	if not ("path" in entry and isinstance(entry["path"], list) and len(entry["path"]) > 0):
+
+	if "path" in entry:
+		if isinstance(entry['path'], list):
+			if len(entry["path"]) > 0:
 				print(f"Entry {entry['id']} has no points!")
 				return_status = 3
 			elif len(entry["path"]) < 3:
 				print(f"Entry {entry['id']} only has {len(entry['path'])} point(s)!")
 				return_status = 3
+		else:
+			for key in entry['path']:
+				path = entry['path'][key]
+				if len(path) > 0:
+					print(f"Period {key} of entry {entry['id']} has no points!")
+					return_status = 3
+				elif len(path) < 3:
+					print(f"Period {key} of entry {entry['id']} only has {len(entry['path'])} point(s)!")
+					return_status = 3
+	else:
+		print(f"Entry {entry['id']} has no path at all!")
+		return_status = 3
+
 	for key in entry:
 		if key in VALIDATE_REGEX and not re.match(VALIDATE_REGEX[key], entry[key]):
 			if return_status < 2: return_status = 2
@@ -296,6 +438,8 @@ def print_(*args, **kwargs):
 	entry = remove_duplicate_points(entry)
 	print_("Updating center...")
 	entry = update_center(entry)
+	print_("Remove empty items...")
+	entry = remove_empty_and_similar(entry)
 	print_("Validating...")
 	status_code = validate(entry)
 	print_("Completed!")

tools/merge_out.py

@@ -0,0 +1,52 @@
+import praw
+import json
+import time
+import re
+import os
+import traceback
+from formatter import format_all, per_line_entries
+
+out_ids = []
+out_dupe_ids = []
+out_edited_added_ids = []
+atlas_ids = []
+
+with open('temp_atlas.json', 'r', encoding='utf-8') as out_file:
+	out_json = json.loads(out_file.read())
+
+with open('../web/atlas.json', 'r', encoding='utf-8') as atlas_file:
+	atlas_json = json.loads(atlas_file.read())
+
+for entry in atlas_json:
+	atlas_ids.append(entry['id'])
+
+for entry in out_json:
+	if (entry['id'] in out_ids):
+		print(f"Entry {entry['id']} has duplicates! Please resolve this conflict. This will be excluded from the merge.")
+		out_dupe_ids.append(entry['id'])
+	out_ids.append(entry['id'])
+	
+for entry in out_json:
+	if entry['id'] in out_dupe_ids:
+		continue
+
+	if ('edit' in entry and entry['edit']) or entry['id'] in out_ids:
+		index = next((i for i, item in enumerate(atlas_json) if item["id"] == entry['id']), None)
+		if 'edit' in entry: 
+			out_edited_added_ids.append(entry['edit'])
+			del entry['edit']
+		if 'submitted_by' in atlas_json[index]:
+			atlas_json[index].contributors = [ atlas_json[index]['submitted_by'] ]
+		entry['contributors'] = atlas_json[index]['contributors'] + list(set(entry['contributors']) - set(atlas_json[index]['contributors']))
+		atlas_json[index] = entry
+	else:
+		atlas_json.append(entry)
+
+print('Writing...')
+with open('../web/atlas.json', 'w', encoding='utf-8') as atlas_file:
+	atlas_file.write(per_line_entries(atlas_json))
+
+with open('../data/edit-ids.txt', 'a', encoding='utf-8') as edit_ids_file:
+	edit_ids_file.write('\n'.join(out_edited_added_ids) + '\n')
+
+print('All done.')

tools/redditcrawl.py

@@ -36,6 +36,11 @@
 	for id in [x.strip() for x in edit_ids_file.readlines()]:
 		existing_ids.append(id)
 
+with open('../data/edit-ids.txt', 'r') as edit_ids_file:
+	for id in [x.strip() for x in edit_ids_file.readlines()]:
+		existing_ids.append(id)
+
+
 def set_flair(submission, flair):
 	if has_write_access and submission.link_flair_text != flair:
 		flair_choices = submission.flair.choices()
@@ -84,7 +89,7 @@ def set_flair(submission, flair):
 		else:
 			continue
 	
-	if (submission.link_flair_text == "New Entry"):
+	if submission.link_flair_text == "New Entry" or submission.link_flair_text == "Edit Entry":
 
 		try:
 
@@ -101,11 +106,41 @@ def set_flair(submission, flair):
 
 			if submission_json:
 
-				submission_json_dummy = {"id": submission.id, "submitted_by": ""}
+				if submission.link_flair_text == "Edit Entry":
+
+					assert submission_json["id"] != 0, "ID is tampered, it must not be 0!"
+					submission_json_dummy = {"id": submission_json["id"], "edit": True, "contributors": []}
+
+					if "submitted_by" in submission_json:
+						submission_json_dummy["contributors"].append(submission_json['submitted_by'])
+						del submission_json['submitted_by']
+					elif "contributors" in submission_json:
+						submission_json_dummy["contributors"] = submission_json["contributors"]
+
 					try:
-					submission_json_dummy["submitted_by"] = submission.author.name
+						if not submission.author.name in submission_json_dummy:
+							submission_json_dummy["contributors"].append(submission.author.name)
 					except AttributeError:
-					submission_json_dummy["submitted_by"] = "unknown"
+						submission_json_dummy["contributors"].append("unknown")
+
+				else:
+
+					assert submission_json["id"] == 0, "ID is tampered, it must be 0!"
+					submission_json_dummy = {"id": submission.id, "contributors": []}
+
+					if "submitted_by" in submission_json:
+						submission_json_dummy["contributors"].append(submission_json['submitted_by'])
+						del submission_json['submitted_by']
+					elif "contributors" in submission_json:
+						submission_json_dummy["contributors"] = submission_json["contributors"]
+
+					try:
+						if not submission.author.name in submission_json_dummy:
+							submission_json_dummy["contributors"].append(submission.author.name)
+					except AttributeError:
+						submission_json_dummy["contributors"].append("unknown")
+
+
 				for key in submission_json:
 					if not key in submission_json_dummy:
 						submission_json_dummy[key] = submission_json[key];

web/_headers

@@ -0,0 +1,11 @@
+/*
+  Access-Control-Allow-Origin: *
+  
+/_img/place/*.png
+  cache-control: public, max-age=604800
+
+/_img/canvas/*/*.png
+  cache-control: public, max-age=604800
+
+/_img/canvas/*.png
+  cache-control: public, max-age=604800