Only extend on crawls

tools/formatter.py

@@ -370,17 +370,11 @@ def per_line_entries(entries: list):
 def format_all(entry: dict, silent=False):
 	"""
 	Format using all the available formatters.
-	Outputs a tuple containing the entry and the validation status code.
-
-	Status code key:
-	0: All valid, no problems
-	1: Informational logs that may be ignored
-	2: Warnings that may effect user experience when interacting with the entry
-	3: Errors that make the entry inaccessible or broken.
 	"""
 	def print_(*args, **kwargs):
 		if not silent:
 			print(*args, **kwargs)
+
 	print_("Fixing r/ capitalization...")
 	entry = fix_r_caps(entry)
 	print_("Fix formatting of subreddit...")
@@ -403,15 +397,26 @@ def print_(*args, **kwargs):
 	entry = remove_empty_and_similar(entry)
 	print_("Sorting image keys...")
 	entry = sort_image_keys(entry)
-	print_("Extending entries to whiteout...")
-	entry = extend_entries_to_whiteout(entry)
 	print_("Flooring points...")
 	entry = floor_points(entry)
 
-	print_("Validating...")
-	status_code = validate(entry)
 	print_("Completed!")
-	return ( entry, status_code )
+	return entry
+
+def format_all_crawl(entry: dict, silent=False):
+	"""
+	Format using all the available formatters.
+	"""
+	def print_(*args, **kwargs):
+		if not silent:
+			print(*args, **kwargs)
+			
+	format_all(entry, silent)
+	print_("Extending entries to whiteout...")
+	entry = extend_entries_to_whiteout(entry)
+
+	print_("Completed!")
+	return entry
 
 if __name__ == '__main__':
 
@@ -424,7 +429,8 @@ def go(path):
 
 		for i in range(len(entries)):
 			try:
-				entry_formatted, validation_status = format_all(entries[i], True)
+				entry_formatted = format_all(entries[i], True)
+				validation_status = validate(entries[i])
 				if validation_status > 2:
 					print(f"Entry {entry_formatted['id']} will be removed! {json.dumps(entry_formatted)}")
 					entries[i] = None

tools/redditcrawl.py

@@ -28,7 +28,7 @@
 import time
 import re
 import traceback
-from formatter import format_all
+from formatter import format_all_crawl, validate
 from migrate_atlas_format import migrate_atlas_format
 
 OUT_FILE = open('temp_atlas.json', 'w', encoding='utf-8')
@@ -119,7 +119,8 @@ def set_flair(submission, flair):
 				for key in submission_json:
 					if not key in submission_json_dummy:
 						submission_json_dummy[key] = submission_json[key];
-				(submission_json, validation_status) = format_all(submission_json_dummy, True)
+				submission_json = format_all_crawl(submission_json_dummy, True)
+				validation_status = validate(submission_json)
 				
 				assert validation_status < 3, \
 					"Submission invalid after validation. This may be caused by not enough points on the path."