From 1e1c007d31a6c223a7fff8b88662a50cd7562b52 Mon Sep 17 00:00:00 2001 From: Hans5958 Date: Sat, 9 Apr 2022 21:07:46 +0700 Subject: [PATCH] Support editing on script, submitted_by to contributors, merge to atlas script --- tools/merge_out.py | 41 +++++++++++++++++++++++++++++++++++++++++ tools/redditcrawl.py | 20 +++++++++++++++++--- 2 files changed, 58 insertions(+), 3 deletions(-) create mode 100644 tools/merge_out.py diff --git a/tools/merge_out.py b/tools/merge_out.py new file mode 100644 index 00000000..aa526705 --- /dev/null +++ b/tools/merge_out.py @@ -0,0 +1,41 @@ +import praw +import json +import time +import re +import os +import traceback +from formatter import format_all, per_line_entries + +out_ids = [] +out_dupe_ids = [] +atlas_ids = [] + +with open('temp_atlas.json', 'r', encoding='utf-8') as out_file: + out_json = json.loads(out_file) + +with open('../atlas.json', 'r', encoding='utf-8') as atlas_file: + atlas_json = json.loads(atlas_file) + +for entry in atlas_json: + atlas_ids.append(entry['id']) + +for entry in out_json: + if (entry['id'] in out_ids): + print(f"Entry {entry['id']} has duplicates! Please resolve this conflict. This will be excluded from the merge.") + out_dupe_ids.append(entry['id']) + out_ids.append(entry['id']) + +for entry in out_json: + if entry['id'] in out_dupe_ids: + continue + + if ('edit' in entry and entry['edit']) or entry['id'] in out_ids: + index = next((i for i, item in enumerate(atlas_json) if item["id"] == entry['id']), None) + if 'edit' in entry: + del entry['edit'] + atlas_json[index] = entry + else: + atlas_json.append(entry) + +with open('../atlas.json', 'w', encoding='utf-8') as atlas_file: + atlas_file.write(per_line_entries(atlas_json)) \ No newline at end of file diff --git a/tools/redditcrawl.py b/tools/redditcrawl.py index a1f5196e..e27c087b 100755 --- a/tools/redditcrawl.py +++ b/tools/redditcrawl.py @@ -101,11 +101,25 @@ def set_flair(submission, flair): if submission_json: - submission_json_dummy = {"id": submission.id, "submitted_by": ""} + submission_json_dummy = {"id": submission.id, "edit": True, "contributors": []} + + if (submission_json.id != 0): + submission_json_dummy["id"] = submission_json.id + else: + del submission_json_dummy["edit"] + + if "submitted_by" in submission_json: + submission_json_dummy["contributors"].append(submission_json['submitted_by']) + del submission_json['submitted_by'] + elif "contributors" in submission_json: + submission_json_dummy["contributors"] = submission_json["contributors"] + try: - submission_json_dummy["submitted_by"] = submission.author.name + if not submission.author.name in submission_json_dummy: + submission_json_dummy["contributors"].append(submission.author.name) except AttributeError: - submission_json_dummy["submitted_by"] = "unknown" + submission_json_dummy["contributors"].append("unknown") + for key in submission_json: if not key in submission_json_dummy: submission_json_dummy[key] = submission_json[key];