diff --git a/core/io/resource.h b/core/io/resource.h index 0f88738f76..e864b371ad 100644 --- a/core/io/resource.h +++ b/core/io/resource.h @@ -31,6 +31,7 @@ #ifndef RESOURCE_H #define RESOURCE_H +#include "core/io/resource_uid.h" #include "core/object/class_db.h" #include "core/object/ref_counted.h" #include "core/templates/safe_refcount.h" diff --git a/core/io/resource_format_binary.cpp b/core/io/resource_format_binary.cpp index e889586afc..38ad541a40 100644 --- a/core/io/resource_format_binary.cpp +++ b/core/io/resource_format_binary.cpp @@ -816,13 +816,18 @@ String ResourceLoaderBinary::get_unicode_string() { } void ResourceLoaderBinary::get_dependencies(FileAccess *p_f, List *p_dependencies, bool p_add_types) { - open(p_f); + open(p_f, false, true); if (error) { return; } for (int i = 0; i < external_resources.size(); i++) { - String dep = external_resources[i].path; + String dep; + if (external_resources[i].uid != ResourceUID::INVALID_ID) { + dep = ResourceUID::get_singleton()->id_to_text(external_resources[i].uid); + } else { + dep = external_resources[i].path; + } if (p_add_types && external_resources[i].type != String()) { dep += "::" + external_resources[i].type; @@ -832,7 +837,7 @@ void ResourceLoaderBinary::get_dependencies(FileAccess *p_f, List *p_dep } } -void ResourceLoaderBinary::open(FileAccess *p_f) { +void ResourceLoaderBinary::open(FileAccess *p_f, bool p_no_resources, bool p_keep_uuid_paths) { error = OK; f = p_f; @@ -891,10 +896,24 @@ void ResourceLoaderBinary::open(FileAccess *p_f) { if (flags & ResourceFormatSaverBinaryInstance::FORMAT_FLAG_NAMED_SCENE_IDS) { using_named_scene_ids = true; } - for (int i = 0; i < 13; i++) { + if (flags & ResourceFormatSaverBinaryInstance::FORMAT_FLAG_UIDS) { + using_uids = true; + } + + if (using_uids) { + uid = f->get_64(); + } else { + uid = ResourceUID::INVALID_ID; + } + + for (int i = 0; i < 5; i++) { f->get_32(); //skip a few reserved fields } + if (p_no_resources) { + return; + } + uint32_t string_table_size = f->get_32(); string_map.resize(string_table_size); for (uint32_t i = 0; i < string_table_size; i++) { @@ -908,8 +927,18 @@ void ResourceLoaderBinary::open(FileAccess *p_f) { for (uint32_t i = 0; i < ext_resources_size; i++) { ExtResource er; er.type = get_unicode_string(); - er.path = get_unicode_string(); + if (using_uids) { + er.uid = f->get_64(); + if (!p_keep_uuid_paths && er.uid != ResourceUID::INVALID_ID) { + if (ResourceUID::get_singleton()->has_id(er.uid)) { + // If a UID is found and the path is valid, it will be used, otherwise, it falls back to the path. + er.path = ResourceUID::get_singleton()->get_id_path(er.uid); + } else { + WARN_PRINT(String(res_path + ": In external resouce #" + itos(i) + ", invalid UUID: " + ResourceUID::get_singleton()->id_to_text(er.uid) + " - using text path instead: " + er.path).utf8().get_data()); + } + } + } external_resources.push_back(er); } @@ -1173,8 +1202,15 @@ Error ResourceFormatLoaderBinary::rename_dependencies(const String &p_path, cons uint64_t importmd_ofs = f->get_64(); fw->store_64(0); //metadata offset - for (int i = 0; i < 14; i++) { - fw->store_32(0); + uint32_t flags = f->get_32(); + bool using_uids = (flags & ResourceFormatSaverBinaryInstance::FORMAT_FLAG_UIDS); + uint64_t uid_data = f->get_64(); + + fw->store_32(flags); + fw->store_64(uid_data); + + for (int i = 0; i < 5; i++) { + f->store_32(0); // reserved f->get_32(); } @@ -1195,6 +1231,16 @@ Error ResourceFormatLoaderBinary::rename_dependencies(const String &p_path, cons String type = get_ustring(f); String path = get_ustring(f); + if (using_uids) { + ResourceUID::ID uid = f->get_64(); + if (uid != ResourceUID::INVALID_ID) { + if (ResourceUID::get_singleton()->has_id(uid)) { + // If a UID is found and the path is valid, it will be used, otherwise, it falls back to the path. + path = ResourceUID::get_singleton()->get_id_path(uid); + } + } + } + bool relative = false; if (!path.begins_with("res://")) { path = local_path.plus_file(path).simplify_path(); @@ -1206,6 +1252,8 @@ Error ResourceFormatLoaderBinary::rename_dependencies(const String &p_path, cons path = np; } + String full_path = path; + if (relative) { //restore relative path = local_path.path_to_file(path); @@ -1213,6 +1261,11 @@ Error ResourceFormatLoaderBinary::rename_dependencies(const String &p_path, cons save_ustring(fw, type); save_ustring(fw, path); + + if (using_uids) { + ResourceUID::ID uid = ResourceSaver::get_resource_id_for_path(full_path); + f->store_64(uid); + } } int64_t size_diff = (int64_t)fw->get_position() - (int64_t)f->get_position(); @@ -1268,6 +1321,28 @@ String ResourceFormatLoaderBinary::get_resource_type(const String &p_path) const return ClassDB::get_compatibility_remapped_class(r); } +ResourceUID::ID ResourceFormatLoaderBinary::get_resource_uid(const String &p_path) const { + String ext = p_path.get_extension().to_lower(); + if (!ClassDB::is_resource_extension(ext)) { + return ResourceUID::INVALID_ID; + } + + FileAccess *f = FileAccess::open(p_path, FileAccess::READ); + if (!f) { + return ResourceUID::INVALID_ID; //could not read + } + + ResourceLoaderBinary loader; + loader.local_path = ProjectSettings::get_singleton()->localize_path(p_path); + loader.res_path = loader.local_path; + //loader.set_local_path( Globals::get_singleton()->localize_path(p_path) ); + loader.open(f, true); + if (loader.error != OK) { + return ResourceUID::INVALID_ID; //could not read + } + return loader.uid; +} + /////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////// /////////////////////////////////////////////////////////// @@ -1824,8 +1899,10 @@ Error ResourceFormatSaverBinaryInstance::save(const String &p_path, const RES &p save_unicode_string(f, p_resource->get_class()); f->store_64(0); //offset to import metadata - f->store_32(FORMAT_FLAG_NAMED_SCENE_IDS); - for (int i = 0; i < 13; i++) { + f->store_32(FORMAT_FLAG_NAMED_SCENE_IDS | FORMAT_FLAG_UIDS); + ResourceUID::ID uid = ResourceSaver::get_resource_id_for_path(p_path, true); + f->store_64(uid); + for (int i = 0; i < 5; i++) { f->store_32(0); // reserved } @@ -1891,6 +1968,8 @@ Error ResourceFormatSaverBinaryInstance::save(const String &p_path, const RES &p String path = save_order[i]->get_path(); path = relative_paths ? local_path.path_to_file(path) : path; save_unicode_string(f, path); + ResourceUID::ID ruid = ResourceSaver::get_resource_id_for_path(save_order[i]->get_path(), false); + f->store_64(ruid); } // save internal resource table f->store_32(saved_resources.size()); //amount of internal resources diff --git a/core/io/resource_format_binary.h b/core/io/resource_format_binary.h index e3dcf44492..ac964d2053 100644 --- a/core/io/resource_format_binary.h +++ b/core/io/resource_format_binary.h @@ -47,6 +47,8 @@ class ResourceLoaderBinary { uint64_t importmd_ofs = 0; + ResourceUID::ID uid = ResourceUID::INVALID_ID; + Vector str_buf; List resource_cache; @@ -57,10 +59,12 @@ class ResourceLoaderBinary { struct ExtResource { String path; String type; + ResourceUID::ID uid = ResourceUID::INVALID_ID; RES cache; }; bool using_named_scene_ids = false; + bool using_uids = false; bool use_sub_threads = false; float *progress = nullptr; Vector external_resources; @@ -94,7 +98,7 @@ public: void set_translation_remapped(bool p_remapped); void set_remaps(const Map &p_remaps) { remaps = p_remaps; } - void open(FileAccess *p_f); + void open(FileAccess *p_f, bool p_no_resources = false, bool p_keep_uuid_paths = false); String recognize(FileAccess *p_f); void get_dependencies(FileAccess *p_f, List *p_dependencies, bool p_add_types); @@ -109,6 +113,7 @@ public: virtual void get_recognized_extensions(List *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; + virtual ResourceUID::ID get_resource_uid(const String &p_path) const; virtual void get_dependencies(const String &p_path, List *p_dependencies, bool p_add_types = false); virtual Error rename_dependencies(const String &p_path, const Map &p_map); }; @@ -157,7 +162,8 @@ class ResourceFormatSaverBinaryInstance { public: enum { - FORMAT_FLAG_NAMED_SCENE_IDS = 1 + FORMAT_FLAG_NAMED_SCENE_IDS = 1, + FORMAT_FLAG_UIDS = 2, }; Error save(const String &p_path, const RES &p_resource, uint32_t p_flags = 0); static void write_variant(FileAccess *f, const Variant &p_property, Map &resource_map, Map &external_resources, Map &string_map, const PropertyInfo &p_hint = PropertyInfo()); diff --git a/core/io/resource_importer.cpp b/core/io/resource_importer.cpp index f612b84404..6819511eb9 100644 --- a/core/io/resource_importer.cpp +++ b/core/io/resource_importer.cpp @@ -93,6 +93,8 @@ Error ResourceFormatImporter::_get_path_and_type(const String &p_path, PathAndTy r_path_and_type.type = ClassDB::get_compatibility_remapped_class(value); } else if (assign == "importer") { r_path_and_type.importer = value; + } else if (assign == "uid") { + r_path_and_type.uid = ResourceUID::get_singleton()->text_to_id(value); } else if (assign == "group_file") { r_path_and_type.group_file = value; } else if (assign == "metadata") { @@ -336,6 +338,17 @@ String ResourceFormatImporter::get_resource_type(const String &p_path) const { return pat.type; } +ResourceUID::ID ResourceFormatImporter::get_resource_uid(const String &p_path) const { + PathAndType pat; + Error err = _get_path_and_type(p_path, pat); + + if (err != OK) { + return ResourceUID::INVALID_ID; + } + + return pat.uid; +} + Variant ResourceFormatImporter::get_resource_metadata(const String &p_path) const { PathAndType pat; Error err = _get_path_and_type(p_path, pat); diff --git a/core/io/resource_importer.h b/core/io/resource_importer.h index 62cb608a42..a1cacbd306 100644 --- a/core/io/resource_importer.h +++ b/core/io/resource_importer.h @@ -42,6 +42,7 @@ class ResourceFormatImporter : public ResourceFormatLoader { String importer; String group_file; Variant metadata; + uint64_t uid = ResourceUID::INVALID_ID; }; Error _get_path_and_type(const String &p_path, PathAndType &r_path_and_type, bool *r_valid = nullptr) const; @@ -63,6 +64,8 @@ public: virtual bool recognize_path(const String &p_path, const String &p_for_type = String()) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; + virtual ResourceUID::ID get_resource_uid(const String &p_path) const; + virtual Variant get_resource_metadata(const String &p_path) const; virtual bool is_import_valid(const String &p_path) const; virtual void get_dependencies(const String &p_path, List *p_dependencies, bool p_add_types = false); diff --git a/core/io/resource_loader.cpp b/core/io/resource_loader.cpp index c5dfe1f2b0..b685f973e8 100644 --- a/core/io/resource_loader.cpp +++ b/core/io/resource_loader.cpp @@ -84,6 +84,14 @@ String ResourceFormatLoader::get_resource_type(const String &p_path) const { return ""; } +ResourceUID::ID ResourceFormatLoader::get_resource_uid(const String &p_path) const { + if (get_script_instance() && get_script_instance()->has_method("_get_resource_uid")) { + return get_script_instance()->call("_get_resource_uid", p_path); + } + + return ResourceUID::INVALID_ID; +} + void ResourceFormatLoader::get_recognized_extensions_for_type(const String &p_type, List *p_extensions) const { if (p_type == "" || handles_type(p_type)) { get_recognized_extensions(p_extensions); @@ -270,13 +278,18 @@ void ResourceLoader::_thread_load_function(void *p_userdata) { thread_load_mutex->unlock(); } -Error ResourceLoader::load_threaded_request(const String &p_path, const String &p_type_hint, bool p_use_sub_threads, ResourceFormatLoader::CacheMode p_cache_mode, const String &p_source_resource) { - String local_path; - if (p_path.is_rel_path()) { - local_path = "res://" + p_path; +static String _validate_local_path(const String &p_path) { + ResourceUID::ID uid = ResourceUID::get_singleton()->text_to_id(p_path); + if (uid != ResourceUID::INVALID_ID) { + return ResourceUID::get_singleton()->get_id_path(uid); + } else if (p_path.is_rel_path()) { + return "res://" + p_path; } else { - local_path = ProjectSettings::get_singleton()->localize_path(p_path); + return ProjectSettings::get_singleton()->localize_path(p_path); } +} +Error ResourceLoader::load_threaded_request(const String &p_path, const String &p_type_hint, bool p_use_sub_threads, ResourceFormatLoader::CacheMode p_cache_mode, const String &p_source_resource) { + String local_path = _validate_local_path(p_path); thread_load_mutex->lock(); @@ -399,12 +412,7 @@ float ResourceLoader::_dependency_get_progress(const String &p_path) { } ResourceLoader::ThreadLoadStatus ResourceLoader::load_threaded_get_status(const String &p_path, float *r_progress) { - String local_path; - if (p_path.is_rel_path()) { - local_path = "res://" + p_path; - } else { - local_path = ProjectSettings::get_singleton()->localize_path(p_path); - } + String local_path = _validate_local_path(p_path); thread_load_mutex->lock(); if (!thread_load_tasks.has(local_path)) { @@ -424,12 +432,7 @@ ResourceLoader::ThreadLoadStatus ResourceLoader::load_threaded_get_status(const } RES ResourceLoader::load_threaded_get(const String &p_path, Error *r_error) { - String local_path; - if (p_path.is_rel_path()) { - local_path = "res://" + p_path; - } else { - local_path = ProjectSettings::get_singleton()->localize_path(p_path); - } + String local_path = _validate_local_path(p_path); thread_load_mutex->lock(); if (!thread_load_tasks.has(local_path)) { @@ -510,12 +513,7 @@ RES ResourceLoader::load(const String &p_path, const String &p_type_hint, Resour *r_error = ERR_CANT_OPEN; } - String local_path; - if (p_path.is_rel_path()) { - local_path = "res://" + p_path; - } else { - local_path = ProjectSettings::get_singleton()->localize_path(p_path); - } + String local_path = _validate_local_path(p_path); if (p_cache_mode != ResourceFormatLoader::CACHE_MODE_IGNORE) { thread_load_mutex->lock(); @@ -612,12 +610,7 @@ RES ResourceLoader::load(const String &p_path, const String &p_type_hint, Resour } bool ResourceLoader::exists(const String &p_path, const String &p_type_hint) { - String local_path; - if (p_path.is_rel_path()) { - local_path = "res://" + p_path; - } else { - local_path = ProjectSettings::get_singleton()->localize_path(p_path); - } + String local_path = _validate_local_path(p_path); if (ResourceCache::has(local_path)) { return true; // If cached, it probably exists @@ -677,14 +670,7 @@ void ResourceLoader::remove_resource_format_loader(Ref p_f } int ResourceLoader::get_import_order(const String &p_path) { - String path = _path_remap(p_path); - - String local_path; - if (path.is_rel_path()) { - local_path = "res://" + path; - } else { - local_path = ProjectSettings::get_singleton()->localize_path(path); - } + String local_path = _path_remap(_validate_local_path(p_path)); for (int i = 0; i < loader_count; i++) { if (!loader[i]->recognize_path(local_path)) { @@ -702,14 +688,7 @@ int ResourceLoader::get_import_order(const String &p_path) { } String ResourceLoader::get_import_group_file(const String &p_path) { - String path = _path_remap(p_path); - - String local_path; - if (path.is_rel_path()) { - local_path = "res://" + path; - } else { - local_path = ProjectSettings::get_singleton()->localize_path(path); - } + String local_path = _path_remap(_validate_local_path(p_path)); for (int i = 0; i < loader_count; i++) { if (!loader[i]->recognize_path(local_path)) { @@ -727,14 +706,7 @@ String ResourceLoader::get_import_group_file(const String &p_path) { } bool ResourceLoader::is_import_valid(const String &p_path) { - String path = _path_remap(p_path); - - String local_path; - if (path.is_rel_path()) { - local_path = "res://" + path; - } else { - local_path = ProjectSettings::get_singleton()->localize_path(path); - } + String local_path = _path_remap(_validate_local_path(p_path)); for (int i = 0; i < loader_count; i++) { if (!loader[i]->recognize_path(local_path)) { @@ -752,14 +724,7 @@ bool ResourceLoader::is_import_valid(const String &p_path) { } bool ResourceLoader::is_imported(const String &p_path) { - String path = _path_remap(p_path); - - String local_path; - if (path.is_rel_path()) { - local_path = "res://" + path; - } else { - local_path = ProjectSettings::get_singleton()->localize_path(path); - } + String local_path = _path_remap(_validate_local_path(p_path)); for (int i = 0; i < loader_count; i++) { if (!loader[i]->recognize_path(local_path)) { @@ -777,14 +742,7 @@ bool ResourceLoader::is_imported(const String &p_path) { } void ResourceLoader::get_dependencies(const String &p_path, List *p_dependencies, bool p_add_types) { - String path = _path_remap(p_path); - - String local_path; - if (path.is_rel_path()) { - local_path = "res://" + path; - } else { - local_path = ProjectSettings::get_singleton()->localize_path(path); - } + String local_path = _path_remap(_validate_local_path(p_path)); for (int i = 0; i < loader_count; i++) { if (!loader[i]->recognize_path(local_path)) { @@ -800,14 +758,7 @@ void ResourceLoader::get_dependencies(const String &p_path, List *p_depe } Error ResourceLoader::rename_dependencies(const String &p_path, const Map &p_map) { - String path = _path_remap(p_path); - - String local_path; - if (path.is_rel_path()) { - local_path = "res://" + path; - } else { - local_path = ProjectSettings::get_singleton()->localize_path(path); - } + String local_path = _path_remap(_validate_local_path(p_path)); for (int i = 0; i < loader_count; i++) { if (!loader[i]->recognize_path(local_path)) { @@ -825,12 +776,7 @@ Error ResourceLoader::rename_dependencies(const String &p_path, const Maplocalize_path(p_path); - } + String local_path = _validate_local_path(p_path); for (int i = 0; i < loader_count; i++) { String result = loader[i]->get_resource_type(local_path); @@ -842,6 +788,19 @@ String ResourceLoader::get_resource_type(const String &p_path) { return ""; } +ResourceUID::ID ResourceLoader::get_resource_uid(const String &p_path) { + String local_path = _validate_local_path(p_path); + + for (int i = 0; i < loader_count; i++) { + ResourceUID::ID id = loader[i]->get_resource_uid(local_path); + if (id != ResourceUID::INVALID_ID) { + return id; + } + } + + return ResourceUID::INVALID_ID; +} + String ResourceLoader::_path_remap(const String &p_path, bool *r_translation_remapped) { String new_path = p_path; diff --git a/core/io/resource_loader.h b/core/io/resource_loader.h index c656b9a69c..b9e234fdd2 100644 --- a/core/io/resource_loader.h +++ b/core/io/resource_loader.h @@ -56,6 +56,7 @@ public: virtual bool recognize_path(const String &p_path, const String &p_for_type = String()) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; + virtual ResourceUID::ID get_resource_uid(const String &p_path) const; virtual void get_dependencies(const String &p_path, List *p_dependencies, bool p_add_types = false); virtual Error rename_dependencies(const String &p_path, const Map &p_map); virtual bool is_import_valid(const String &p_path) const { return true; } @@ -157,6 +158,7 @@ public: static void add_resource_format_loader(Ref p_format_loader, bool p_at_front = false); static void remove_resource_format_loader(Ref p_format_loader); static String get_resource_type(const String &p_path); + static ResourceUID::ID get_resource_uid(const String &p_path); static void get_dependencies(const String &p_path, List *p_dependencies, bool p_add_types = false); static Error rename_dependencies(const String &p_path, const Map &p_map); static bool is_import_valid(const String &p_path); diff --git a/core/io/resource_saver.cpp b/core/io/resource_saver.cpp index 80cb85fba3..bdfc7538a0 100644 --- a/core/io/resource_saver.cpp +++ b/core/io/resource_saver.cpp @@ -39,6 +39,7 @@ Ref ResourceSaver::saver[MAX_SAVERS]; int ResourceSaver::saver_count = 0; bool ResourceSaver::timestamp_on_save = false; ResourceSavedCallback ResourceSaver::save_callback = nullptr; +ResourceSaverGetResourceIDForPath ResourceSaver::save_get_id_for_path = nullptr; Error ResourceFormatSaver::save(const String &p_path, const RES &p_resource, uint32_t p_flags) { if (get_script_instance() && get_script_instance()->has_method("_save")) { @@ -259,3 +260,14 @@ void ResourceSaver::remove_custom_savers() { remove_resource_format_saver(custom_savers[i]); } } + +ResourceUID::ID ResourceSaver::get_resource_id_for_path(const String &p_path, bool p_generate) { + if (save_get_id_for_path) { + return save_get_id_for_path(p_path, p_generate); + } + return ResourceUID::INVALID_ID; +} + +void ResourceSaver::set_get_resource_id_for_path(ResourceSaverGetResourceIDForPath p_callback) { + save_get_id_for_path = p_callback; +} diff --git a/core/io/resource_saver.h b/core/io/resource_saver.h index 07154aac4d..2fc8d32126 100644 --- a/core/io/resource_saver.h +++ b/core/io/resource_saver.h @@ -48,6 +48,7 @@ public: }; typedef void (*ResourceSavedCallback)(Ref p_resource, const String &p_path); +typedef ResourceUID::ID (*ResourceSaverGetResourceIDForPath)(const String &p_path, bool p_generate); class ResourceSaver { enum { @@ -58,6 +59,7 @@ class ResourceSaver { static int saver_count; static bool timestamp_on_save; static ResourceSavedCallback save_callback; + static ResourceSaverGetResourceIDForPath save_get_id_for_path; static Ref _find_custom_resource_format_saver(String path); @@ -80,7 +82,10 @@ public: static void set_timestamp_on_save(bool p_timestamp) { timestamp_on_save = p_timestamp; } static bool get_timestamp_on_save() { return timestamp_on_save; } + static ResourceUID::ID get_resource_id_for_path(const String &p_path, bool p_generate = false); + static void set_save_callback(ResourceSavedCallback p_callback); + static void set_get_resource_id_for_path(ResourceSaverGetResourceIDForPath p_callback); static bool add_custom_resource_format_saver(String script_path); static void remove_custom_resource_format_saver(String script_path); diff --git a/core/io/resource_uid.cpp b/core/io/resource_uid.cpp new file mode 100644 index 0000000000..d4e8fcb6b0 --- /dev/null +++ b/core/io/resource_uid.cpp @@ -0,0 +1,262 @@ +/*************************************************************************/ +/* resource_uid.cpp */ +/*************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/*************************************************************************/ +/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md). */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/*************************************************************************/ + +#include "resource_uid.h" +#include "core/crypto/crypto.h" +#include "core/io/dir_access.h" +#include "core/io/file_access.h" + +static constexpr uint32_t char_count = ('z' - 'a'); +static constexpr uint32_t base = char_count + ('9' - '0'); + +const char *ResourceUID::CACHE_FILE = "res://.godot/uid_cache.bin"; + +String ResourceUID::id_to_text(ID p_id) const { + if (p_id < 0) { + return "uid://"; + } + String txt; + + while (p_id) { + uint32_t c = p_id % base; + if (c < char_count) { + txt = String::chr('a' + c) + txt; + } else { + txt = String::chr('0' + (c - char_count)) + txt; + } + p_id /= base; + } + + return "uid://" + txt; +} + +ResourceUID::ID ResourceUID::text_to_id(const String &p_text) const { + if (!p_text.begins_with("uid://") || p_text == "uid://") { + return INVALID_ID; + } + + uint32_t l = p_text.length(); + uint64_t uid = 0; + for (uint32_t i = 6; i < l; i++) { + uid *= base; + uint32_t c = p_text[i]; + if (c >= 'a' && c <= 'z') { + uid += c - 'a'; + } else if (c >= '0' && c <= '9') { + uid += c - '0' + char_count; + } else { + return INVALID_ID; + } + } + return ID(uid & 0x7FFFFFFFFFFFFFFF); +} + +ResourceUID::ID ResourceUID::create_id() const { + mutex.lock(); + if (crypto.is_null()) { + crypto = Ref(Crypto::create()); + } + mutex.unlock(); + while (true) { + PackedByteArray bytes = crypto->generate_random_bytes(8); + ERR_FAIL_COND_V(bytes.size() != 8, INVALID_ID); + const uint64_t *ptr64 = (const uint64_t *)bytes.ptr(); + ID id = int64_t((*ptr64) & 0x7FFFFFFFFFFFFFFF); + mutex.lock(); + bool exists = unique_ids.has(id); + mutex.unlock(); + if (!exists) { + return id; + } + } +} + +bool ResourceUID::has_id(ID p_id) const { + MutexLock l(mutex); + return unique_ids.has(p_id); +} +void ResourceUID::add_id(ID p_id, const String &p_path) { + MutexLock l(mutex); + ERR_FAIL_COND(unique_ids.has(p_id)); + Cache c; + c.cs = p_path.utf8(); + unique_ids[p_id] = c; + changed = true; +} + +void ResourceUID::set_id(ID p_id, const String &p_path) { + MutexLock l(mutex); + ERR_FAIL_COND(!unique_ids.has(p_id)); + CharString cs = p_path.utf8(); + if (strcmp(cs.ptr(), unique_ids[p_id].cs.ptr()) != 0) { + unique_ids[p_id].cs = cs; + unique_ids[p_id].saved_to_cache = false; //changed + changed = true; + } +} + +String ResourceUID::get_id_path(ID p_id) const { + MutexLock l(mutex); + ERR_FAIL_COND_V(!unique_ids.has(p_id), String()); + const CharString &cs = unique_ids[p_id].cs; + String s(cs.ptr()); + return s; +} +void ResourceUID::remove_id(ID p_id) { + MutexLock l(mutex); + ERR_FAIL_COND(!unique_ids.has(p_id)); + unique_ids.erase(p_id); +} + +Error ResourceUID::save_to_cache() { + if (!FileAccess::exists(CACHE_FILE)) { + DirAccessRef d = DirAccess::create(DirAccess::ACCESS_RESOURCES); + d->make_dir_recursive(String(CACHE_FILE).get_base_dir()); //ensure base dir exists + } + + FileAccessRef f = FileAccess::open(CACHE_FILE, FileAccess::WRITE); + if (!f) { + return ERR_CANT_OPEN; + } + + MutexLock l(mutex); + f->store_32(unique_ids.size()); + + cache_entries = 0; + + for (OrderedHashMap::Element E = unique_ids.front(); E; E = E.next()) { + f->store_64(E.key()); + uint32_t s = E.get().cs.length(); + f->store_32(s); + f->store_buffer((const uint8_t *)E.get().cs.ptr(), s); + E.get().saved_to_cache = true; + cache_entries++; + } + + changed = false; + return OK; +} + +Error ResourceUID::load_from_cache() { + FileAccessRef f = FileAccess::open(CACHE_FILE, FileAccess::READ); + if (!f) { + return ERR_CANT_OPEN; + } + + MutexLock l(mutex); + unique_ids.clear(); + + uint32_t entry_count = f->get_32(); + for (uint32_t i = 0; i < entry_count; i++) { + int64_t id = f->get_64(); + int32_t len = f->get_32(); + Cache c; + c.cs.resize(len + 1); + ERR_FAIL_COND_V(c.cs.size() != len + 1, ERR_FILE_CORRUPT); // out of memory + c.cs[len] = 0; + int32_t rl = f->get_buffer((uint8_t *)c.cs.ptrw(), len); + ERR_FAIL_COND_V(rl != len, ERR_FILE_CORRUPT); + + c.saved_to_cache = true; + unique_ids[id] = c; + } + + cache_entries = entry_count; + changed = false; + return OK; +} + +Error ResourceUID::update_cache() { + if (!changed) { + return OK; + } + + if (cache_entries == 0) { + return save_to_cache(); + } + MutexLock l(mutex); + + FileAccess *f = nullptr; + for (OrderedHashMap::Element E = unique_ids.front(); E; E = E.next()) { + if (!E.get().saved_to_cache) { + if (f == nullptr) { + f = FileAccess::open(CACHE_FILE, FileAccess::READ_WRITE); //append + if (!f) { + return ERR_CANT_OPEN; + } + f->seek_end(); + } + f->store_64(E.key()); + uint32_t s = E.get().cs.length(); + f->store_32(s); + f->store_buffer((const uint8_t *)E.get().cs.ptr(), s); + E.get().saved_to_cache = true; + cache_entries++; + } + } + + if (f != nullptr) { + f->seek(0); + f->store_32(cache_entries); //update amount of entries + f->close(); + memdelete(f); + } + + changed = false; + + return OK; +} + +void ResourceUID::clear() { + cache_entries = 0; + unique_ids.clear(); + changed = false; +} +void ResourceUID::_bind_methods() { + ClassDB::bind_method(D_METHOD("id_to_text", "id"), &ResourceUID::id_to_text); + ClassDB::bind_method(D_METHOD("text_to_id", "text_id"), &ResourceUID::text_to_id); + + ClassDB::bind_method(D_METHOD("create_id"), &ResourceUID::create_id); + + ClassDB::bind_method(D_METHOD("has_id", "id"), &ResourceUID::has_id); + ClassDB::bind_method(D_METHOD("add_id", "id", "path"), &ResourceUID::add_id); + ClassDB::bind_method(D_METHOD("set_id", "id", "path"), &ResourceUID::set_id); + ClassDB::bind_method(D_METHOD("get_id_path", "id"), &ResourceUID::get_id_path); + ClassDB::bind_method(D_METHOD("remove_id", "id", "path"), &ResourceUID::remove_id); + + BIND_CONSTANT(INVALID_ID) +} +ResourceUID *ResourceUID::singleton = nullptr; +ResourceUID::ResourceUID() { + ERR_FAIL_COND(singleton != nullptr); + singleton = this; +} +ResourceUID::~ResourceUID() { +} diff --git a/core/io/resource_uid.h b/core/io/resource_uid.h new file mode 100644 index 0000000000..b12138425a --- /dev/null +++ b/core/io/resource_uid.h @@ -0,0 +1,89 @@ +/*************************************************************************/ +/* resource_uid.h */ +/*************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/*************************************************************************/ +/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md). */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/*************************************************************************/ + +#ifndef RESOURCE_UUID_H +#define RESOURCE_UUID_H + +#include "core/object/ref_counted.h" +#include "core/string/string_name.h" +#include "core/templates/ordered_hash_map.h" + +class Crypto; +class ResourceUID : public Object { + GDCLASS(ResourceUID, Object) +public: + typedef int64_t ID; + enum { + INVALID_ID = -1 + }; + + static const char *CACHE_FILE; + +private: + mutable Ref crypto; + Mutex mutex; + struct Cache { + CharString cs; + bool saved_to_cache = false; + }; + + OrderedHashMap unique_ids; //unique IDs and utf8 paths (less memory used) + static ResourceUID *singleton; + + uint32_t cache_entries = 0; + bool changed = false; + +protected: + static void _bind_methods(); + +public: + String id_to_text(ID p_id) const; + ID text_to_id(const String &p_text) const; + + ID create_id() const; + bool has_id(ID p_id) const; + void add_id(ID p_id, const String &p_path); + void set_id(ID p_id, const String &p_path); + String get_id_path(ID p_id) const; + void remove_id(ID p_id); + + Error load_from_cache(); + Error save_to_cache(); + Error update_cache(); + + void clear(); + + static ResourceUID *get_singleton() { return singleton; } + + ResourceUID(); + ~ResourceUID(); +}; + +#endif // RESOURCEUUID_H diff --git a/core/object/class_db.cpp b/core/object/class_db.cpp index e2db5918e3..f871dd464a 100644 --- a/core/object/class_db.cpp +++ b/core/object/class_db.cpp @@ -1496,6 +1496,10 @@ void ClassDB::get_resource_base_extensions(List *p_extensions) { } } +bool ClassDB::is_resource_extension(const StringName &p_extension) { + return resource_base_extensions.has(p_extension); +} + void ClassDB::get_extensions_for_type(const StringName &p_class, List *p_extensions) { const StringName *K = nullptr; diff --git a/core/object/class_db.h b/core/object/class_db.h index fd574fd2d8..3a84e9ab38 100644 --- a/core/object/class_db.h +++ b/core/object/class_db.h @@ -396,6 +396,7 @@ public: static void add_resource_base_extension(const StringName &p_extension, const StringName &p_class); static void get_resource_base_extensions(List *p_extensions); static void get_extensions_for_type(const StringName &p_class, List *p_extensions); + static bool is_resource_extension(const StringName &p_extension); static void add_compatibility_class(const StringName &p_class, const StringName &p_fallback); diff --git a/core/register_core_types.cpp b/core/register_core_types.cpp index eb37267546..c088390cf0 100644 --- a/core/register_core_types.cpp +++ b/core/register_core_types.cpp @@ -56,6 +56,7 @@ #include "core/io/pck_packer.h" #include "core/io/resource_format_binary.h" #include "core/io/resource_importer.h" +#include "core/io/resource_uid.h" #include "core/io/stream_peer_ssl.h" #include "core/io/tcp_server.h" #include "core/io/translation_loader_po.h" @@ -102,6 +103,8 @@ static NativeExtensionManager *native_extension_manager = nullptr; extern void register_global_constants(); extern void unregister_global_constants(); +static ResourceUID *resource_uid = nullptr; + void register_core_types() { //consistency check static_assert(sizeof(Callable) <= 16); @@ -225,6 +228,10 @@ void register_core_types() { GDREGISTER_VIRTUAL_CLASS(NativeExtensionManager); + GDREGISTER_VIRTUAL_CLASS(ResourceUID); + + resource_uid = memnew(ResourceUID); + native_extension_manager = memnew(NativeExtensionManager); ip = IP::create(); @@ -286,6 +293,7 @@ void register_core_singletons() { Engine::get_singleton()->add_singleton(Engine::Singleton("EngineDebugger", _EngineDebugger::get_singleton())); Engine::get_singleton()->add_singleton(Engine::Singleton("Time", Time::get_singleton())); Engine::get_singleton()->add_singleton(Engine::Singleton("NativeExtensionManager", NativeExtensionManager::get_singleton())); + Engine::get_singleton()->add_singleton(Engine::Singleton("ResourceUID", ResourceUID::get_singleton())); } void register_core_extensions() { @@ -304,6 +312,8 @@ void unregister_core_types() { native_extension_manager->deinitialize_extensions(NativeExtension::INITIALIZATION_LEVEL_CORE); memdelete(native_extension_manager); + + memdelete(resource_uid); memdelete(_resource_loader); memdelete(_resource_saver); memdelete(_os); diff --git a/doc/classes/@GlobalScope.xml b/doc/classes/@GlobalScope.xml index d0bd517050..c0e8b76969 100644 --- a/doc/classes/@GlobalScope.xml +++ b/doc/classes/@GlobalScope.xml @@ -1259,6 +1259,8 @@ The [ResourceSaver] singleton. + + The [TextServerManager] singleton. diff --git a/doc/classes/ResourceUID.xml b/doc/classes/ResourceUID.xml new file mode 100644 index 0000000000..407aed72b2 --- /dev/null +++ b/doc/classes/ResourceUID.xml @@ -0,0 +1,73 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/drivers/unix/file_access_unix.cpp b/drivers/unix/file_access_unix.cpp index 6ea55219bb..f07c654bd6 100644 --- a/drivers/unix/file_access_unix.cpp +++ b/drivers/unix/file_access_unix.cpp @@ -111,7 +111,7 @@ Error FileAccessUnix::_open(const String &p_path, int p_mode_flags) { } } - if (is_backup_save_enabled() && (p_mode_flags & WRITE) && !(p_mode_flags & READ)) { + if (is_backup_save_enabled() && (p_mode_flags == WRITE)) { save_path = path; path = path + ".tmp"; } diff --git a/drivers/windows/file_access_windows.cpp b/drivers/windows/file_access_windows.cpp index d6deda7b5d..f504bad60d 100644 --- a/drivers/windows/file_access_windows.cpp +++ b/drivers/windows/file_access_windows.cpp @@ -108,7 +108,7 @@ Error FileAccessWindows::_open(const String &p_path, int p_mode_flags) { } #endif - if (is_backup_save_enabled() && p_mode_flags & WRITE && !(p_mode_flags & READ)) { + if (is_backup_save_enabled() && p_mode_flags == WRITE) { save_path = path; path = path + ".tmp"; } diff --git a/editor/dependency_editor.cpp b/editor/dependency_editor.cpp index c18b8743cd..9824397076 100644 --- a/editor/dependency_editor.cpp +++ b/editor/dependency_editor.cpp @@ -180,6 +180,15 @@ void DependencyEditor::_update_list() { path = n; type = "Resource"; } + + ResourceUID::ID uid = ResourceUID::get_singleton()->text_to_id(path); + if (uid != ResourceUID::INVALID_ID) { + // dependency is in uid format, obtain proper path + ERR_CONTINUE(!ResourceUID::get_singleton()->has_id(uid)); + + path = ResourceUID::get_singleton()->get_id_path(uid); + } + String name = path.get_file(); Ref icon = EditorNode::get_singleton()->get_class_icon(type); diff --git a/editor/editor_export.cpp b/editor/editor_export.cpp index fc483b46b7..3866d866d6 100644 --- a/editor/editor_export.cpp +++ b/editor/editor_export.cpp @@ -1050,6 +1050,13 @@ Error EditorExportPlatform::export_project_files(const Ref & return err; } } + if (FileAccess::exists(ResourceUID::CACHE_FILE)) { + Vector array = FileAccess::get_file_as_array(ResourceUID::CACHE_FILE); + err = p_func(p_udata, ResourceUID::CACHE_FILE, array, idx, total, enc_in_filters, enc_ex_filters, key); + if (err != OK) { + return err; + } + } // Store text server data if it is supported. if (TS->has_feature(TextServer::FEATURE_USE_SUPPORT_DATA)) { diff --git a/editor/editor_file_system.cpp b/editor/editor_file_system.cpp index 82a71c0e0c..8d7096e9f0 100644 --- a/editor/editor_file_system.cpp +++ b/editor/editor_file_system.cpp @@ -43,7 +43,7 @@ EditorFileSystem *EditorFileSystem::singleton = nullptr; //the name is the version, to keep compatibility with different versions of Godot -#define CACHE_FILE_NAME "filesystem_cache6" +#define CACHE_FILE_NAME "filesystem_cache7" void EditorFileSystemDirectory::sort_files() { files.sort_custom(); @@ -116,7 +116,26 @@ String EditorFileSystemDirectory::get_file_path(int p_idx) const { Vector EditorFileSystemDirectory::get_file_deps(int p_idx) const { ERR_FAIL_INDEX_V(p_idx, files.size(), Vector()); - return files[p_idx]->deps; + Vector deps; + + for (int i = 0; i < files[p_idx]->deps.size(); i++) { + String dep = files[p_idx]->deps[i]; + int sep_idx = dep.find("::"); //may contain type information, unwanted + if (sep_idx != -1) { + dep = dep.substr(0, sep_idx); + } + ResourceUID::ID uid = ResourceUID::get_singleton()->text_to_id(dep); + if (uid != ResourceUID::INVALID_ID) { + //return proper dependency resoure from uid + if (ResourceUID::get_singleton()->has_id(uid)) { + dep = ResourceUID::get_singleton()->get_id_path(uid); + } else { + continue; + } + } + deps.push_back(dep); + } + return deps; } bool EditorFileSystemDirectory::get_file_import_is_valid(int p_idx) const { @@ -234,7 +253,7 @@ void EditorFileSystem::_scan_filesystem() { } else { Vector split = l.split("::"); - ERR_CONTINUE(split.size() != 8); + ERR_CONTINUE(split.size() != 9); String name = split[0]; String file; @@ -243,15 +262,16 @@ void EditorFileSystem::_scan_filesystem() { FileCache fc; fc.type = split[1]; - fc.modification_time = split[2].to_int(); - fc.import_modification_time = split[3].to_int(); - fc.import_valid = split[4].to_int() != 0; - fc.import_group_file = split[5].strip_edges(); - fc.script_class_name = split[6].get_slice("<>", 0); - fc.script_class_extends = split[6].get_slice("<>", 1); - fc.script_class_icon_path = split[6].get_slice("<>", 2); + fc.uid = split[2].to_int(); + fc.modification_time = split[3].to_int(); + fc.import_modification_time = split[4].to_int(); + fc.import_valid = split[5].to_int() != 0; + fc.import_group_file = split[6].strip_edges(); + fc.script_class_name = split[7].get_slice("<>", 0); + fc.script_class_extends = split[7].get_slice("<>", 1); + fc.script_class_icon_path = split[7].get_slice("<>", 2); - String deps = split[7].strip_edges(); + String deps = split[8].strip_edges(); if (deps.length()) { Vector dp = deps.split("<>"); for (int i = 0; i < dp.size(); i++) { @@ -368,6 +388,7 @@ bool EditorFileSystem::_test_for_reimport(const String &p_path, bool p_only_impo Vector dest_files; String dest_md5 = ""; int version = 0; + bool found_uid = false; while (true) { assign = Variant(); @@ -395,6 +416,8 @@ bool EditorFileSystem::_test_for_reimport(const String &p_path, bool p_only_impo version = value; } else if (assign == "importer") { importer_name = value; + } else if (assign == "uid") { + found_uid = true; } else if (!p_only_imported_files) { if (assign == "source_file") { source_file = value; @@ -414,6 +437,10 @@ bool EditorFileSystem::_test_for_reimport(const String &p_path, bool p_only_impo return false; //keep mode, do not reimport } + if (!found_uid) { + return true; //UUID not found, old format, reimport. + } + Ref importer = ResourceFormatImporter::get_singleton()->get_importer_by_name(importer_name); if (importer->get_format_version() > version) { @@ -582,6 +609,9 @@ bool EditorFileSystem::_update_scan_actions() { if (reimports.size()) { reimport_files(reimports); + } else { + //reimport files will update the uid cache file so if nothing was reimported, update it manually + ResourceUID::get_singleton()->update_cache(); } if (first_scan) { @@ -756,6 +786,7 @@ void EditorFileSystem::_scan_new_dir(EditorFileSystemDirectory *p_dir, DirAccess if (fc && fc->modification_time == mt && fc->import_modification_time == import_mt && !_test_for_reimport(path, true)) { fi->type = fc->type; + fi->uid = fc->uid; fi->deps = fc->deps; fi->modified_time = fc->modification_time; fi->import_modified_time = fc->import_modification_time; @@ -781,8 +812,14 @@ void EditorFileSystem::_scan_new_dir(EditorFileSystemDirectory *p_dir, DirAccess //note: I think this should not happen any longer.. } + if (fc->uid == ResourceUID::INVALID_ID) { + // imported files should always have a UUID, so attempt to fetch it. + fi->uid = ResourceLoader::get_resource_uid(path); + } + } else { fi->type = ResourceFormatImporter::get_singleton()->get_resource_type(path); + fi->uid = ResourceFormatImporter::get_singleton()->get_resource_uid(path); fi->import_group_file = ResourceFormatImporter::get_singleton()->get_import_group_file(path); fi->script_class_name = _get_global_script_class(fi->type, path, &fi->script_class_extends, &fi->script_class_icon_path); fi->modified_time = 0; @@ -799,6 +836,7 @@ void EditorFileSystem::_scan_new_dir(EditorFileSystemDirectory *p_dir, DirAccess if (fc && fc->modification_time == mt) { //not imported, so just update type if changed fi->type = fc->type; + fi->uid = fc->uid; fi->modified_time = fc->modification_time; fi->deps = fc->deps; fi->import_modified_time = 0; @@ -809,6 +847,7 @@ void EditorFileSystem::_scan_new_dir(EditorFileSystemDirectory *p_dir, DirAccess } else { //new or modified time fi->type = ResourceLoader::get_resource_type(path); + fi->uid = ResourceLoader::get_resource_uid(path); fi->script_class_name = _get_global_script_class(fi->type, path, &fi->script_class_extends, &fi->script_class_icon_path); fi->deps = _get_dependencies(path); fi->modified_time = mt; @@ -817,6 +856,14 @@ void EditorFileSystem::_scan_new_dir(EditorFileSystemDirectory *p_dir, DirAccess } } + if (fi->uid != ResourceUID::INVALID_ID) { + if (ResourceUID::get_singleton()->has_id(fi->uid)) { + ResourceUID::get_singleton()->set_id(fi->uid, path); + } else { + ResourceUID::get_singleton()->add_id(fi->uid, path); + } + } + for (int i = 0; i < ScriptServer::get_language_count(); i++) { ScriptLanguage *lang = ScriptServer::get_language(i); if (lang->supports_documentation() && fi->type == lang->get_type()) { @@ -1181,7 +1228,7 @@ void EditorFileSystem::_save_filesystem_cache(EditorFileSystemDirectory *p_dir, if (p_dir->files[i]->import_group_file != String()) { group_file_cache.insert(p_dir->files[i]->import_group_file); } - String s = p_dir->files[i]->file + "::" + p_dir->files[i]->type + "::" + itos(p_dir->files[i]->modified_time) + "::" + itos(p_dir->files[i]->import_modified_time) + "::" + itos(p_dir->files[i]->import_valid) + "::" + p_dir->files[i]->import_group_file + "::" + p_dir->files[i]->script_class_name + "<>" + p_dir->files[i]->script_class_extends + "<>" + p_dir->files[i]->script_class_icon_path; + String s = p_dir->files[i]->file + "::" + p_dir->files[i]->type + "::" + itos(p_dir->files[i]->uid) + "::" + itos(p_dir->files[i]->modified_time) + "::" + itos(p_dir->files[i]->import_modified_time) + "::" + itos(p_dir->files[i]->import_valid) + "::" + p_dir->files[i]->import_group_file + "::" + p_dir->files[i]->script_class_name + "<>" + p_dir->files[i]->script_class_extends + "<>" + p_dir->files[i]->script_class_icon_path; s += "::"; for (int j = 0; j < p_dir->files[i]->deps.size(); j++) { if (j > 0) { @@ -1462,6 +1509,11 @@ void EditorFileSystem::update_file(const String &p_file) { //was removed _delete_internal_files(p_file); if (cpos != -1) { // Might've never been part of the editor file system (*.* files deleted in Open dialog). + if (fs->files[cpos]->uid != ResourceUID::INVALID_ID) { + if (ResourceUID::get_singleton()->has_id(fs->files[cpos]->uid)) { + ResourceUID::get_singleton()->remove_id(fs->files[cpos]->uid); + } + } memdelete(fs->files[cpos]); fs->files.remove(cpos); } @@ -1472,6 +1524,7 @@ void EditorFileSystem::update_file(const String &p_file) { } String type = ResourceLoader::get_resource_type(p_file); + ResourceUID::ID uid = ResourceLoader::get_resource_uid(p_file); if (cpos == -1) { // The file did not exist, it was added. @@ -1504,12 +1557,22 @@ void EditorFileSystem::update_file(const String &p_file) { } fs->files[cpos]->type = type; + fs->files[cpos]->uid = uid; fs->files[cpos]->script_class_name = _get_global_script_class(type, p_file, &fs->files[cpos]->script_class_extends, &fs->files[cpos]->script_class_icon_path); fs->files[cpos]->import_group_file = ResourceLoader::get_import_group_file(p_file); fs->files[cpos]->modified_time = FileAccess::get_modified_time(p_file); fs->files[cpos]->deps = _get_dependencies(p_file); fs->files[cpos]->import_valid = ResourceLoader::is_import_valid(p_file); + if (uid != ResourceUID::INVALID_ID) { + if (ResourceUID::get_singleton()->has_id(uid)) { + ResourceUID::get_singleton()->set_id(uid, p_file); + } else { + ResourceUID::get_singleton()->add_id(uid, p_file); + } + + ResourceUID::get_singleton()->update_cache(); + } // Update preview EditorResourcePreview::get_singleton()->check_for_invalidation(p_file); @@ -1532,7 +1595,6 @@ Error EditorFileSystem::_reimport_group(const String &p_group_file, const Vector ERR_CONTINUE(file_importer_name == String()); if (importer_name != String() && importer_name != file_importer_name) { - print_line("one importer '" + importer_name + "' the other '" + file_importer_name + "'."); EditorNode::get_singleton()->show_warning(vformat(TTR("There are multiple importers for different types pointing to file %s, import aborted"), p_group_file)); ERR_FAIL_V(ERR_FILE_CORRUPT); } @@ -1702,6 +1764,8 @@ void EditorFileSystem::_reimport_file(const String &p_file, const Mapget()] = cf->get_value("params", E->get()); } } - if (p_custom_importer == String() && cf->has_section("remap")) { - importer_name = cf->get_value("remap", "importer"); + + if (cf->has_section("remap")) { + if (p_custom_importer == String()) { + importer_name = cf->get_value("remap", "importer"); + } + + if (cf->has_section_key("remap", "uid")) { + String uidt = cf->get_value("remap", "uid"); + uid = ResourceUID::get_singleton()->text_to_id(uidt); + } } } } @@ -1800,6 +1872,12 @@ void EditorFileSystem::_reimport_file(const String &p_file, const Mapstore_line("type=\"" + importer->get_resource_type() + "\""); } + if (uid == ResourceUID::INVALID_ID) { + uid = ResourceUID::get_singleton()->create_id(); + } + + f->store_line("uid=\"" + ResourceUID::get_singleton()->id_to_text(uid) + "\""); //store in readable format + Vector dest_paths; if (err == OK) { @@ -1885,8 +1963,15 @@ void EditorFileSystem::_reimport_file(const String &p_file, const Mapfiles[cpos]->import_modified_time = FileAccess::get_modified_time(p_file + ".import"); fs->files[cpos]->deps = _get_dependencies(p_file); fs->files[cpos]->type = importer->get_resource_type(); + fs->files[cpos]->uid = uid; fs->files[cpos]->import_valid = ResourceLoader::is_import_valid(p_file); + if (ResourceUID::get_singleton()->has_id(uid)) { + ResourceUID::get_singleton()->set_id(uid, p_file); + } else { + ResourceUID::get_singleton()->add_id(uid, p_file); + } + //if file is currently up, maybe the source it was loaded from changed, so import math must be updated for it //to reload properly if (ResourceCache::has(p_file)) { @@ -1937,11 +2022,18 @@ void EditorFileSystem::reimport_files(const Vector &p_files) { Set groups_to_reimport; for (int i = 0; i < p_files.size(); i++) { - String group_file = ResourceFormatImporter::get_singleton()->get_import_group_file(p_files[i]); + String file = p_files[i]; - if (group_file_cache.has(p_files[i])) { + ResourceUID::ID uid = ResourceUID::get_singleton()->text_to_id(file); + if (uid != ResourceUID::INVALID_ID && ResourceUID::get_singleton()->has_id(uid)) { + file = ResourceUID::get_singleton()->get_id_path(uid); + } + + String group_file = ResourceFormatImporter::get_singleton()->get_import_group_file(file); + + if (group_file_cache.has(file)) { //maybe the file itself is a group! - groups_to_reimport.insert(p_files[i]); + groups_to_reimport.insert(file); //groups do not belong to grups group_file = String(); } else if (group_file != String()) { @@ -1950,15 +2042,15 @@ void EditorFileSystem::reimport_files(const Vector &p_files) { } else { //it's a regular file ImportFile ifile; - ifile.path = p_files[i]; - ResourceFormatImporter::get_singleton()->get_import_order_threads_and_importer(p_files[i], ifile.order, ifile.threaded, ifile.importer); + ifile.path = file; + ResourceFormatImporter::get_singleton()->get_import_order_threads_and_importer(file, ifile.order, ifile.threaded, ifile.importer); reimport_files.push_back(ifile); } //group may have changed, so also update group reference EditorFileSystemDirectory *fs = nullptr; int cpos = -1; - if (_find_file(p_files[i], &fs, cpos)) { + if (_find_file(file, &fs, cpos)) { fs->files.write[cpos]->import_group_file = group_file; } } @@ -2023,6 +2115,8 @@ void EditorFileSystem::reimport_files(const Vector &p_files) { } } + ResourceUID::get_singleton()->update_cache(); //after reimporting, update the cache + _save_filesystem_cache(); importing = false; if (!is_scanning()) { @@ -2108,6 +2202,30 @@ void EditorFileSystem::move_group_file(const String &p_path, const String &p_new } } +ResourceUID::ID EditorFileSystem::_resource_saver_get_resource_id_for_path(const String &p_path, bool p_generate) { + if (!p_path.is_resource_file() || p_path.begins_with("res://.godot")) { + //saved externally (configuration file) or internal file, do not assign an ID. + return ResourceUID::INVALID_ID; + } + + EditorFileSystemDirectory *fs = nullptr; + int cpos = -1; + + if (!singleton->_find_file(p_path, &fs, cpos)) { + if (p_generate) { + return ResourceUID::get_singleton()->create_id(); //just create a new one, we will be notified of save anyway and fetch the right UUID at that time, to keep things simple. + } else { + return ResourceUID::INVALID_ID; + } + } else if (fs->files[cpos]->uid != ResourceUID::INVALID_ID) { + return fs->files[cpos]->uid; + } else if (p_generate) { + return ResourceUID::get_singleton()->create_id(); //just create a new one, we will be notified of save anyway and fetch the right UUID at that time, to keep things simple. + } else { + return ResourceUID::INVALID_ID; + } +} + void EditorFileSystem::_bind_methods() { ClassDB::bind_method(D_METHOD("get_filesystem"), &EditorFileSystem::get_filesystem); ClassDB::bind_method(D_METHOD("is_scanning"), &EditorFileSystem::is_scanning); @@ -2169,8 +2287,11 @@ EditorFileSystem::EditorFileSystem() { scan_changes_pending = false; revalidate_import_files = false; import_threads.init(); + ResourceUID::get_singleton()->clear(); //will be updated on scan + ResourceSaver::set_get_resource_id_for_path(_resource_saver_get_resource_id_for_path); } EditorFileSystem::~EditorFileSystem() { import_threads.finish(); + ResourceSaver::set_get_resource_id_for_path(nullptr); } diff --git a/editor/editor_file_system.h b/editor/editor_file_system.h index 37eee13c16..9dce29d09c 100644 --- a/editor/editor_file_system.h +++ b/editor/editor_file_system.h @@ -55,6 +55,7 @@ class EditorFileSystemDirectory : public Object { struct FileInfo { String file; StringName type; + ResourceUID::ID uid = ResourceUID::INVALID_ID; uint64_t modified_time = 0; uint64_t import_modified_time = 0; bool import_valid = false; @@ -159,6 +160,7 @@ class EditorFileSystem : public Node { /* Used for reading the filesystem cache file */ struct FileCache { String type; + ResourceUID::ID uid = ResourceUID::INVALID_ID; uint64_t modification_time = 0; uint64_t import_modification_time = 0; Vector deps; @@ -251,6 +253,8 @@ class EditorFileSystem : public Node { void _reimport_thread(uint32_t p_index, ImportThreadData *p_import_data); + static ResourceUID::ID _resource_saver_get_resource_id_for_path(const String &p_path, bool p_generate); + protected: void _notification(int p_what); static void _bind_methods(); diff --git a/main/main.cpp b/main/main.cpp index b7f67e8f6c..0a8c814d51 100644 --- a/main/main.cpp +++ b/main/main.cpp @@ -1089,6 +1089,8 @@ Error Main::setup(const char *execpath, int argc, char *argv[], bool p_second_ph // Initialize user data dir. OS::get_singleton()->ensure_user_data_dir(); + ResourceUID::get_singleton()->load_from_cache(); // load UUIDs from cache. + GLOBAL_DEF("memory/limits/multithreaded_server/rid_pool_prealloc", 60); ProjectSettings::get_singleton()->set_custom_property_info("memory/limits/multithreaded_server/rid_pool_prealloc", PropertyInfo(Variant::INT, diff --git a/scene/resources/resource_format_text.cpp b/scene/resources/resource_format_text.cpp index 5cf107e8ea..a0a8ad08d5 100644 --- a/scene/resources/resource_format_text.cpp +++ b/scene/resources/resource_format_text.cpp @@ -413,6 +413,17 @@ Error ResourceLoaderText::load() { String type = next_tag.fields["type"]; String id = next_tag.fields["id"]; + if (next_tag.fields.has("uid")) { + String uidt = next_tag.fields["uid"]; + ResourceUID::ID uid = ResourceUID::get_singleton()->text_to_id(uidt); + if (uid != ResourceUID::INVALID_ID && ResourceUID::get_singleton()->has_id(uid)) { + // If a UID is found and the path is valid, it will be used, otherwise, it falls back to the path. + path = ResourceUID::get_singleton()->get_id_path(uid); + } else { + WARN_PRINT(String(res_path + ":" + itos(lines) + " - ext_resource, invalid UUID: " + uidt + " - using text path instead: " + path).utf8().get_data()); + } + } + if (path.find("://") == -1 && path.is_rel_path()) { // path is relative to file being loaded, so convert to a resource path path = ProjectSettings::get_singleton()->localize_path(local_path.get_base_dir().plus_file(path)); @@ -746,7 +757,18 @@ void ResourceLoaderText::get_dependencies(FileAccess *p_f, List *p_depen String path = next_tag.fields["path"]; String type = next_tag.fields["type"]; - if (path.find("://") == -1 && path.is_rel_path()) { + bool using_uid = false; + if (next_tag.fields.has("uid")) { + //if uid exists, return uid in text format, not the path + String uidt = next_tag.fields["uid"]; + ResourceUID::ID uid = ResourceUID::get_singleton()->text_to_id(uidt); + if (uid != ResourceUID::INVALID_ID) { + path = ResourceUID::get_singleton()->id_to_text(uid); + using_uid = true; + } + } + + if (!using_uid && path.find("://") == -1 && path.is_rel_path()) { // path is relative to file being loaded, so convert to a resource path path = ProjectSettings::get_singleton()->localize_path(local_path.get_base_dir().plus_file(path)); } @@ -819,6 +841,14 @@ Error ResourceLoaderText::rename_dependencies(FileAccess *p_f, const String &p_p String id = next_tag.fields["id"]; String type = next_tag.fields["type"]; + if (next_tag.fields.has("uid")) { + String uidt = next_tag.fields["uid"]; + ResourceUID::ID uid = ResourceUID::get_singleton()->text_to_id(uidt); + if (uid != ResourceUID::INVALID_ID && ResourceUID::get_singleton()->has_id(uid)) { + // If a UID is found and the path is valid, it will be used, otherwise, it falls back to the path. + path = ResourceUID::get_singleton()->get_id_path(uid); + } + } bool relative = false; if (!path.begins_with("res://")) { path = base_path.plus_file(path).simplify_path(); @@ -835,7 +865,14 @@ Error ResourceLoaderText::rename_dependencies(FileAccess *p_f, const String &p_p path = base_path.path_to_file(path); } - fw->store_line("[ext_resource path=\"" + path + "\" type=\"" + type + "\" id=\"" + id + "\"]"); + String s = "[ext_resource type=\"" + type + "\""; + + ResourceUID::ID uid = ResourceSaver::get_resource_id_for_path(path); + if (uid != ResourceUID::INVALID_ID) { + s += " uid=\"" + ResourceUID::get_singleton()->id_to_text(uid) + "\""; + } + s += " path=\"" + path + "\" id=\"" + id + "\"]"; + fw->store_line(s); // Bundled. tag_end = f->get_position(); } @@ -921,6 +958,12 @@ void ResourceLoaderText::open(FileAccess *p_f, bool p_skip_first_tag) { return; } + if (tag.fields.has("uid")) { + res_uid = ResourceUID::get_singleton()->text_to_id(tag.fields["uid"]); + } else { + res_uid = ResourceUID::INVALID_ID; + } + if (tag.fields.has("load_steps")) { resources_total = tag.fields["load_steps"]; } else { @@ -976,7 +1019,12 @@ Error ResourceLoaderText::save_as_binary(FileAccess *p_f, const String &p_path) bs_save_unicode_string(wf.f, is_scene ? "PackedScene" : resource_type); wf->store_64(0); //offset to import metadata, this is no longer used - for (int i = 0; i < 14; i++) { + + f->store_32(ResourceFormatSaverBinaryInstance::FORMAT_FLAG_NAMED_SCENE_IDS | ResourceFormatSaverBinaryInstance::FORMAT_FLAG_UIDS); + + f->store_64(res_uid); + + for (int i = 0; i < 5; i++) { wf->store_32(0); // reserved } @@ -1018,9 +1066,15 @@ Error ResourceLoaderText::save_as_binary(FileAccess *p_f, const String &p_path) String path = next_tag.fields["path"]; String type = next_tag.fields["type"]; String id = next_tag.fields["id"]; + ResourceUID::ID uid = ResourceUID::INVALID_ID; + if (next_tag.fields.has("uid")) { + String uidt = next_tag.fields["uid"]; + uid = ResourceUID::get_singleton()->text_to_id(uidt); + } bs_save_unicode_string(wf.f, type); bs_save_unicode_string(wf.f, path); + wf.f->store_64(uid); int lindex = dummy_read.external_resources.size(); Ref dr; @@ -1257,6 +1311,32 @@ String ResourceLoaderText::recognize(FileAccess *p_f) { return tag.fields["type"]; } +ResourceUID::ID ResourceLoaderText::get_uid(FileAccess *p_f) { + error = OK; + + lines = 1; + f = p_f; + + stream.f = f; + + ignore_resource_parsing = true; + + VariantParser::Tag tag; + Error err = VariantParser::parse_tag(&stream, lines, error_text, tag); + + if (err) { + _printerr(); + return ResourceUID::INVALID_ID; + } + + if (tag.fields.has("uid")) { //field is optional + String uidt = tag.fields["uid"]; + return ResourceUID::get_singleton()->text_to_id(uidt); + } + + return ResourceUID::INVALID_ID; +} + ///////////////////// RES ResourceFormatLoaderText::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress, CacheMode p_cache_mode) { @@ -1277,7 +1357,6 @@ RES ResourceFormatLoaderText::load(const String &p_path, const String &p_origina loader.local_path = ProjectSettings::get_singleton()->localize_path(path); loader.progress = r_progress; loader.res_path = loader.local_path; - //loader.set_local_path( ProjectSettings::get_singleton()->localize_path(p_path) ); loader.open(f); err = loader.load(); if (r_error) { @@ -1330,11 +1409,28 @@ String ResourceFormatLoaderText::get_resource_type(const String &p_path) const { ResourceLoaderText loader; loader.local_path = ProjectSettings::get_singleton()->localize_path(p_path); loader.res_path = loader.local_path; - //loader.set_local_path( ProjectSettings::get_singleton()->localize_path(p_path) ); String r = loader.recognize(f); return ClassDB::get_compatibility_remapped_class(r); } +ResourceUID::ID ResourceFormatLoaderText::get_resource_uid(const String &p_path) const { + String ext = p_path.get_extension().to_lower(); + + if (ext != "tscn" && ext != "tres") { + return ResourceUID::INVALID_ID; + } + + FileAccess *f = FileAccess::open(p_path, FileAccess::READ); + if (!f) { + return ResourceUID::INVALID_ID; //could not read + } + + ResourceLoaderText loader; + loader.local_path = ProjectSettings::get_singleton()->localize_path(p_path); + loader.res_path = loader.local_path; + return loader.get_uid(f); +} + void ResourceFormatLoaderText::get_dependencies(const String &p_path, List *p_dependencies, bool p_add_types) { FileAccess *f = FileAccess::open(p_path, FileAccess::READ); if (!f) { @@ -1344,7 +1440,6 @@ void ResourceFormatLoaderText::get_dependencies(const String &p_path, Listlocalize_path(p_path); loader.res_path = loader.local_path; - //loader.set_local_path( ProjectSettings::get_singleton()->localize_path(p_path) ); loader.get_dependencies(f, p_dependencies, p_add_types); } @@ -1357,7 +1452,6 @@ Error ResourceFormatLoaderText::rename_dependencies(const String &p_path, const ResourceLoaderText loader; loader.local_path = ProjectSettings::get_singleton()->localize_path(p_path); loader.res_path = loader.local_path; - //loader.set_local_path( ProjectSettings::get_singleton()->localize_path(p_path) ); return loader.rename_dependencies(f, p_path, p_map); } @@ -1373,7 +1467,6 @@ Error ResourceFormatLoaderText::convert_file_to_binary(const String &p_src_path, const String &path = p_src_path; loader.local_path = ProjectSettings::get_singleton()->localize_path(path); loader.res_path = loader.local_path; - //loader.set_local_path( ProjectSettings::get_singleton()->localize_path(p_path) ); loader.open(f); return loader.save_as_binary(f, p_dst_path); } @@ -1548,6 +1641,12 @@ Error ResourceFormatSaverTextInstance::save(const String &p_path, const RES &p_r } title += "format=" + itos(FORMAT_VERSION) + ""; + ResourceUID::ID uid = ResourceSaver::get_resource_id_for_path(local_path, true); + + if (uid != ResourceUID::INVALID_ID) { + title += " uid=\"" + ResourceUID::get_singleton()->id_to_text(uid) + "\""; + } + f->store_string(title); f->store_line("]\n"); // One empty line. } @@ -1612,7 +1711,14 @@ Error ResourceFormatSaverTextInstance::save(const String &p_path, const RES &p_r for (int i = 0; i < sorted_er.size(); i++) { String p = sorted_er[i].resource->get_path(); - f->store_string("[ext_resource path=\"" + p + "\" type=\"" + sorted_er[i].resource->get_save_class() + "\" id=\"" + sorted_er[i].id + "\"]\n"); // Bundled. + String s = "[ext_resource type=\"" + sorted_er[i].resource->get_save_class() + "\""; + + ResourceUID::ID uid = ResourceSaver::get_resource_id_for_path(p, false); + if (uid != ResourceUID::INVALID_ID) { + s += " uid=\"" + ResourceUID::get_singleton()->id_to_text(uid) + "\""; + } + s += " path=\"" + p + "\" id=\"" + sorted_er[i].id + "\"]\n"; + f->store_string(s); // Bundled. } if (external_resources.size()) { diff --git a/scene/resources/resource_format_text.h b/scene/resources/resource_format_text.h index 5173619a17..373e71b2c4 100644 --- a/scene/resources/resource_format_text.h +++ b/scene/resources/resource_format_text.h @@ -74,6 +74,8 @@ class ResourceLoaderText { mutable int lines = 0; + ResourceUID::ID res_uid = ResourceUID::INVALID_ID; + Map remaps; static Error _parse_sub_resources(void *p_self, VariantParser::Stream *p_stream, Ref &r_res, int &line, String &r_err_str) { return reinterpret_cast(p_self)->_parse_sub_resource(p_stream, r_res, line, r_err_str); } @@ -120,6 +122,7 @@ public: void open(FileAccess *p_f, bool p_skip_first_tag = false); String recognize(FileAccess *p_f); + ResourceUID::ID get_uid(FileAccess *p_f); void get_dependencies(FileAccess *p_f, List *p_dependencies, bool p_add_types); Error rename_dependencies(FileAccess *p_f, const String &p_path, const Map &p_map); @@ -136,6 +139,7 @@ public: virtual void get_recognized_extensions(List *p_extensions) const; virtual bool handles_type(const String &p_type) const; virtual String get_resource_type(const String &p_path) const; + virtual ResourceUID::ID get_resource_uid(const String &p_path) const; virtual void get_dependencies(const String &p_path, List *p_dependencies, bool p_add_types = false); virtual Error rename_dependencies(const String &p_path, const Map &p_map);