maxmustermann/godot
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Implement Resource UIDs

Browse source 
* Most resource types now have unique identifiers.
* Applies to text, binary and imported resources.
* File formats reference both by text and UID (when available). UID always has priority.
* Resource UIDs are 64 bits for better compatibility with the engine.
* Can be represented and used textually, example `uuid://dapwmgsmnl28u`.
* A special binary cache file is used and exported, containing the mappings.

Example of how it looks:

```GDScript
[gd_scene load_steps=2 format=3 uid="uid://dw86wq31afig2"]

[ext_resource type="PackedScene" uid="uid://bt36ojelx8q6c" path="res://subscene.scn" id="1_t56hs"]
```

GDScript, shaders and other special resource files can't currently provide UIDs, but this should be doable with special keywords on the files.
This will be reserved for future PRs.
This commit is contained in:
reduz 2021-07-23 16:01:18 -03:00
parent c25fa02c1c
commit 32b43cfeb3
25 changed files with 900 additions and 126 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
core/io/resource.h
View file

@ -31,6 +31,7 @@
#ifndef RESOURCE_H
#define RESOURCE_H
#include "core/io/resource_uid.h"
#include "core/object/class_db.h"
#include "core/object/ref_counted.h"
#include "core/templates/safe_refcount.h"

97
core/io/resource_format_binary.cpp
View file

@ -816,13 +816,18 @@ String ResourceLoaderBinary::get_unicode_string() {
}
void ResourceLoaderBinary::get_dependencies(FileAccess *p_f, List<String> *p_dependencies, bool p_add_types) {
open(p_f);
open(p_f, false, true);
if (error) {
return;
}
for (int i = 0; i < external_resources.size(); i++) {
String dep = external_resources[i].path;
String dep;
if (external_resources[i].uid != ResourceUID::INVALID_ID) {
dep = ResourceUID::get_singleton()->id_to_text(external_resources[i].uid);
} else {
dep = external_resources[i].path;
}
if (p_add_types && external_resources[i].type != String()) {
dep += "::" + external_resources[i].type;
@ -832,7 +837,7 @@ void ResourceLoaderBinary::get_dependencies(FileAccess *p_f, List<String> *p_dep
}
}
void ResourceLoaderBinary::open(FileAccess *p_f) {
void ResourceLoaderBinary::open(FileAccess *p_f, bool p_no_resources, bool p_keep_uuid_paths) {
error = OK;
f = p_f;
@ -891,10 +896,24 @@ void ResourceLoaderBinary::open(FileAccess *p_f) {
if (flags & ResourceFormatSaverBinaryInstance::FORMAT_FLAG_NAMED_SCENE_IDS) {
using_named_scene_ids = true;
}
for (int i = 0; i < 13; i++) {
if (flags & ResourceFormatSaverBinaryInstance::FORMAT_FLAG_UIDS) {
using_uids = true;
}
if (using_uids) {
uid = f->get_64();
} else {
uid = ResourceUID::INVALID_ID;
}
for (int i = 0; i < 5; i++) {
f->get_32(); //skip a few reserved fields
}
if (p_no_resources) {
return;
}
uint32_t string_table_size = f->get_32();
string_map.resize(string_table_size);
for (uint32_t i = 0; i < string_table_size; i++) {
@ -908,8 +927,18 @@ void ResourceLoaderBinary::open(FileAccess *p_f) {
for (uint32_t i = 0; i < ext_resources_size; i++) {
ExtResource er;
er.type = get_unicode_string();
er.path = get_unicode_string();
if (using_uids) {
er.uid = f->get_64();
if (!p_keep_uuid_paths && er.uid != ResourceUID::INVALID_ID) {
if (ResourceUID::get_singleton()->has_id(er.uid)) {
// If a UID is found and the path is valid, it will be used, otherwise, it falls back to the path.
er.path = ResourceUID::get_singleton()->get_id_path(er.uid);
} else {
WARN_PRINT(String(res_path + ": In external resouce #" + itos(i) + ", invalid UUID: " + ResourceUID::get_singleton()->id_to_text(er.uid) + " - using text path instead: " + er.path).utf8().get_data());
}
}
}
external_resources.push_back(er);
}
@ -1173,8 +1202,15 @@ Error ResourceFormatLoaderBinary::rename_dependencies(const String &p_path, cons
uint64_t importmd_ofs = f->get_64();
fw->store_64(0); //metadata offset
for (int i = 0; i < 14; i++) {
fw->store_32(0);
uint32_t flags = f->get_32();
bool using_uids = (flags & ResourceFormatSaverBinaryInstance::FORMAT_FLAG_UIDS);
uint64_t uid_data = f->get_64();
fw->store_32(flags);
fw->store_64(uid_data);
for (int i = 0; i < 5; i++) {
f->store_32(0); // reserved
f->get_32();
}
@ -1195,6 +1231,16 @@ Error ResourceFormatLoaderBinary::rename_dependencies(const String &p_path, cons
String type = get_ustring(f);
String path = get_ustring(f);
if (using_uids) {
ResourceUID::ID uid = f->get_64();
if (uid != ResourceUID::INVALID_ID) {
if (ResourceUID::get_singleton()->has_id(uid)) {
// If a UID is found and the path is valid, it will be used, otherwise, it falls back to the path.
path = ResourceUID::get_singleton()->get_id_path(uid);
}
}
}
bool relative = false;
if (!path.begins_with("res://")) {
path = local_path.plus_file(path).simplify_path();
@ -1206,6 +1252,8 @@ Error ResourceFormatLoaderBinary::rename_dependencies(const String &p_path, cons
path = np;
}
String full_path = path;
if (relative) {
//restore relative
path = local_path.path_to_file(path);
@ -1213,6 +1261,11 @@ Error ResourceFormatLoaderBinary::rename_dependencies(const String &p_path, cons
save_ustring(fw, type);
save_ustring(fw, path);
if (using_uids) {
ResourceUID::ID uid = ResourceSaver::get_resource_id_for_path(full_path);
f->store_64(uid);
}
}
int64_t size_diff = (int64_t)fw->get_position() - (int64_t)f->get_position();
@ -1268,6 +1321,28 @@ String ResourceFormatLoaderBinary::get_resource_type(const String &p_path) const
return ClassDB::get_compatibility_remapped_class(r);
}
ResourceUID::ID ResourceFormatLoaderBinary::get_resource_uid(const String &p_path) const {
String ext = p_path.get_extension().to_lower();
if (!ClassDB::is_resource_extension(ext)) {
return ResourceUID::INVALID_ID;
}
FileAccess *f = FileAccess::open(p_path, FileAccess::READ);
if (!f) {
return ResourceUID::INVALID_ID; //could not read
}
ResourceLoaderBinary loader;
loader.local_path = ProjectSettings::get_singleton()->localize_path(p_path);
loader.res_path = loader.local_path;
//loader.set_local_path( Globals::get_singleton()->localize_path(p_path) );
loader.open(f, true);
if (loader.error != OK) {
return ResourceUID::INVALID_ID; //could not read
}
return loader.uid;
}
///////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////
@ -1824,8 +1899,10 @@ Error ResourceFormatSaverBinaryInstance::save(const String &p_path, const RES &p
save_unicode_string(f, p_resource->get_class());
f->store_64(0); //offset to import metadata
f->store_32(FORMAT_FLAG_NAMED_SCENE_IDS);
for (int i = 0; i < 13; i++) {
f->store_32(FORMAT_FLAG_NAMED_SCENE_IDS | FORMAT_FLAG_UIDS);
ResourceUID::ID uid = ResourceSaver::get_resource_id_for_path(p_path, true);
f->store_64(uid);
for (int i = 0; i < 5; i++) {
f->store_32(0); // reserved
}
@ -1891,6 +1968,8 @@ Error ResourceFormatSaverBinaryInstance::save(const String &p_path, const RES &p
String path = save_order[i]->get_path();
path = relative_paths ? local_path.path_to_file(path) : path;
save_unicode_string(f, path);
ResourceUID::ID ruid = ResourceSaver::get_resource_id_for_path(save_order[i]->get_path(), false);
f->store_64(ruid);
}
// save internal resource table
f->store_32(saved_resources.size()); //amount of internal resources

10
core/io/resource_format_binary.h
View file

@ -47,6 +47,8 @@ class ResourceLoaderBinary {
uint64_t importmd_ofs = 0;
ResourceUID::ID uid = ResourceUID::INVALID_ID;
Vector<char> str_buf;
List<RES> resource_cache;
@ -57,10 +59,12 @@ class ResourceLoaderBinary {
struct ExtResource {
String path;
String type;
ResourceUID::ID uid = ResourceUID::INVALID_ID;
RES cache;
};
bool using_named_scene_ids = false;
bool using_uids = false;
bool use_sub_threads = false;
float *progress = nullptr;
Vector<ExtResource> external_resources;
@ -94,7 +98,7 @@ public:
void set_translation_remapped(bool p_remapped);
void set_remaps(const Map<String, String> &p_remaps) { remaps = p_remaps; }
void open(FileAccess *p_f);
void open(FileAccess *p_f, bool p_no_resources = false, bool p_keep_uuid_paths = false);
String recognize(FileAccess *p_f);
void get_dependencies(FileAccess *p_f, List<String> *p_dependencies, bool p_add_types);
@ -109,6 +113,7 @@ public:
virtual void get_recognized_extensions(List<String> *p_extensions) const;
virtual bool handles_type(const String &p_type) const;
virtual String get_resource_type(const String &p_path) const;
virtual ResourceUID::ID get_resource_uid(const String &p_path) const;
virtual void get_dependencies(const String &p_path, List<String> *p_dependencies, bool p_add_types = false);
virtual Error rename_dependencies(const String &p_path, const Map<String, String> &p_map);
};
@ -157,7 +162,8 @@ class ResourceFormatSaverBinaryInstance {
public:
enum {
FORMAT_FLAG_NAMED_SCENE_IDS = 1
FORMAT_FLAG_NAMED_SCENE_IDS = 1,
FORMAT_FLAG_UIDS = 2,
};
Error save(const String &p_path, const RES &p_resource, uint32_t p_flags = 0);
static void write_variant(FileAccess *f, const Variant &p_property, Map<RES, int> &resource_map, Map<RES, int> &external_resources, Map<StringName, int> &string_map, const PropertyInfo &p_hint = PropertyInfo());

13
core/io/resource_importer.cpp
View file

@ -93,6 +93,8 @@ Error ResourceFormatImporter::_get_path_and_type(const String &p_path, PathAndTy
r_path_and_type.type = ClassDB::get_compatibility_remapped_class(value);
} else if (assign == "importer") {
r_path_and_type.importer = value;
} else if (assign == "uid") {
r_path_and_type.uid = ResourceUID::get_singleton()->text_to_id(value);
} else if (assign == "group_file") {
r_path_and_type.group_file = value;
} else if (assign == "metadata") {
@ -336,6 +338,17 @@ String ResourceFormatImporter::get_resource_type(const String &p_path) const {
return pat.type;
}
ResourceUID::ID ResourceFormatImporter::get_resource_uid(const String &p_path) const {
PathAndType pat;
Error err = _get_path_and_type(p_path, pat);
if (err != OK) {
return ResourceUID::INVALID_ID;
}
return pat.uid;
}
Variant ResourceFormatImporter::get_resource_metadata(const String &p_path) const {
PathAndType pat;
Error err = _get_path_and_type(p_path, pat);

3
core/io/resource_importer.h
View file

@ -42,6 +42,7 @@ class ResourceFormatImporter : public ResourceFormatLoader {
String importer;
String group_file;
Variant metadata;
uint64_t uid = ResourceUID::INVALID_ID;
};
Error _get_path_and_type(const String &p_path, PathAndType &r_path_and_type, bool *r_valid = nullptr) const;
@ -63,6 +64,8 @@ public:
virtual bool recognize_path(const String &p_path, const String &p_for_type = String()) const;
virtual bool handles_type(const String &p_type) const;
virtual String get_resource_type(const String &p_path) const;
virtual ResourceUID::ID get_resource_uid(const String &p_path) const;
virtual Variant get_resource_metadata(const String &p_path) const;
virtual bool is_import_valid(const String &p_path) const;
virtual void get_dependencies(const String &p_path, List<String> *p_dependencies, bool p_add_types = false);

125
core/io/resource_loader.cpp
View file

@ -84,6 +84,14 @@ String ResourceFormatLoader::get_resource_type(const String &p_path) const {
return "";
}
ResourceUID::ID ResourceFormatLoader::get_resource_uid(const String &p_path) const {
if (get_script_instance() && get_script_instance()->has_method("_get_resource_uid")) {
return get_script_instance()->call("_get_resource_uid", p_path);
}
return ResourceUID::INVALID_ID;
}
void ResourceFormatLoader::get_recognized_extensions_for_type(const String &p_type, List<String> *p_extensions) const {
if (p_type == "" || handles_type(p_type)) {
get_recognized_extensions(p_extensions);
@ -270,13 +278,18 @@ void ResourceLoader::_thread_load_function(void *p_userdata) {
thread_load_mutex->unlock();
}
Error ResourceLoader::load_threaded_request(const String &p_path, const String &p_type_hint, bool p_use_sub_threads, ResourceFormatLoader::CacheMode p_cache_mode, const String &p_source_resource) {
String local_path;
if (p_path.is_rel_path()) {
local_path = "res://" + p_path;
static String _validate_local_path(const String &p_path) {
ResourceUID::ID uid = ResourceUID::get_singleton()->text_to_id(p_path);
if (uid != ResourceUID::INVALID_ID) {
return ResourceUID::get_singleton()->get_id_path(uid);
} else if (p_path.is_rel_path()) {
return "res://" + p_path;
} else {
local_path = ProjectSettings::get_singleton()->localize_path(p_path);
return ProjectSettings::get_singleton()->localize_path(p_path);
}
}
Error ResourceLoader::load_threaded_request(const String &p_path, const String &p_type_hint, bool p_use_sub_threads, ResourceFormatLoader::CacheMode p_cache_mode, const String &p_source_resource) {
String local_path = _validate_local_path(p_path);
thread_load_mutex->lock();
@ -399,12 +412,7 @@ float ResourceLoader::_dependency_get_progress(const String &p_path) {
}
ResourceLoader::ThreadLoadStatus ResourceLoader::load_threaded_get_status(const String &p_path, float *r_progress) {
String local_path;
if (p_path.is_rel_path()) {
local_path = "res://" + p_path;
} else {
local_path = ProjectSettings::get_singleton()->localize_path(p_path);
}
String local_path = _validate_local_path(p_path);
thread_load_mutex->lock();
if (!thread_load_tasks.has(local_path)) {
@ -424,12 +432,7 @@ ResourceLoader::ThreadLoadStatus ResourceLoader::load_threaded_get_status(const
}
RES ResourceLoader::load_threaded_get(const String &p_path, Error *r_error) {
String local_path;
if (p_path.is_rel_path()) {
local_path = "res://" + p_path;
} else {
local_path = ProjectSettings::get_singleton()->localize_path(p_path);
}
String local_path = _validate_local_path(p_path);
thread_load_mutex->lock();
if (!thread_load_tasks.has(local_path)) {
@ -510,12 +513,7 @@ RES ResourceLoader::load(const String &p_path, const String &p_type_hint, Resour
*r_error = ERR_CANT_OPEN;
}
String local_path;
if (p_path.is_rel_path()) {
local_path = "res://" + p_path;
} else {
local_path = ProjectSettings::get_singleton()->localize_path(p_path);
}
String local_path = _validate_local_path(p_path);
if (p_cache_mode != ResourceFormatLoader::CACHE_MODE_IGNORE) {
thread_load_mutex->lock();
@ -612,12 +610,7 @@ RES ResourceLoader::load(const String &p_path, const String &p_type_hint, Resour
}
bool ResourceLoader::exists(const String &p_path, const String &p_type_hint) {
String local_path;
if (p_path.is_rel_path()) {
local_path = "res://" + p_path;
} else {
local_path = ProjectSettings::get_singleton()->localize_path(p_path);
}
String local_path = _validate_local_path(p_path);
if (ResourceCache::has(local_path)) {
return true; // If cached, it probably exists
@ -677,14 +670,7 @@ void ResourceLoader::remove_resource_format_loader(Ref<ResourceFormatLoader> p_f
}
int ResourceLoader::get_import_order(const String &p_path) {
String path = _path_remap(p_path);
String local_path;
if (path.is_rel_path()) {
local_path = "res://" + path;
} else {
local_path = ProjectSettings::get_singleton()->localize_path(path);
}
String local_path = _path_remap(_validate_local_path(p_path));
for (int i = 0; i < loader_count; i++) {
if (!loader[i]->recognize_path(local_path)) {
@ -702,14 +688,7 @@ int ResourceLoader::get_import_order(const String &p_path) {
}
String ResourceLoader::get_import_group_file(const String &p_path) {
String path = _path_remap(p_path);
String local_path;
if (path.is_rel_path()) {
local_path = "res://" + path;
} else {
local_path = ProjectSettings::get_singleton()->localize_path(path);
}
String local_path = _path_remap(_validate_local_path(p_path));
for (int i = 0; i < loader_count; i++) {
if (!loader[i]->recognize_path(local_path)) {
@ -727,14 +706,7 @@ String ResourceLoader::get_import_group_file(const String &p_path) {
}
bool ResourceLoader::is_import_valid(const String &p_path) {
String path = _path_remap(p_path);
String local_path;
if (path.is_rel_path()) {
local_path = "res://" + path;
} else {
local_path = ProjectSettings::get_singleton()->localize_path(path);
}
String local_path = _path_remap(_validate_local_path(p_path));
for (int i = 0; i < loader_count; i++) {
if (!loader[i]->recognize_path(local_path)) {
@ -752,14 +724,7 @@ bool ResourceLoader::is_import_valid(const String &p_path) {
}
bool ResourceLoader::is_imported(const String &p_path) {
String path = _path_remap(p_path);
String local_path;
if (path.is_rel_path()) {
local_path = "res://" + path;
} else {
local_path = ProjectSettings::get_singleton()->localize_path(path);
}
String local_path = _path_remap(_validate_local_path(p_path));
for (int i = 0; i < loader_count; i++) {
if (!loader[i]->recognize_path(local_path)) {
@ -777,14 +742,7 @@ bool ResourceLoader::is_imported(const String &p_path) {
}
void ResourceLoader::get_dependencies(const String &p_path, List<String> *p_dependencies, bool p_add_types) {
String path = _path_remap(p_path);
String local_path;
if (path.is_rel_path()) {
local_path = "res://" + path;
} else {
local_path = ProjectSettings::get_singleton()->localize_path(path);
}
String local_path = _path_remap(_validate_local_path(p_path));
for (int i = 0; i < loader_count; i++) {
if (!loader[i]->recognize_path(local_path)) {
@ -800,14 +758,7 @@ void ResourceLoader::get_dependencies(const String &p_path, List<String> *p_depe
}
Error ResourceLoader::rename_dependencies(const String &p_path, const Map<String, String> &p_map) {
String path = _path_remap(p_path);
String local_path;
if (path.is_rel_path()) {
local_path = "res://" + path;
} else {
local_path = ProjectSettings::get_singleton()->localize_path(path);
}
String local_path = _path_remap(_validate_local_path(p_path));
for (int i = 0; i < loader_count; i++) {
if (!loader[i]->recognize_path(local_path)) {
@ -825,12 +776,7 @@ Error ResourceLoader::rename_dependencies(const String &p_path, const Map<String
}
String ResourceLoader::get_resource_type(const String &p_path) {
String local_path;
if (p_path.is_rel_path()) {
local_path = "res://" + p_path;
} else {
local_path = ProjectSettings::get_singleton()->localize_path(p_path);
}
String local_path = _validate_local_path(p_path);
for (int i = 0; i < loader_count; i++) {
String result = loader[i]->get_resource_type(local_path);
@ -842,6 +788,19 @@ String ResourceLoader::get_resource_type(const String &p_path) {
return "";
}
ResourceUID::ID ResourceLoader::get_resource_uid(const String &p_path) {
String local_path = _validate_local_path(p_path);
for (int i = 0; i < loader_count; i++) {
ResourceUID::ID id = loader[i]->get_resource_uid(local_path);
if (id != ResourceUID::INVALID_ID) {
return id;
}
}
return ResourceUID::INVALID_ID;
}
String ResourceLoader::_path_remap(const String &p_path, bool *r_translation_remapped) {
String new_path = p_path;

2
core/io/resource_loader.h
View file

@ -56,6 +56,7 @@ public:
virtual bool recognize_path(const String &p_path, const String &p_for_type = String()) const;
virtual bool handles_type(const String &p_type) const;
virtual String get_resource_type(const String &p_path) const;
virtual ResourceUID::ID get_resource_uid(const String &p_path) const;
virtual void get_dependencies(const String &p_path, List<String> *p_dependencies, bool p_add_types = false);
virtual Error rename_dependencies(const String &p_path, const Map<String, String> &p_map);
virtual bool is_import_valid(const String &p_path) const { return true; }
@ -157,6 +158,7 @@ public:
static void add_resource_format_loader(Ref<ResourceFormatLoader> p_format_loader, bool p_at_front = false);
static void remove_resource_format_loader(Ref<ResourceFormatLoader> p_format_loader);
static String get_resource_type(const String &p_path);
static ResourceUID::ID get_resource_uid(const String &p_path);
static void get_dependencies(const String &p_path, List<String> *p_dependencies, bool p_add_types = false);
static Error rename_dependencies(const String &p_path, const Map<String, String> &p_map);
static bool is_import_valid(const String &p_path);

12
core/io/resource_saver.cpp
View file

@ -39,6 +39,7 @@ Ref<ResourceFormatSaver> ResourceSaver::saver[MAX_SAVERS];
int ResourceSaver::saver_count = 0;
bool ResourceSaver::timestamp_on_save = false;
ResourceSavedCallback ResourceSaver::save_callback = nullptr;
ResourceSaverGetResourceIDForPath ResourceSaver::save_get_id_for_path = nullptr;
Error ResourceFormatSaver::save(const String &p_path, const RES &p_resource, uint32_t p_flags) {
if (get_script_instance() && get_script_instance()->has_method("_save")) {
@ -259,3 +260,14 @@ void ResourceSaver::remove_custom_savers() {
remove_resource_format_saver(custom_savers[i]);
}
}
ResourceUID::ID ResourceSaver::get_resource_id_for_path(const String &p_path, bool p_generate) {
if (save_get_id_for_path) {
return save_get_id_for_path(p_path, p_generate);
}
return ResourceUID::INVALID_ID;
}
void ResourceSaver::set_get_resource_id_for_path(ResourceSaverGetResourceIDForPath p_callback) {
save_get_id_for_path = p_callback;
}

5
core/io/resource_saver.h
View file

@ -48,6 +48,7 @@ public:
};
typedef void (*ResourceSavedCallback)(Ref<Resource> p_resource, const String &p_path);
typedef ResourceUID::ID (*ResourceSaverGetResourceIDForPath)(const String &p_path, bool p_generate);
class ResourceSaver {
enum {
@ -58,6 +59,7 @@ class ResourceSaver {
static int saver_count;
static bool timestamp_on_save;
static ResourceSavedCallback save_callback;
static ResourceSaverGetResourceIDForPath save_get_id_for_path;
static Ref<ResourceFormatSaver> _find_custom_resource_format_saver(String path);
@ -80,7 +82,10 @@ public:
static void set_timestamp_on_save(bool p_timestamp) { timestamp_on_save = p_timestamp; }
static bool get_timestamp_on_save() { return timestamp_on_save; }
static ResourceUID::ID get_resource_id_for_path(const String &p_path, bool p_generate = false);
static void set_save_callback(ResourceSavedCallback p_callback);
static void set_get_resource_id_for_path(ResourceSaverGetResourceIDForPath p_callback);
static bool add_custom_resource_format_saver(String script_path);
static void remove_custom_resource_format_saver(String script_path);

262
core/io/resource_uid.cpp Normal file
View file

@ -0,0 +1,262 @@
/*************************************************************************/
/* resource_uid.cpp */
/*************************************************************************/
/* This file is part of: */
/* GODOT ENGINE */
/* https://godotengine.org */
/*************************************************************************/
/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur. */
/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md). */
/* */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the */
/* "Software"), to deal in the Software without restriction, including */
/* without limitation the rights to use, copy, modify, merge, publish, */
/* distribute, sublicense, and/or sell copies of the Software, and to */
/* permit persons to whom the Software is furnished to do so, subject to */
/* the following conditions: */
/* */
/* The above copyright notice and this permission notice shall be */
/* included in all copies or substantial portions of the Software. */
/* */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
/*************************************************************************/
#include "resource_uid.h"
#include "core/crypto/crypto.h"
#include "core/io/dir_access.h"
#include "core/io/file_access.h"
static constexpr uint32_t char_count = ('z' - 'a');
static constexpr uint32_t base = char_count + ('9' - '0');
const char *ResourceUID::CACHE_FILE = "res://.godot/uid_cache.bin";
String ResourceUID::id_to_text(ID p_id) const {
if (p_id < 0) {
return "uid://<invalid>";
}
String txt;
while (p_id) {
uint32_t c = p_id % base;
if (c < char_count) {
txt = String::chr('a' + c) + txt;
} else {
txt = String::chr('0' + (c - char_count)) + txt;
}
p_id /= base;
}
return "uid://" + txt;
}
ResourceUID::ID ResourceUID::text_to_id(const String &p_text) const {
if (!p_text.begins_with("uid://") || p_text == "uid://<invalid>") {
return INVALID_ID;
}
uint32_t l = p_text.length();
uint64_t uid = 0;
for (uint32_t i = 6; i < l; i++) {
uid *= base;
uint32_t c = p_text[i];
if (c >= 'a' && c <= 'z') {
uid += c - 'a';
} else if (c >= '0' && c <= '9') {
uid += c - '0' + char_count;
} else {
return INVALID_ID;
}
}
return ID(uid & 0x7FFFFFFFFFFFFFFF);
}
ResourceUID::ID ResourceUID::create_id() const {
mutex.lock();
if (crypto.is_null()) {
crypto = Ref<Crypto>(Crypto::create());
}
mutex.unlock();
while (true) {
PackedByteArray bytes = crypto->generate_random_bytes(8);
ERR_FAIL_COND_V(bytes.size() != 8, INVALID_ID);
const uint64_t *ptr64 = (const uint64_t *)bytes.ptr();
ID id = int64_t((*ptr64) & 0x7FFFFFFFFFFFFFFF);
mutex.lock();
bool exists = unique_ids.has(id);
mutex.unlock();
if (!exists) {
return id;
}
}
}
bool ResourceUID::has_id(ID p_id) const {
MutexLock l(mutex);
return unique_ids.has(p_id);
}
void ResourceUID::add_id(ID p_id, const String &p_path) {
MutexLock l(mutex);
ERR_FAIL_COND(unique_ids.has(p_id));
Cache c;
c.cs = p_path.utf8();
unique_ids[p_id] = c;
changed = true;
}
void ResourceUID::set_id(ID p_id, const String &p_path) {
MutexLock l(mutex);
ERR_FAIL_COND(!unique_ids.has(p_id));
CharString cs = p_path.utf8();
if (strcmp(cs.ptr(), unique_ids[p_id].cs.ptr()) != 0) {
unique_ids[p_id].cs = cs;
unique_ids[p_id].saved_to_cache = false; //changed
changed = true;
}
}
String ResourceUID::get_id_path(ID p_id) const {
MutexLock l(mutex);
ERR_FAIL_COND_V(!unique_ids.has(p_id), String());
const CharString &cs = unique_ids[p_id].cs;
String s(cs.ptr());
return s;
}
void ResourceUID::remove_id(ID p_id) {
MutexLock l(mutex);
ERR_FAIL_COND(!unique_ids.has(p_id));
unique_ids.erase(p_id);
}
Error ResourceUID::save_to_cache() {
if (!FileAccess::exists(CACHE_FILE)) {
DirAccessRef d = DirAccess::create(DirAccess::ACCESS_RESOURCES);
d->make_dir_recursive(String(CACHE_FILE).get_base_dir()); //ensure base dir exists
}
FileAccessRef f = FileAccess::open(CACHE_FILE, FileAccess::WRITE);
if (!f) {
return ERR_CANT_OPEN;
}
MutexLock l(mutex);
f->store_32(unique_ids.size());
cache_entries = 0;
for (OrderedHashMap<ID, Cache>::Element E = unique_ids.front(); E; E = E.next()) {
f->store_64(E.key());
uint32_t s = E.get().cs.length();
f->store_32(s);
f->store_buffer((const uint8_t *)E.get().cs.ptr(), s);
E.get().saved_to_cache = true;
cache_entries++;
}
changed = false;
return OK;
}
Error ResourceUID::load_from_cache() {
FileAccessRef f = FileAccess::open(CACHE_FILE, FileAccess::READ);
if (!f) {
return ERR_CANT_OPEN;
}
MutexLock l(mutex);
unique_ids.clear();
uint32_t entry_count = f->get_32();
for (uint32_t i = 0; i < entry_count; i++) {
int64_t id = f->get_64();
int32_t len = f->get_32();
Cache c;
c.cs.resize(len + 1);
ERR_FAIL_COND_V(c.cs.size() != len + 1, ERR_FILE_CORRUPT); // out of memory
c.cs[len] = 0;
int32_t rl = f->get_buffer((uint8_t *)c.cs.ptrw(), len);
ERR_FAIL_COND_V(rl != len, ERR_FILE_CORRUPT);
c.saved_to_cache = true;
unique_ids[id] = c;
}
cache_entries = entry_count;
changed = false;
return OK;
}
Error ResourceUID::update_cache() {
if (!changed) {
return OK;
}
if (cache_entries == 0) {
return save_to_cache();
}
MutexLock l(mutex);
FileAccess *f = nullptr;
for (OrderedHashMap<ID, Cache>::Element E = unique_ids.front(); E; E = E.next()) {
if (!E.get().saved_to_cache) {
if (f == nullptr) {
f = FileAccess::open(CACHE_FILE, FileAccess::READ_WRITE); //append
if (!f) {
return ERR_CANT_OPEN;
}
f->seek_end();
}
f->store_64(E.key());
uint32_t s = E.get().cs.length();
f->store_32(s);
f->store_buffer((const uint8_t *)E.get().cs.ptr(), s);
E.get().saved_to_cache = true;
cache_entries++;
}
}
if (f != nullptr) {
f->seek(0);
f->store_32(cache_entries); //update amount of entries
f->close();
memdelete(f);
}
changed = false;
return OK;
}
void ResourceUID::clear() {
cache_entries = 0;
unique_ids.clear();
changed = false;
}
void ResourceUID::_bind_methods() {
ClassDB::bind_method(D_METHOD("id_to_text", "id"), &ResourceUID::id_to_text);
ClassDB::bind_method(D_METHOD("text_to_id", "text_id"), &ResourceUID::text_to_id);
ClassDB::bind_method(D_METHOD("create_id"), &ResourceUID::create_id);
ClassDB::bind_method(D_METHOD("has_id", "id"), &ResourceUID::has_id);
ClassDB::bind_method(D_METHOD("add_id", "id", "path"), &ResourceUID::add_id);
ClassDB::bind_method(D_METHOD("set_id", "id", "path"), &ResourceUID::set_id);
ClassDB::bind_method(D_METHOD("get_id_path", "id"), &ResourceUID::get_id_path);
ClassDB::bind_method(D_METHOD("remove_id", "id", "path"), &ResourceUID::remove_id);
BIND_CONSTANT(INVALID_ID)
}
ResourceUID *ResourceUID::singleton = nullptr;
ResourceUID::ResourceUID() {
ERR_FAIL_COND(singleton != nullptr);
singleton = this;
}
ResourceUID::~ResourceUID() {
}

89
core/io/resource_uid.h Normal file
View file

@ -0,0 +1,89 @@
/*************************************************************************/
/* resource_uid.h */
/*************************************************************************/
/* This file is part of: */
/* GODOT ENGINE */
/* https://godotengine.org */
/*************************************************************************/
/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur. */
/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md). */
/* */
/* Permission is hereby granted, free of charge, to any person obtaining */
/* a copy of this software and associated documentation files (the */
/* "Software"), to deal in the Software without restriction, including */
/* without limitation the rights to use, copy, modify, merge, publish, */
/* distribute, sublicense, and/or sell copies of the Software, and to */
/* permit persons to whom the Software is furnished to do so, subject to */
/* the following conditions: */
/* */
/* The above copyright notice and this permission notice shall be */
/* included in all copies or substantial portions of the Software. */
/* */
/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */
/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */
/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */
/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */
/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */
/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */
/*************************************************************************/
#ifndef RESOURCE_UUID_H
#define RESOURCE_UUID_H
#include "core/object/ref_counted.h"
#include "core/string/string_name.h"
#include "core/templates/ordered_hash_map.h"
class Crypto;
class ResourceUID : public Object {
GDCLASS(ResourceUID, Object)
public:
typedef int64_t ID;
enum {
INVALID_ID = -1
};
static const char *CACHE_FILE;
private:
mutable Ref<Crypto> crypto;
Mutex mutex;
struct Cache {
CharString cs;
bool saved_to_cache = false;
};
OrderedHashMap<ID, Cache> unique_ids; //unique IDs and utf8 paths (less memory used)
static ResourceUID *singleton;
uint32_t cache_entries = 0;
bool changed = false;
protected:
static void _bind_methods();
public:
String id_to_text(ID p_id) const;
ID text_to_id(const String &p_text) const;
ID create_id() const;
bool has_id(ID p_id) const;
void add_id(ID p_id, const String &p_path);
void set_id(ID p_id, const String &p_path);
String get_id_path(ID p_id) const;
void remove_id(ID p_id);
Error load_from_cache();
Error save_to_cache();
Error update_cache();
void clear();
static ResourceUID *get_singleton() { return singleton; }
ResourceUID();
~ResourceUID();
};
#endif // RESOURCEUUID_H

4
core/object/class_db.cpp
View file

@ -1496,6 +1496,10 @@ void ClassDB::get_resource_base_extensions(List<String> *p_extensions) {
}
}
bool ClassDB::is_resource_extension(const StringName &p_extension) {
return resource_base_extensions.has(p_extension);
}
void ClassDB::get_extensions_for_type(const StringName &p_class, List<String> *p_extensions) {
const StringName *K = nullptr;

1
core/object/class_db.h
View file

@ -396,6 +396,7 @@ public:
static void add_resource_base_extension(const StringName &p_extension, const StringName &p_class);
static void get_resource_base_extensions(List<String> *p_extensions);
static void get_extensions_for_type(const StringName &p_class, List<String> *p_extensions);
static bool is_resource_extension(const StringName &p_extension);
static void add_compatibility_class(const StringName &p_class, const StringName &p_fallback);

10
core/register_core_types.cpp
View file

@ -56,6 +56,7 @@
#include "core/io/pck_packer.h"
#include "core/io/resource_format_binary.h"
#include "core/io/resource_importer.h"
#include "core/io/resource_uid.h"
#include "core/io/stream_peer_ssl.h"
#include "core/io/tcp_server.h"
#include "core/io/translation_loader_po.h"
@ -102,6 +103,8 @@ static NativeExtensionManager *native_extension_manager = nullptr;
extern void register_global_constants();
extern void unregister_global_constants();
static ResourceUID *resource_uid = nullptr;
void register_core_types() {
//consistency check
static_assert(sizeof(Callable) <= 16);
@ -225,6 +228,10 @@ void register_core_types() {
GDREGISTER_VIRTUAL_CLASS(NativeExtensionManager);
GDREGISTER_VIRTUAL_CLASS(ResourceUID);
resource_uid = memnew(ResourceUID);
native_extension_manager = memnew(NativeExtensionManager);
ip = IP::create();
@ -286,6 +293,7 @@ void register_core_singletons() {
Engine::get_singleton()->add_singleton(Engine::Singleton("EngineDebugger", _EngineDebugger::get_singleton()));
Engine::get_singleton()->add_singleton(Engine::Singleton("Time", Time::get_singleton()));
Engine::get_singleton()->add_singleton(Engine::Singleton("NativeExtensionManager", NativeExtensionManager::get_singleton()));
Engine::get_singleton()->add_singleton(Engine::Singleton("ResourceUID", ResourceUID::get_singleton()));
}
void register_core_extensions() {
@ -304,6 +312,8 @@ void unregister_core_types() {
native_extension_manager->deinitialize_extensions(NativeExtension::INITIALIZATION_LEVEL_CORE);
memdelete(native_extension_manager);
memdelete(resource_uid);
memdelete(_resource_loader);
memdelete(_resource_saver);
memdelete(_os);

2
doc/classes/@GlobalScope.xml
View file

@ -1259,6 +1259,8 @@
<member name="ResourceSaver" type="ResourceSaver" setter="" getter="">
The [ResourceSaver] singleton.
</member>
<member name="ResourceUID" type="ResourceUID" setter="" getter="">
</member>
<member name="TextServerManager" type="TextServerManager" setter="" getter="">
The [TextServerManager] singleton.
</member>

73
doc/classes/ResourceUID.xml Normal file
View file

@ -0,0 +1,73 @@
<?xml version="1.0" encoding="UTF-8" ?>
<class name="ResourceUID" inherits="Object" version="4.0">
<brief_description>
</brief_description>
<description>
</description>
<tutorials>
</tutorials>
<methods>
<method name="add_id">
<return type="void">
</return>
<argument index="0" name="id" type="int">
</argument>
<argument index="1" name="path" type="String">
</argument>
<description>
</description>
</method>
<method name="create_id" qualifiers="const">
<return type="int">
</return>
<description>
</description>
</method>
<method name="get_id_path" qualifiers="const">
<return type="String">
</return>
<argument index="0" name="id" type="int">
</argument>
<description>
</description>
</method>
<method name="has_id" qualifiers="const">
<return type="bool">
</return>
<argument index="0" name="id" type="int">
</argument>
<description>
</description>
</method>
<method name="id_to_text" qualifiers="const">
<return type="String">
</return>
<argument index="0" name="id" type="int">
</argument>
<description>
</description>
</method>
<method name="set_id">
<return type="void">
</return>
<argument index="0" name="id" type="int">
</argument>
<argument index="1" name="path" type="String">
</argument>
<description>
</description>
</method>
<method name="text_to_id" qualifiers="const">
<return type="int">
</return>
<argument index="0" name="text_id" type="String">
</argument>
<description>
</description>
</method>
</methods>
<constants>
<constant name="INVALID_ID" value="-1">
</constant>
</constants>
</class>

2
drivers/unix/file_access_unix.cpp
View file

@ -111,7 +111,7 @@ Error FileAccessUnix::_open(const String &p_path, int p_mode_flags) {
}
}
if (is_backup_save_enabled() && (p_mode_flags & WRITE) && !(p_mode_flags & READ)) {
if (is_backup_save_enabled() && (p_mode_flags == WRITE)) {
save_path = path;
path = path + ".tmp";
}

2
drivers/windows/file_access_windows.cpp
View file

@ -108,7 +108,7 @@ Error FileAccessWindows::_open(const String &p_path, int p_mode_flags) {
}
#endif
if (is_backup_save_enabled() && p_mode_flags & WRITE && !(p_mode_flags & READ)) {
if (is_backup_save_enabled() && p_mode_flags == WRITE) {
save_path = path;
path = path + ".tmp";
}

9
editor/dependency_editor.cpp
View file

@ -180,6 +180,15 @@ void DependencyEditor::_update_list() {
path = n;
type = "Resource";
}
ResourceUID::ID uid = ResourceUID::get_singleton()->text_to_id(path);
if (uid != ResourceUID::INVALID_ID) {
// dependency is in uid format, obtain proper path
ERR_CONTINUE(!ResourceUID::get_singleton()->has_id(uid));
path = ResourceUID::get_singleton()->get_id_path(uid);
}
String name = path.get_file();
Ref<Texture2D> icon = EditorNode::get_singleton()->get_class_icon(type);

7
editor/editor_export.cpp
View file

@ -1050,6 +1050,13 @@ Error EditorExportPlatform::export_project_files(const Ref<EditorExportPreset> &
return err;
}
}
if (FileAccess::exists(ResourceUID::CACHE_FILE)) {
Vector<uint8_t> array = FileAccess::get_file_as_array(ResourceUID::CACHE_FILE);
err = p_func(p_udata, ResourceUID::CACHE_FILE, array, idx, total, enc_in_filters, enc_ex_filters, key);
if (err != OK) {
return err;
}
}
// Store text server data if it is supported.
if (TS->has_feature(TextServer::FEATURE_USE_SUPPORT_DATA)) {

163
editor/editor_file_system.cpp
View file

@ -43,7 +43,7 @@
EditorFileSystem *EditorFileSystem::singleton = nullptr;
//the name is the version, to keep compatibility with different versions of Godot
#define CACHE_FILE_NAME "filesystem_cache6"
#define CACHE_FILE_NAME "filesystem_cache7"
void EditorFileSystemDirectory::sort_files() {
files.sort_custom<FileInfoSort>();
@ -116,7 +116,26 @@ String EditorFileSystemDirectory::get_file_path(int p_idx) const {
Vector<String> EditorFileSystemDirectory::get_file_deps(int p_idx) const {
ERR_FAIL_INDEX_V(p_idx, files.size(), Vector<String>());
return files[p_idx]->deps;
Vector<String> deps;
for (int i = 0; i < files[p_idx]->deps.size(); i++) {
String dep = files[p_idx]->deps[i];
int sep_idx = dep.find("::"); //may contain type information, unwanted
if (sep_idx != -1) {
dep = dep.substr(0, sep_idx);
}
ResourceUID::ID uid = ResourceUID::get_singleton()->text_to_id(dep);
if (uid != ResourceUID::INVALID_ID) {
//return proper dependency resoure from uid
if (ResourceUID::get_singleton()->has_id(uid)) {
dep = ResourceUID::get_singleton()->get_id_path(uid);
} else {
continue;
}
}
deps.push_back(dep);
}
return deps;
}
bool EditorFileSystemDirectory::get_file_import_is_valid(int p_idx) const {
@ -234,7 +253,7 @@ void EditorFileSystem::_scan_filesystem() {
} else {
Vector<String> split = l.split("::");
ERR_CONTINUE(split.size() != 8);
ERR_CONTINUE(split.size() != 9);
String name = split[0];
String file;
@ -243,15 +262,16 @@ void EditorFileSystem::_scan_filesystem() {
FileCache fc;
fc.type = split[1];
fc.modification_time = split[2].to_int();
fc.import_modification_time = split[3].to_int();
fc.import_valid = split[4].to_int() != 0;
fc.import_group_file = split[5].strip_edges();
fc.script_class_name = split[6].get_slice("<>", 0);
fc.script_class_extends = split[6].get_slice("<>", 1);
fc.script_class_icon_path = split[6].get_slice("<>", 2);
fc.uid = split[2].to_int();
fc.modification_time = split[3].to_int();
fc.import_modification_time = split[4].to_int();
fc.import_valid = split[5].to_int() != 0;
fc.import_group_file = split[6].strip_edges();
fc.script_class_name = split[7].get_slice("<>", 0);
fc.script_class_extends = split[7].get_slice("<>", 1);
fc.script_class_icon_path = split[7].get_slice("<>", 2);
String deps = split[7].strip_edges();
String deps = split[8].strip_edges();
if (deps.length()) {
Vector<String> dp = deps.split("<>");
for (int i = 0; i < dp.size(); i++) {
@ -368,6 +388,7 @@ bool EditorFileSystem::_test_for_reimport(const String &p_path, bool p_only_impo
Vector<String> dest_files;
String dest_md5 = "";
int version = 0;
bool found_uid = false;
while (true) {
assign = Variant();
@ -395,6 +416,8 @@ bool EditorFileSystem::_test_for_reimport(const String &p_path, bool p_only_impo
version = value;
} else if (assign == "importer") {
importer_name = value;
} else if (assign == "uid") {
found_uid = true;
} else if (!p_only_imported_files) {
if (assign == "source_file") {
source_file = value;
@ -414,6 +437,10 @@ bool EditorFileSystem::_test_for_reimport(const String &p_path, bool p_only_impo
return false; //keep mode, do not reimport
}
if (!found_uid) {
return true; //UUID not found, old format, reimport.
}
Ref<ResourceImporter> importer = ResourceFormatImporter::get_singleton()->get_importer_by_name(importer_name);
if (importer->get_format_version() > version) {
@ -582,6 +609,9 @@ bool EditorFileSystem::_update_scan_actions() {
if (reimports.size()) {
reimport_files(reimports);
} else {
//reimport files will update the uid cache file so if nothing was reimported, update it manually
ResourceUID::get_singleton()->update_cache();
}
if (first_scan) {
@ -756,6 +786,7 @@ void EditorFileSystem::_scan_new_dir(EditorFileSystemDirectory *p_dir, DirAccess
if (fc && fc->modification_time == mt && fc->import_modification_time == import_mt && !_test_for_reimport(path, true)) {
fi->type = fc->type;
fi->uid = fc->uid;
fi->deps = fc->deps;
fi->modified_time = fc->modification_time;
fi->import_modified_time = fc->import_modification_time;
@ -781,8 +812,14 @@ void EditorFileSystem::_scan_new_dir(EditorFileSystemDirectory *p_dir, DirAccess
//note: I think this should not happen any longer..
}
if (fc->uid == ResourceUID::INVALID_ID) {
// imported files should always have a UUID, so attempt to fetch it.
fi->uid = ResourceLoader::get_resource_uid(path);
}
} else {
fi->type = ResourceFormatImporter::get_singleton()->get_resource_type(path);
fi->uid = ResourceFormatImporter::get_singleton()->get_resource_uid(path);
fi->import_group_file = ResourceFormatImporter::get_singleton()->get_import_group_file(path);
fi->script_class_name = _get_global_script_class(fi->type, path, &fi->script_class_extends, &fi->script_class_icon_path);
fi->modified_time = 0;
@ -799,6 +836,7 @@ void EditorFileSystem::_scan_new_dir(EditorFileSystemDirectory *p_dir, DirAccess
if (fc && fc->modification_time == mt) {
//not imported, so just update type if changed
fi->type = fc->type;
fi->uid = fc->uid;
fi->modified_time = fc->modification_time;
fi->deps = fc->deps;
fi->import_modified_time = 0;
@ -809,6 +847,7 @@ void EditorFileSystem::_scan_new_dir(EditorFileSystemDirectory *p_dir, DirAccess
} else {
//new or modified time
fi->type = ResourceLoader::get_resource_type(path);
fi->uid = ResourceLoader::get_resource_uid(path);
fi->script_class_name = _get_global_script_class(fi->type, path, &fi->script_class_extends, &fi->script_class_icon_path);
fi->deps = _get_dependencies(path);
fi->modified_time = mt;
@ -817,6 +856,14 @@ void EditorFileSystem::_scan_new_dir(EditorFileSystemDirectory *p_dir, DirAccess
}
}
if (fi->uid != ResourceUID::INVALID_ID) {
if (ResourceUID::get_singleton()->has_id(fi->uid)) {
ResourceUID::get_singleton()->set_id(fi->uid, path);
} else {
ResourceUID::get_singleton()->add_id(fi->uid, path);
}
}
for (int i = 0; i < ScriptServer::get_language_count(); i++) {
ScriptLanguage *lang = ScriptServer::get_language(i);
if (lang->supports_documentation() && fi->type == lang->get_type()) {
@ -1181,7 +1228,7 @@ void EditorFileSystem::_save_filesystem_cache(EditorFileSystemDirectory *p_dir,
if (p_dir->files[i]->import_group_file != String()) {
group_file_cache.insert(p_dir->files[i]->import_group_file);
}
String s = p_dir->files[i]->file + "::" + p_dir->files[i]->type + "::" + itos(p_dir->files[i]->modified_time) + "::" + itos(p_dir->files[i]->import_modified_time) + "::" + itos(p_dir->files[i]->import_valid) + "::" + p_dir->files[i]->import_group_file + "::" + p_dir->files[i]->script_class_name + "<>" + p_dir->files[i]->script_class_extends + "<>" + p_dir->files[i]->script_class_icon_path;
String s = p_dir->files[i]->file + "::" + p_dir->files[i]->type + "::" + itos(p_dir->files[i]->uid) + "::" + itos(p_dir->files[i]->modified_time) + "::" + itos(p_dir->files[i]->import_modified_time) + "::" + itos(p_dir->files[i]->import_valid) + "::" + p_dir->files[i]->import_group_file + "::" + p_dir->files[i]->script_class_name + "<>" + p_dir->files[i]->script_class_extends + "<>" + p_dir->files[i]->script_class_icon_path;
s += "::";
for (int j = 0; j < p_dir->files[i]->deps.size(); j++) {
if (j > 0) {
@ -1462,6 +1509,11 @@ void EditorFileSystem::update_file(const String &p_file) {
//was removed
_delete_internal_files(p_file);
if (cpos != -1) { // Might've never been part of the editor file system (*.* files deleted in Open dialog).
if (fs->files[cpos]->uid != ResourceUID::INVALID_ID) {
if (ResourceUID::get_singleton()->has_id(fs->files[cpos]->uid)) {
ResourceUID::get_singleton()->remove_id(fs->files[cpos]->uid);
}
}
memdelete(fs->files[cpos]);
fs->files.remove(cpos);
}
@ -1472,6 +1524,7 @@ void EditorFileSystem::update_file(const String &p_file) {
}
String type = ResourceLoader::get_resource_type(p_file);
ResourceUID::ID uid = ResourceLoader::get_resource_uid(p_file);
if (cpos == -1) {
// The file did not exist, it was added.
@ -1504,12 +1557,22 @@ void EditorFileSystem::update_file(const String &p_file) {
}
fs->files[cpos]->type = type;
fs->files[cpos]->uid = uid;
fs->files[cpos]->script_class_name = _get_global_script_class(type, p_file, &fs->files[cpos]->script_class_extends, &fs->files[cpos]->script_class_icon_path);
fs->files[cpos]->import_group_file = ResourceLoader::get_import_group_file(p_file);
fs->files[cpos]->modified_time = FileAccess::get_modified_time(p_file);
fs->files[cpos]->deps = _get_dependencies(p_file);
fs->files[cpos]->import_valid = ResourceLoader::is_import_valid(p_file);
if (uid != ResourceUID::INVALID_ID) {
if (ResourceUID::get_singleton()->has_id(uid)) {
ResourceUID::get_singleton()->set_id(uid, p_file);
} else {
ResourceUID::get_singleton()->add_id(uid, p_file);
}
ResourceUID::get_singleton()->update_cache();
}
// Update preview
EditorResourcePreview::get_singleton()->check_for_invalidation(p_file);
@ -1532,7 +1595,6 @@ Error EditorFileSystem::_reimport_group(const String &p_group_file, const Vector
ERR_CONTINUE(file_importer_name == String());
if (importer_name != String() && importer_name != file_importer_name) {
print_line("one importer '" + importer_name + "' the other '" + file_importer_name + "'.");
EditorNode::get_singleton()->show_warning(vformat(TTR("There are multiple importers for different types pointing to file %s, import aborted"), p_group_file));
ERR_FAIL_V(ERR_FILE_CORRUPT);
}
@ -1702,6 +1764,8 @@ void EditorFileSystem::_reimport_file(const String &p_file, const Map<StringName
params = *p_custom_options;
}
ResourceUID::ID uid = ResourceUID::INVALID_ID;
if (FileAccess::exists(p_file + ".import")) {
//use existing
if (p_custom_options == nullptr) {
@ -1716,8 +1780,16 @@ void EditorFileSystem::_reimport_file(const String &p_file, const Map<StringName
params[E->get()] = cf->get_value("params", E->get());
}
}
if (p_custom_importer == String() && cf->has_section("remap")) {
importer_name = cf->get_value("remap", "importer");
if (cf->has_section("remap")) {
if (p_custom_importer == String()) {
importer_name = cf->get_value("remap", "importer");
}
if (cf->has_section_key("remap", "uid")) {
String uidt = cf->get_value("remap", "uid");
uid = ResourceUID::get_singleton()->text_to_id(uidt);
}
}
}
}
@ -1800,6 +1872,12 @@ void EditorFileSystem::_reimport_file(const String &p_file, const Map<StringName
f->store_line("type=\"" + importer->get_resource_type() + "\"");
}
if (uid == ResourceUID::INVALID_ID) {
uid = ResourceUID::get_singleton()->create_id();
}
f->store_line("uid=\"" + ResourceUID::get_singleton()->id_to_text(uid) + "\""); //store in readable format
Vector<String> dest_paths;
if (err == OK) {
@ -1885,8 +1963,15 @@ void EditorFileSystem::_reimport_file(const String &p_file, const Map<StringName
fs->files[cpos]->import_modified_time = FileAccess::get_modified_time(p_file + ".import");
fs->files[cpos]->deps = _get_dependencies(p_file);
fs->files[cpos]->type = importer->get_resource_type();
fs->files[cpos]->uid = uid;
fs->files[cpos]->import_valid = ResourceLoader::is_import_valid(p_file);
if (ResourceUID::get_singleton()->has_id(uid)) {
ResourceUID::get_singleton()->set_id(uid, p_file);
} else {
ResourceUID::get_singleton()->add_id(uid, p_file);
}
//if file is currently up, maybe the source it was loaded from changed, so import math must be updated for it
//to reload properly
if (ResourceCache::has(p_file)) {
@ -1937,11 +2022,18 @@ void EditorFileSystem::reimport_files(const Vector<String> &p_files) {
Set<String> groups_to_reimport;
for (int i = 0; i < p_files.size(); i++) {
String group_file = ResourceFormatImporter::get_singleton()->get_import_group_file(p_files[i]);
String file = p_files[i];
if (group_file_cache.has(p_files[i])) {
ResourceUID::ID uid = ResourceUID::get_singleton()->text_to_id(file);
if (uid != ResourceUID::INVALID_ID && ResourceUID::get_singleton()->has_id(uid)) {
file = ResourceUID::get_singleton()->get_id_path(uid);
}
String group_file = ResourceFormatImporter::get_singleton()->get_import_group_file(file);
if (group_file_cache.has(file)) {
//maybe the file itself is a group!
groups_to_reimport.insert(p_files[i]);
groups_to_reimport.insert(file);
//groups do not belong to grups
group_file = String();
} else if (group_file != String()) {
@ -1950,15 +2042,15 @@ void EditorFileSystem::reimport_files(const Vector<String> &p_files) {
} else {
//it's a regular file
ImportFile ifile;
ifile.path = p_files[i];
ResourceFormatImporter::get_singleton()->get_import_order_threads_and_importer(p_files[i], ifile.order, ifile.threaded, ifile.importer);
ifile.path = file;
ResourceFormatImporter::get_singleton()->get_import_order_threads_and_importer(file, ifile.order, ifile.threaded, ifile.importer);
reimport_files.push_back(ifile);
}
//group may have changed, so also update group reference
EditorFileSystemDirectory *fs = nullptr;
int cpos = -1;
if (_find_file(p_files[i], &fs, cpos)) {
if (_find_file(file, &fs, cpos)) {
fs->files.write[cpos]->import_group_file = group_file;
}
}
@ -2023,6 +2115,8 @@ void EditorFileSystem::reimport_files(const Vector<String> &p_files) {
}
}
ResourceUID::get_singleton()->update_cache(); //after reimporting, update the cache
_save_filesystem_cache();
importing = false;
if (!is_scanning()) {
@ -2108,6 +2202,30 @@ void EditorFileSystem::move_group_file(const String &p_path, const String &p_new
}
}
ResourceUID::ID EditorFileSystem::_resource_saver_get_resource_id_for_path(const String &p_path, bool p_generate) {
if (!p_path.is_resource_file() || p_path.begins_with("res://.godot")) {
//saved externally (configuration file) or internal file, do not assign an ID.
return ResourceUID::INVALID_ID;
}
EditorFileSystemDirectory *fs = nullptr;
int cpos = -1;
if (!singleton->_find_file(p_path, &fs, cpos)) {
if (p_generate) {
return ResourceUID::get_singleton()->create_id(); //just create a new one, we will be notified of save anyway and fetch the right UUID at that time, to keep things simple.
} else {
return ResourceUID::INVALID_ID;
}
} else if (fs->files[cpos]->uid != ResourceUID::INVALID_ID) {
return fs->files[cpos]->uid;
} else if (p_generate) {
return ResourceUID::get_singleton()->create_id(); //just create a new one, we will be notified of save anyway and fetch the right UUID at that time, to keep things simple.
} else {
return ResourceUID::INVALID_ID;
}
}
void EditorFileSystem::_bind_methods() {
ClassDB::bind_method(D_METHOD("get_filesystem"), &EditorFileSystem::get_filesystem);
ClassDB::bind_method(D_METHOD("is_scanning"), &EditorFileSystem::is_scanning);
@ -2169,8 +2287,11 @@ EditorFileSystem::EditorFileSystem() {
scan_changes_pending = false;
revalidate_import_files = false;
import_threads.init();
ResourceUID::get_singleton()->clear(); //will be updated on scan
ResourceSaver::set_get_resource_id_for_path(_resource_saver_get_resource_id_for_path);
}
EditorFileSystem::~EditorFileSystem() {
import_threads.finish();
ResourceSaver::set_get_resource_id_for_path(nullptr);
}

4
editor/editor_file_system.h
View file

@ -55,6 +55,7 @@ class EditorFileSystemDirectory : public Object {
struct FileInfo {
String file;
StringName type;
ResourceUID::ID uid = ResourceUID::INVALID_ID;
uint64_t modified_time = 0;
uint64_t import_modified_time = 0;
bool import_valid = false;
@ -159,6 +160,7 @@ class EditorFileSystem : public Node {
/* Used for reading the filesystem cache file */
struct FileCache {
String type;
ResourceUID::ID uid = ResourceUID::INVALID_ID;
uint64_t modification_time = 0;
uint64_t import_modification_time = 0;
Vector<String> deps;
@ -251,6 +253,8 @@ class EditorFileSystem : public Node {
void _reimport_thread(uint32_t p_index, ImportThreadData *p_import_data);
static ResourceUID::ID _resource_saver_get_resource_id_for_path(const String &p_path, bool p_generate);
protected:
void _notification(int p_what);
static void _bind_methods();

2
main/main.cpp
View file

@ -1089,6 +1089,8 @@ Error Main::setup(const char *execpath, int argc, char *argv[], bool p_second_ph
// Initialize user data dir.
OS::get_singleton()->ensure_user_data_dir();
ResourceUID::get_singleton()->load_from_cache(); // load UUIDs from cache.
GLOBAL_DEF("memory/limits/multithreaded_server/rid_pool_prealloc", 60);
ProjectSettings::get_singleton()->set_custom_property_info("memory/limits/multithreaded_server/rid_pool_prealloc",
PropertyInfo(Variant::INT,

124
scene/resources/resource_format_text.cpp
View file

@ -413,6 +413,17 @@ Error ResourceLoaderText::load() {
String type = next_tag.fields["type"];
String id = next_tag.fields["id"];
if (next_tag.fields.has("uid")) {
String uidt = next_tag.fields["uid"];
ResourceUID::ID uid = ResourceUID::get_singleton()->text_to_id(uidt);
if (uid != ResourceUID::INVALID_ID && ResourceUID::get_singleton()->has_id(uid)) {
// If a UID is found and the path is valid, it will be used, otherwise, it falls back to the path.
path = ResourceUID::get_singleton()->get_id_path(uid);
} else {
WARN_PRINT(String(res_path + ":" + itos(lines) + " - ext_resource, invalid UUID: " + uidt + " - using text path instead: " + path).utf8().get_data());
}
}
if (path.find("://") == -1 && path.is_rel_path()) {
// path is relative to file being loaded, so convert to a resource path
path = ProjectSettings::get_singleton()->localize_path(local_path.get_base_dir().plus_file(path));
@ -746,7 +757,18 @@ void ResourceLoaderText::get_dependencies(FileAccess *p_f, List<String> *p_depen
String path = next_tag.fields["path"];
String type = next_tag.fields["type"];
if (path.find("://") == -1 && path.is_rel_path()) {
bool using_uid = false;
if (next_tag.fields.has("uid")) {
//if uid exists, return uid in text format, not the path
String uidt = next_tag.fields["uid"];
ResourceUID::ID uid = ResourceUID::get_singleton()->text_to_id(uidt);
if (uid != ResourceUID::INVALID_ID) {
path = ResourceUID::get_singleton()->id_to_text(uid);
using_uid = true;
}
}
if (!using_uid && path.find("://") == -1 && path.is_rel_path()) {
// path is relative to file being loaded, so convert to a resource path
path = ProjectSettings::get_singleton()->localize_path(local_path.get_base_dir().plus_file(path));
}
@ -819,6 +841,14 @@ Error ResourceLoaderText::rename_dependencies(FileAccess *p_f, const String &p_p
String id = next_tag.fields["id"];
String type = next_tag.fields["type"];
if (next_tag.fields.has("uid")) {
String uidt = next_tag.fields["uid"];
ResourceUID::ID uid = ResourceUID::get_singleton()->text_to_id(uidt);
if (uid != ResourceUID::INVALID_ID && ResourceUID::get_singleton()->has_id(uid)) {
// If a UID is found and the path is valid, it will be used, otherwise, it falls back to the path.
path = ResourceUID::get_singleton()->get_id_path(uid);
}
}
bool relative = false;
if (!path.begins_with("res://")) {
path = base_path.plus_file(path).simplify_path();
@ -835,7 +865,14 @@ Error ResourceLoaderText::rename_dependencies(FileAccess *p_f, const String &p_p
path = base_path.path_to_file(path);
}
fw->store_line("[ext_resource path=\"" + path + "\" type=\"" + type + "\" id=\"" + id + "\"]");
String s = "[ext_resource type=\"" + type + "\"";
ResourceUID::ID uid = ResourceSaver::get_resource_id_for_path(path);
if (uid != ResourceUID::INVALID_ID) {
s += " uid=\"" + ResourceUID::get_singleton()->id_to_text(uid) + "\"";
}
s += " path=\"" + path + "\" id=\"" + id + "\"]";
fw->store_line(s); // Bundled.
tag_end = f->get_position();
}
@ -921,6 +958,12 @@ void ResourceLoaderText::open(FileAccess *p_f, bool p_skip_first_tag) {
return;
}
if (tag.fields.has("uid")) {
res_uid = ResourceUID::get_singleton()->text_to_id(tag.fields["uid"]);
} else {
res_uid = ResourceUID::INVALID_ID;
}
if (tag.fields.has("load_steps")) {
resources_total = tag.fields["load_steps"];
} else {
@ -976,7 +1019,12 @@ Error ResourceLoaderText::save_as_binary(FileAccess *p_f, const String &p_path)
bs_save_unicode_string(wf.f, is_scene ? "PackedScene" : resource_type);
wf->store_64(0); //offset to import metadata, this is no longer used
for (int i = 0; i < 14; i++) {
f->store_32(ResourceFormatSaverBinaryInstance::FORMAT_FLAG_NAMED_SCENE_IDS | ResourceFormatSaverBinaryInstance::FORMAT_FLAG_UIDS);
f->store_64(res_uid);
for (int i = 0; i < 5; i++) {
wf->store_32(0); // reserved
}
@ -1018,9 +1066,15 @@ Error ResourceLoaderText::save_as_binary(FileAccess *p_f, const String &p_path)
String path = next_tag.fields["path"];
String type = next_tag.fields["type"];
String id = next_tag.fields["id"];
ResourceUID::ID uid = ResourceUID::INVALID_ID;
if (next_tag.fields.has("uid")) {
String uidt = next_tag.fields["uid"];
uid = ResourceUID::get_singleton()->text_to_id(uidt);
}
bs_save_unicode_string(wf.f, type);
bs_save_unicode_string(wf.f, path);
wf.f->store_64(uid);
int lindex = dummy_read.external_resources.size();
Ref<DummyResource> dr;
@ -1257,6 +1311,32 @@ String ResourceLoaderText::recognize(FileAccess *p_f) {
return tag.fields["type"];
}
ResourceUID::ID ResourceLoaderText::get_uid(FileAccess *p_f) {
error = OK;
lines = 1;
f = p_f;
stream.f = f;
ignore_resource_parsing = true;
VariantParser::Tag tag;
Error err = VariantParser::parse_tag(&stream, lines, error_text, tag);
if (err) {
_printerr();
return ResourceUID::INVALID_ID;
}
if (tag.fields.has("uid")) { //field is optional
String uidt = tag.fields["uid"];
return ResourceUID::get_singleton()->text_to_id(uidt);
}
return ResourceUID::INVALID_ID;
}
/////////////////////
RES ResourceFormatLoaderText::load(const String &p_path, const String &p_original_path, Error *r_error, bool p_use_sub_threads, float *r_progress, CacheMode p_cache_mode) {
@ -1277,7 +1357,6 @@ RES ResourceFormatLoaderText::load(const String &p_path, const String &p_origina
loader.local_path = ProjectSettings::get_singleton()->localize_path(path);
loader.progress = r_progress;
loader.res_path = loader.local_path;
//loader.set_local_path( ProjectSettings::get_singleton()->localize_path(p_path) );
loader.open(f);
err = loader.load();
if (r_error) {
@ -1330,11 +1409,28 @@ String ResourceFormatLoaderText::get_resource_type(const String &p_path) const {
ResourceLoaderText loader;
loader.local_path = ProjectSettings::get_singleton()->localize_path(p_path);
loader.res_path = loader.local_path;
//loader.set_local_path( ProjectSettings::get_singleton()->localize_path(p_path) );
String r = loader.recognize(f);
return ClassDB::get_compatibility_remapped_class(r);
}
ResourceUID::ID ResourceFormatLoaderText::get_resource_uid(const String &p_path) const {
String ext = p_path.get_extension().to_lower();
if (ext != "tscn" && ext != "tres") {
return ResourceUID::INVALID_ID;
}
FileAccess *f = FileAccess::open(p_path, FileAccess::READ);
if (!f) {
return ResourceUID::INVALID_ID; //could not read
}
ResourceLoaderText loader;
loader.local_path = ProjectSettings::get_singleton()->localize_path(p_path);
loader.res_path = loader.local_path;
return loader.get_uid(f);
}
void ResourceFormatLoaderText::get_dependencies(const String &p_path, List<String> *p_dependencies, bool p_add_types) {
FileAccess *f = FileAccess::open(p_path, FileAccess::READ);
if (!f) {
@ -1344,7 +1440,6 @@ void ResourceFormatLoaderText::get_dependencies(const String &p_path, List<Strin
ResourceLoaderText loader;
loader.local_path = ProjectSettings::get_singleton()->localize_path(p_path);
loader.res_path = loader.local_path;
//loader.set_local_path( ProjectSettings::get_singleton()->localize_path(p_path) );
loader.get_dependencies(f, p_dependencies, p_add_types);
}
@ -1357,7 +1452,6 @@ Error ResourceFormatLoaderText::rename_dependencies(const String &p_path, const
ResourceLoaderText loader;
loader.local_path = ProjectSettings::get_singleton()->localize_path(p_path);
loader.res_path = loader.local_path;
//loader.set_local_path( ProjectSettings::get_singleton()->localize_path(p_path) );
return loader.rename_dependencies(f, p_path, p_map);
}
@ -1373,7 +1467,6 @@ Error ResourceFormatLoaderText::convert_file_to_binary(const String &p_src_path,
const String &path = p_src_path;
loader.local_path = ProjectSettings::get_singleton()->localize_path(path);
loader.res_path = loader.local_path;
//loader.set_local_path( ProjectSettings::get_singleton()->localize_path(p_path) );
loader.open(f);
return loader.save_as_binary(f, p_dst_path);
}
@ -1548,6 +1641,12 @@ Error ResourceFormatSaverTextInstance::save(const String &p_path, const RES &p_r
}
title += "format=" + itos(FORMAT_VERSION) + "";
ResourceUID::ID uid = ResourceSaver::get_resource_id_for_path(local_path, true);
if (uid != ResourceUID::INVALID_ID) {
title += " uid=\"" + ResourceUID::get_singleton()->id_to_text(uid) + "\"";
}
f->store_string(title);
f->store_line("]\n"); // One empty line.
}
@ -1612,7 +1711,14 @@ Error ResourceFormatSaverTextInstance::save(const String &p_path, const RES &p_r
for (int i = 0; i < sorted_er.size(); i++) {
String p = sorted_er[i].resource->get_path();
f->store_string("[ext_resource path=\"" + p + "\" type=\"" + sorted_er[i].resource->get_save_class() + "\" id=\"" + sorted_er[i].id + "\"]\n"); // Bundled.
String s = "[ext_resource type=\"" + sorted_er[i].resource->get_save_class() + "\"";
ResourceUID::ID uid = ResourceSaver::get_resource_id_for_path(p, false);
if (uid != ResourceUID::INVALID_ID) {
s += " uid=\"" + ResourceUID::get_singleton()->id_to_text(uid) + "\"";
}
s += " path=\"" + p + "\" id=\"" + sorted_er[i].id + "\"]\n";
f->store_string(s); // Bundled.
}
if (external_resources.size()) {

4
scene/resources/resource_format_text.h
View file

@ -74,6 +74,8 @@ class ResourceLoaderText {
mutable int lines = 0;
ResourceUID::ID res_uid = ResourceUID::INVALID_ID;
Map<String, String> remaps;
static Error _parse_sub_resources(void *p_self, VariantParser::Stream *p_stream, Ref<Resource> &r_res, int &line, String &r_err_str) { return reinterpret_cast<ResourceLoaderText *>(p_self)->_parse_sub_resource(p_stream, r_res, line, r_err_str); }
@ -120,6 +122,7 @@ public:
void open(FileAccess *p_f, bool p_skip_first_tag = false);
String recognize(FileAccess *p_f);
ResourceUID::ID get_uid(FileAccess *p_f);
void get_dependencies(FileAccess *p_f, List<String> *p_dependencies, bool p_add_types);
Error rename_dependencies(FileAccess *p_f, const String &p_path, const Map<String, String> &p_map);
@ -136,6 +139,7 @@ public:
virtual void get_recognized_extensions(List<String> *p_extensions) const;
virtual bool handles_type(const String &p_type) const;
virtual String get_resource_type(const String &p_path) const;
virtual ResourceUID::ID get_resource_uid(const String &p_path) const;
virtual void get_dependencies(const String &p_path, List<String> *p_dependencies, bool p_add_types = false);
virtual Error rename_dependencies(const String &p_path, const Map<String, String> &p_map);