From 7b8264ddc5c237a4daece558e75e7826fbc889e8 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Fri, 17 Apr 2026 06:37:18 -0700 Subject: [PATCH 001/134] vtracer: split disconnected pixels --- external/vtracer/src/converter.rs | 67 +++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 3 deletions(-) diff --git a/external/vtracer/src/converter.rs b/external/vtracer/src/converter.rs index 99751d89a..574a0ea4a 100644 --- a/external/vtracer/src/converter.rs +++ b/external/vtracer/src/converter.rs @@ -5,7 +5,7 @@ use std::{fs::File, io::Write}; use super::config::{ColorMode, Config, ConverterConfig, Hierarchical}; use super::svg::SvgFile; // use fastrand::Rng; -use visioncortex::color_clusters::{Cluster, ClusterIndex, Clusters, KeyingAction, HIERARCHICAL_MAX}; +use visioncortex::color_clusters::{Cluster, ClusterIndex, Clusters, ClustersView, HIERARCHICAL_MAX, KeyingAction}; use visioncortex::{Color, ColorImage, ColorName, PathSimplifyMode}; use crate::runner::{Runner, RunnerConfig}; @@ -113,6 +113,67 @@ fn should_key_image(img: &ColorImage, config: &ConverterConfig) -> bool { false } +fn to_image_with_holes(cluster: &Cluster, view: &ClustersView<'_>, hole: bool) -> visioncortex::BinaryImage { + let width = view.width as usize; + let height = view.height as usize; + let mut image = visioncortex::BinaryImage::new_w_h(width, height); + + for &i in cluster.iter() { + let x = (i as i32 % width as i32) ; + let y = (i as i32 / width as i32) ; + image.set_pixel(x as usize, y as usize, true); + } + + if hole { + for &i in cluster.holes.iter() { + let x = (i as i32 % width as i32) ; + let y = (i as i32 / width as i32) ; + image.set_pixel(x as usize, y as usize, false); + } + } + image +} + +fn new_cluster_from_indices(cluster: &Cluster, indices: &Vec, color: Color) -> Cluster { + let mut new_cluster = cluster.clone(); + new_cluster.indices = indices.clone(); + new_cluster.residue_sum.r = color.r as u32 * new_cluster.indices.len() as u32; + new_cluster.residue_sum.g = color.g as u32 * new_cluster.indices.len() as u32; + new_cluster.residue_sum.b = color.b as u32 * new_cluster.indices.len() as u32; + new_cluster.residue_sum.a = color.a as u32 * new_cluster.indices.len() as u32; + new_cluster.residue_sum.counter = new_cluster.indices.len() as u32; + new_cluster +} + +fn split_disconnected_pixels(cluster: &Cluster, view: &ClustersView<'_>) -> Vec{ + if cluster.indices.len() <= 1 { + return vec![cluster.clone()]; + } + // for all the indices in the cluster, check if a pixel is not connected to the cluster + let mut cluster = cluster.clone(); + let mut new_clusters = vec![]; + let himage = to_image_with_holes(&cluster, &view, true); + let indices = cluster.indices.clone(); + for (i, index) in indices.iter().enumerate().rev() { + let x = *index as usize % himage.width; + let y = *index as usize / himage.width; + if himage.get_pixel(x, y) == true { + // check all the ones adjacent to it; if they are not connected, create a new cluster + if (x + 1 >= himage.width as usize || himage.get_pixel(x + 1, y) == false) && + (x as i32 - 1 < 0 || himage.get_pixel(x - 1, y) == false) && + (y + 1 >= himage.height as usize || himage.get_pixel(x, y + 1) == false) && + (y as i32 - 1 < 0 || himage.get_pixel(x, y - 1) == false) { + cluster.indices.remove(i); + new_clusters.insert(0, new_cluster_from_indices(&cluster, &vec![*index], cluster.residue_color())); + } + } + } + if cluster.indices.len() > 0 { + new_clusters.insert(0, cluster); + } + new_clusters +} + fn remove_pixels_from_lower_layers(p_clusters: &Clusters) -> Vec{ let view = p_clusters.view(); @@ -169,11 +230,11 @@ fn remove_pixels_from_lower_layers(p_clusters: &Clusters) -> Vec{ new_cluster.residue_sum.b = color.b as u32 * new_cluster.indices.len() as u32; new_cluster.residue_sum.a = color.a as u32 * new_cluster.indices.len() as u32; new_cluster.residue_sum.counter = new_cluster.indices.len() as u32; - clusters.push(new_cluster); + clusters.extend(split_disconnected_pixels(&new_cluster, &view)); } continue; } - clusters.push(cluster); + clusters.extend(split_disconnected_pixels(&cluster, &view)); } clusters } From 3804134aee07e1318f1721c3e9a0ed929a0c0ddc Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Fri, 17 Apr 2026 06:37:18 -0700 Subject: [PATCH 002/134] fix svg tests --- tests/test_imagesaver.h | 54 +++++++++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/tests/test_imagesaver.h b/tests/test_imagesaver.h index 94bed6df4..b465b9b52 100644 --- a/tests/test_imagesaver.h +++ b/tests/test_imagesaver.h @@ -6,6 +6,11 @@ #include "utility/common.h" #include "utility/image_saver.h" namespace TestImageSaver { + +static inline String get_color_string(const Color &color) { + return vformat("(r: %d, g: %d, b: %d, a: %d)", color.get_r8(), color.get_g8(), color.get_b8(), color.get_a8()); +} + void test_svg_saving(const String &file, const String &test_files_dir, const String &output_dir) { Ref img = gdre::load_image_from_file(test_files_dir.path_join(file)); REQUIRE(img.is_valid()); @@ -13,22 +18,55 @@ void test_svg_saving(const String &file, const String &test_files_dir, const Str Ref resaved_image = gdre::load_image_from_file(output_dir.path_join(file)); REQUIRE(resaved_image.is_valid()); -#ifdef DEBUG_ENABLED if (resaved_image->get_data() != img->get_data()) { // save them both as pngs to the output path - auto a_path = output_dir.path_join(file.get_basename() + ".png"); - auto b_path = output_dir.path_join(file.get_basename() + "_resaved.png"); - img->save_png(a_path); - resaved_image->save_png(b_path); + Ref diff_image = Image::create_empty(img->get_width(), img->get_height(), false, Image::FORMAT_RGBA8); + Vector>> diff_colors; for (int i = 0; i < img->get_width(); i++) { for (int j = 0; j < img->get_height(); j++) { - CHECK(img->get_pixel(i, j) == resaved_image->get_pixel(i, j)); + Color img_color = img->get_pixel(i, j); + Color resaved_color = resaved_image->get_pixel(i, j); + if (img_color != resaved_color) { + bool foo = false; + } + if (abs(img_color.get_r8() - resaved_color.get_r8()) > 1 || + abs(img_color.get_g8() - resaved_color.get_g8()) > 1 || + abs(img_color.get_b8() - resaved_color.get_b8()) > 1 || + abs(img_color.get_a8() - resaved_color.get_a8()) > 1) { + Color diff = img_color - resaved_color; + diff.r = abs(diff.r); + diff.g = abs(diff.g); + diff.b = abs(diff.b); + diff.a = abs(diff.a); + if (diff.a == 0) { + diff.a = 1; + } + diff_image->set_pixel(i, j, diff); + diff_colors.push_back({ Vector2i(i, j), { img->get_pixel(i, j), resaved_image->get_pixel(i, j) } }); + } else { + diff_image->set_pixel(i, j, Color(0, 0, 0, 0)); + } } } - } + String error_message = ""; + if (diff_colors.size() > 0) { +#ifdef DEBUG_ENABLED + auto a_path = output_dir.path_join(file.get_basename() + ".png"); + auto b_path = output_dir.path_join(file.get_basename() + "_resaved.png"); + img->save_png(a_path); + resaved_image->save_png(b_path); + diff_image->save_png(output_dir.path_join(file.get_basename() + "_diff.png")); #endif + error_message = vformat("%d pixels differ in %s: ", diff_colors.size(), file); + for (const auto &diff : diff_colors) { + error_message += vformat("Diff at %d, %d: %s vs %s", diff.first.x, diff.first.y, get_color_string(diff.second.first), get_color_string(diff.second.second)) + "\n"; + } + } + CHECK(error_message == ""); + } + // the svg that we save should be pixel-for-pixel accurate to the original when loaded as raster images - CHECK(resaved_image->get_data() == img->get_data()); + // CHECK(resaved_image->get_data() == img->get_data()); } } //namespace TestImageSaver From f289de559aefe6041ec8362ce887594b976bd9c4 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Fri, 17 Apr 2026 06:37:18 -0700 Subject: [PATCH 003/134] Fix bytecode function arg count testing --- bytecode/bytecode_base.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/bytecode/bytecode_base.cpp b/bytecode/bytecode_base.cpp index d91722071..0a1cc6567 100644 --- a/bytecode/bytecode_base.cpp +++ b/bytecode/bytecode_base.cpp @@ -1479,6 +1479,8 @@ int GDScriptDecomp::get_func_arg_count_and_params(int curr_pos, const Vector 0) { r_arguments.push_back(curr_arg); + curr_arg.clear(); } break; } From d7c7a32881869047242e0dacde03bfd35293c354 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Fri, 17 Apr 2026 06:37:18 -0700 Subject: [PATCH 004/134] fix rust warnings --- external/vtracer/src/converter.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/external/vtracer/src/converter.rs b/external/vtracer/src/converter.rs index 574a0ea4a..d29ad1e18 100644 --- a/external/vtracer/src/converter.rs +++ b/external/vtracer/src/converter.rs @@ -119,15 +119,15 @@ fn to_image_with_holes(cluster: &Cluster, view: &ClustersView<'_>, hole: bool) - let mut image = visioncortex::BinaryImage::new_w_h(width, height); for &i in cluster.iter() { - let x = (i as i32 % width as i32) ; - let y = (i as i32 / width as i32) ; + let x = i as i32 % width as i32 ; + let y = i as i32 / width as i32 ; image.set_pixel(x as usize, y as usize, true); } if hole { for &i in cluster.holes.iter() { - let x = (i as i32 % width as i32) ; - let y = (i as i32 / width as i32) ; + let x = i as i32 % width as i32 ; + let y = i as i32 / width as i32 ; image.set_pixel(x as usize, y as usize, false); } } From 8d9a66042665baa6d22e304c594c3dfcd839adbe Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Fri, 17 Apr 2026 06:37:18 -0700 Subject: [PATCH 005/134] resync with master @ 90113707f2f --- .github/workflows/all_builds.yml | 2 +- .scripts/rebase_godot.sh | 2 -- README.md | 2 +- compat/fake_csharp_script.cpp | 5 ----- compat/fake_csharp_script.h | 1 - compat/fake_gdscript.cpp | 5 ----- compat/fake_gdscript.h | 1 - compat/fake_script.cpp | 4 ---- compat/fake_script.h | 1 - compat/texture_loader_compat.cpp | 2 +- exporters/texture_exporter.cpp | 2 +- utility/gdre_config.cpp | 1 + utility/gdre_settings.cpp | 2 +- 13 files changed, 6 insertions(+), 24 deletions(-) diff --git a/.github/workflows/all_builds.yml b/.github/workflows/all_builds.yml index 7cc6336d6..29f93da95 100644 --- a/.github/workflows/all_builds.yml +++ b/.github/workflows/all_builds.yml @@ -28,7 +28,7 @@ env: # TODO: change this back to godotengine/godot and target master when #109685 and #109475 are merged GODOT_REPOSITORY: nikitalita/godot # Change the README too - GODOT_MAIN_SYNC_REF: gdre-wb-babc272d44e + GODOT_MAIN_SYNC_REF: gdre-wb-90113707f2f SCONSFLAGS: verbose=yes warnings=all werror=no module_text_server_fb_enabled=yes minizip=yes deprecated=yes SCONSFLAGS_TEMPLATE: disable_path_overrides=no no_editor_splash=yes module_camera_enabled=no module_mobile_vr_enabled=no module_upnp_enabled=no module_websocket_enabled=no module_csg_enabled=yes module_gridmap_enabled=yes use_static_cpp=yes builtin_freetype=yes builtin_libpng=yes builtin_zlib=yes builtin_libwebp=yes builtin_libvorbis=yes builtin_libogg=yes disable_3d=no SCONS_CACHE_MSVC_CONFIG: true diff --git a/.scripts/rebase_godot.sh b/.scripts/rebase_godot.sh index e74d0f74a..17a3e0836 100755 --- a/.scripts/rebase_godot.sh +++ b/.scripts/rebase_godot.sh @@ -31,10 +31,8 @@ BRANCHES_TO_MERGE=( material-fix-deprecated-param fix-pack-error convert-3.x-escn - fix-svg fix-compat-array-shapes fix-diraccess-windows - gltf-fix-value-track-interpolation ) # set fail on error diff --git a/README.md b/README.md index 8caba56a7..81e23f174 100644 --- a/README.md +++ b/README.md @@ -149,7 +149,7 @@ Note: Make sure to build the editor build first, and to launch the editor to edi ### Requirements -[Our fork of godot](https://github.com/nikitalita/godot) @ branch `gdre-wb-babc272d44e` +[Our fork of godot](https://github.com/nikitalita/godot) @ branch `gdre-wb-90113707f2f` - Support for building on 3.x has been dropped and no new features are being pushed - Godot RE Tools still retains the ability to decompile 3.x and 2.x projects, however. diff --git a/compat/fake_csharp_script.cpp b/compat/fake_csharp_script.cpp index 8c93cae19..5ab461387 100644 --- a/compat/fake_csharp_script.cpp +++ b/compat/fake_csharp_script.cpp @@ -100,11 +100,6 @@ PlaceHolderScriptInstance *FakeCSharpScript::placeholder_instance_create(Object return si; } -bool FakeCSharpScript::instance_has(const Object *p_this) const { - // For now, return false as we don't track instances - return false; -} - PropertyHint string_to_property_hint(const String &p_string) { String name = p_string.to_upper(); if (name == "NONE") { diff --git a/compat/fake_csharp_script.h b/compat/fake_csharp_script.h index 4652de38d..476cd40e7 100644 --- a/compat/fake_csharp_script.h +++ b/compat/fake_csharp_script.h @@ -64,7 +64,6 @@ class FakeCSharpScript : public FakeScript { // this may not work in all scripts, will return empty if so virtual ScriptInstance *instance_create(Object *p_this) override; virtual PlaceHolderScriptInstance *placeholder_instance_create(Object *p_this) override; - virtual bool instance_has(const Object *p_this) const override; // virtual bool has_source_code() const override; // virtual String get_source_code() const override; diff --git a/compat/fake_gdscript.cpp b/compat/fake_gdscript.cpp index 6d99b3924..8ac437c98 100644 --- a/compat/fake_gdscript.cpp +++ b/compat/fake_gdscript.cpp @@ -121,11 +121,6 @@ PlaceHolderScriptInstance *FakeGDScript::placeholder_instance_create(Object *p_t return si; } -bool FakeGDScript::instance_has(const Object *p_this) const { - // TODO? - return true; -} - void FakeGDScript::set_source_code(const String &p_code) { is_binary = false; source = p_code; diff --git a/compat/fake_gdscript.h b/compat/fake_gdscript.h index 4024c1bc8..99db58639 100644 --- a/compat/fake_gdscript.h +++ b/compat/fake_gdscript.h @@ -75,7 +75,6 @@ class FakeGDScript : public FakeScript { // this may not work in all scripts, will return empty if so virtual ScriptInstance *instance_create(Object *p_this) override; virtual PlaceHolderScriptInstance *placeholder_instance_create(Object *p_this) override; - virtual bool instance_has(const Object *p_this) const override; // virtual bool has_source_code() const override; // virtual String get_source_code() const override; diff --git a/compat/fake_script.cpp b/compat/fake_script.cpp index 4be44a53f..7b8737423 100644 --- a/compat/fake_script.cpp +++ b/compat/fake_script.cpp @@ -103,10 +103,6 @@ PlaceHolderScriptInstance *FakeScript::placeholder_instance_create(Object *p_thi return si; } -bool FakeScript::instance_has(const Object *p_this) const { - return true; -} - bool FakeScript::has_source_code() const { return !source.is_empty(); } diff --git a/compat/fake_script.h b/compat/fake_script.h index 7ab946a67..c1a59b4ea 100644 --- a/compat/fake_script.h +++ b/compat/fake_script.h @@ -43,7 +43,6 @@ class FakeScript : public Script { virtual StringName get_instance_base_type() const override; virtual ScriptInstance *instance_create(Object *p_this) override; virtual PlaceHolderScriptInstance *placeholder_instance_create(Object *p_this) override; - virtual bool instance_has(const Object *p_this) const override; virtual bool has_source_code() const override; virtual String get_source_code() const override; diff --git a/compat/texture_loader_compat.cpp b/compat/texture_loader_compat.cpp index 4c26731d6..9f0eba87f 100644 --- a/compat/texture_loader_compat.cpp +++ b/compat/texture_loader_compat.cpp @@ -11,7 +11,7 @@ #include "core/error/error_list.h" #include "core/error/error_macros.h" #include "core/io/file_access.h" -#include "core/io/image_loader.h" +#include "core/io/image_resource_format.h" #include "core/io/missing_resource.h" #include "core/variant/dictionary.h" #include "scene/resources/compressed_texture.h" diff --git a/exporters/texture_exporter.cpp b/exporters/texture_exporter.cpp index dcf5fe4a7..2d7a05e47 100644 --- a/exporters/texture_exporter.cpp +++ b/exporters/texture_exporter.cpp @@ -9,7 +9,7 @@ #include "core/error/error_list.h" #include "core/io/file_access.h" -#include "core/io/image_loader.h" +#include "core/io/image_resource_format.h" #include "core/io/resource_loader.h" #include "exporters/export_report.h" #include "scene/resources/atlas_texture.h" diff --git a/utility/gdre_config.cpp b/utility/gdre_config.cpp index 827c25d11..9839b4edf 100644 --- a/utility/gdre_config.cpp +++ b/utility/gdre_config.cpp @@ -5,6 +5,7 @@ #include "core/io/config_file.h" #include "core/io/json.h" #include "core/object/class_db.h" +#include "core/object/worker_thread_pool.h" #include "core/os/os.h" #include "gdre_logger.h" #include "gdre_settings.h" diff --git a/utility/gdre_settings.cpp b/utility/gdre_settings.cpp index b9b005ae8..eddf707a1 100644 --- a/utility/gdre_settings.cpp +++ b/utility/gdre_settings.cpp @@ -15,7 +15,7 @@ #include "crypto/custom_decryptor.h" #include "exporters/translation_exporter.h" #include "main/main.h" -#include "modules/gdscript/gdscript.h" +#include "modules/gdscript/gdscript_resource_format.h" #include "modules/zip/zip_reader.h" #include "plugin_manager/plugin_manager.h" #include "utility/common.h" From a7c26db1b0bce0b672ebad19bfeaef583bfe4e88 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Fri, 17 Apr 2026 06:37:18 -0700 Subject: [PATCH 006/134] fix warnings --- tests/test_imagesaver.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/tests/test_imagesaver.h b/tests/test_imagesaver.h index b465b9b52..cf695ba78 100644 --- a/tests/test_imagesaver.h +++ b/tests/test_imagesaver.h @@ -26,9 +26,6 @@ void test_svg_saving(const String &file, const String &test_files_dir, const Str for (int j = 0; j < img->get_height(); j++) { Color img_color = img->get_pixel(i, j); Color resaved_color = resaved_image->get_pixel(i, j); - if (img_color != resaved_color) { - bool foo = false; - } if (abs(img_color.get_r8() - resaved_color.get_r8()) > 1 || abs(img_color.get_g8() - resaved_color.get_g8()) > 1 || abs(img_color.get_b8() - resaved_color.get_b8()) > 1 || From 817b87a0ad6624de288571f7efb30f810f013c98 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Fri, 17 Apr 2026 06:37:18 -0700 Subject: [PATCH 007/134] Fix ResourceCompatLoader bindings after resync --- compat/resource_loader_compat.cpp | 86 +++++++++++++++-------------- compat/resource_loader_compat.h | 47 ++++++++++------ register_types.cpp | 2 +- standalone/gdre_media_player.gd | 4 +- standalone/gdre_resource_preview.gd | 8 +-- 5 files changed, 84 insertions(+), 63 deletions(-) diff --git a/compat/resource_loader_compat.cpp b/compat/resource_loader_compat.cpp index 6811c232a..76da52e5b 100644 --- a/compat/resource_loader_compat.cpp +++ b/compat/resource_loader_compat.cpp @@ -207,7 +207,7 @@ Ref ResourceCompatLoader::gltf_load(const String &p_path, const String return ResourceCompatLoader::custom_load(p_path, p_type_hint, ResourceInfo::LoadType::GLTF_LOAD, r_error); } -Ref ResourceCompatLoader::real_load(const String &p_path, const String &p_type_hint, Error *r_error, ResourceFormatLoader::CacheMode p_cache_mode) { +Ref ResourceCompatLoader::real_load(const String &p_path, const String &p_type_hint, Error *r_error, ResourceCompatLoader::CacheMode p_cache_mode) { return ResourceCompatLoader::custom_load(p_path, p_type_hint, ResourceInfo::LoadType::REAL_LOAD, r_error, true, p_cache_mode); } namespace { @@ -225,7 +225,7 @@ String _validate_local_path(const String &p_path) { } //namespace thread_local HashSet currently_loading_paths; -Ref ResourceCompatLoader::custom_load(const String &p_path, const String &p_type_hint, ResourceInfo::LoadType p_type, Error *r_error, bool use_threads, ResourceFormatLoader::CacheMode p_cache_mode) { +Ref ResourceCompatLoader::custom_load(const String &p_path, const String &p_type_hint, ResourceInfo::LoadType p_type, Error *r_error, bool use_threads, ResourceCompatLoader::CacheMode p_cache_mode) { String local_path = _validate_local_path(p_path); String res_path = GDRESettings::get_singleton()->get_mapped_path(p_path); bool is_real_load = p_type == ResourceInfo::LoadType::REAL_LOAD || p_type == ResourceInfo::LoadType::GLTF_LOAD; @@ -271,7 +271,7 @@ Ref ResourceCompatLoader::custom_load(const String &p_path, const Stri return res; } -Ref ResourceCompatLoader::load_with_real_resource_loader(const String &p_path, const String &p_type_hint, Error *r_error, bool use_threads, ResourceFormatLoader::CacheMode p_cache_mode) { +Ref ResourceCompatLoader::load_with_real_resource_loader(const String &p_path, const String &p_type_hint, Error *r_error, bool use_threads, ResourceCompatLoader::CacheMode p_cache_mode) { String local_path = _validate_local_path(p_path); if (use_threads) { return ResourceLoader::load(local_path, p_type_hint, p_cache_mode, r_error); @@ -569,18 +569,6 @@ String ResourceCompatLoader::get_resource_type(const String &p_path) { return loader->get_resource_type(p_path); } -Vector ResourceCompatLoader::_get_dependencies(const String &p_path, bool p_add_types) { - auto loader = get_loader_for_path(p_path, ""); - ERR_FAIL_COND_V_MSG(loader.is_null(), Vector(), "Failed to load resource '" + p_path + "'. ResourceFormatLoader::load was not implemented for this resource type."); - List dependencies; - loader->get_dependencies(p_path, &dependencies, p_add_types); - Vector deps; - for (List::Element *E = dependencies.front(); E; E = E->next()) { - deps.push_back(E->get()); - } - return deps; -} - bool ResourceCompatLoader::is_default_gltf_load() { return doing_gltf_load; } @@ -645,25 +633,38 @@ String ResourceCompatConverter::get_resource_name(const Ref &re } return name; } +namespace CoreBind { + +Vector ResourceCompatLoader::_get_dependencies(const String &p_path, bool p_add_types) { + auto loader = ::ResourceCompatLoader::get_loader_for_path(p_path, ""); + ERR_FAIL_COND_V_MSG(loader.is_null(), Vector(), "Failed to load resource '" + p_path + "'. ResourceFormatLoader::load was not implemented for this resource type."); + List dependencies; + loader->get_dependencies(p_path, &dependencies, p_add_types); + Vector deps; + for (List::Element *E = dependencies.front(); E; E = E->next()) { + deps.push_back(E->get()); + } + return deps; +} Ref ResourceCompatLoader::_fake_load(const String &p_path, const String &p_type_hint) { - return fake_load(p_path, p_type_hint, nullptr); + return ::ResourceCompatLoader::fake_load(p_path, p_type_hint, nullptr); } Ref ResourceCompatLoader::_non_global_load(const String &p_path, const String &p_type_hint) { - return non_global_load(p_path, p_type_hint, nullptr); + return ::ResourceCompatLoader::non_global_load(p_path, p_type_hint, nullptr); } Ref ResourceCompatLoader::_gltf_load(const String &p_path, const String &p_type_hint) { - return gltf_load(p_path, p_type_hint, nullptr); + return ::ResourceCompatLoader::gltf_load(p_path, p_type_hint, nullptr); } -Ref ResourceCompatLoader::_real_load(const String &p_path, const String &p_type_hint, ResourceFormatLoader::CacheMode p_cache_mode) { - return real_load(p_path, p_type_hint, nullptr, p_cache_mode); +Ref ResourceCompatLoader::_real_load(const String &p_path, const String &p_type_hint, CacheMode p_cache_mode) { + return ::ResourceCompatLoader::real_load(p_path, p_type_hint, nullptr, (::ResourceCompatLoader::CacheMode)p_cache_mode); } Dictionary ResourceCompatLoader::_get_resource_info(const String &p_path, const String &p_type_hint) { - Ref info = get_resource_info(p_path, p_type_hint, nullptr); + Ref info = ::ResourceCompatLoader::get_resource_info(p_path, p_type_hint, nullptr); if (info.is_valid()) { return info->to_dict(); } @@ -671,35 +672,40 @@ Dictionary ResourceCompatLoader::_get_resource_info(const String &p_path, const } void ResourceCompatLoader::_bind_methods() { + BIND_ENUM_CONSTANT(CACHE_MODE_IGNORE); + BIND_ENUM_CONSTANT(CACHE_MODE_REUSE); + BIND_ENUM_CONSTANT(CACHE_MODE_REPLACE); + BIND_ENUM_CONSTANT(CACHE_MODE_IGNORE_DEEP); + BIND_ENUM_CONSTANT(CACHE_MODE_REPLACE_DEEP); ClassDB::bind_static_method(get_class_static(), D_METHOD("fake_load", "path", "type_hint"), &ResourceCompatLoader::_fake_load, DEFVAL("")); ClassDB::bind_static_method(get_class_static(), D_METHOD("non_global_load", "path", "type_hint"), &ResourceCompatLoader::_non_global_load, DEFVAL("")); ClassDB::bind_static_method(get_class_static(), D_METHOD("gltf_load", "path", "type_hint"), &ResourceCompatLoader::_gltf_load, DEFVAL("")); - ClassDB::bind_static_method(get_class_static(), D_METHOD("real_load", "path", "type_hint", "cache_mode"), &ResourceCompatLoader::_real_load, DEFVAL(""), DEFVAL(ResourceFormatLoader::CACHE_MODE_REUSE)); - ClassDB::bind_static_method(get_class_static(), D_METHOD("add_resource_format_loader", "loader", "at_front"), &ResourceCompatLoader::add_resource_format_loader, DEFVAL(false)); - ClassDB::bind_static_method(get_class_static(), D_METHOD("remove_resource_format_loader", "loader"), &ResourceCompatLoader::remove_resource_format_loader); - ClassDB::bind_static_method(get_class_static(), D_METHOD("add_resource_object_converter", "converter", "at_front"), &ResourceCompatLoader::add_resource_object_converter, DEFVAL(false)); - ClassDB::bind_static_method(get_class_static(), D_METHOD("remove_resource_object_converter", "converter"), &ResourceCompatLoader::remove_resource_object_converter); + ClassDB::bind_static_method(get_class_static(), D_METHOD("real_load", "path", "type_hint", "cache_mode"), &ResourceCompatLoader::_real_load, DEFVAL(""), DEFVAL(CACHE_MODE_REUSE)); + ClassDB::bind_static_method(get_class_static(), D_METHOD("add_resource_format_loader", "loader", "at_front"), &::ResourceCompatLoader::add_resource_format_loader, DEFVAL(false)); + ClassDB::bind_static_method(get_class_static(), D_METHOD("remove_resource_format_loader", "loader"), &::ResourceCompatLoader::remove_resource_format_loader); + ClassDB::bind_static_method(get_class_static(), D_METHOD("add_resource_object_converter", "converter", "at_front"), &::ResourceCompatLoader::add_resource_object_converter, DEFVAL(false)); + ClassDB::bind_static_method(get_class_static(), D_METHOD("remove_resource_object_converter", "converter"), &::ResourceCompatLoader::remove_resource_object_converter); ClassDB::bind_static_method(get_class_static(), D_METHOD("get_resource_info", "path", "type_hint"), &ResourceCompatLoader::_get_resource_info, DEFVAL("")); ClassDB::bind_static_method(get_class_static(), D_METHOD("get_dependencies", "path", "add_types"), &ResourceCompatLoader::_get_dependencies, DEFVAL(false)); - ClassDB::bind_static_method(get_class_static(), D_METHOD("to_text", "path", "dst", "flags", "original_path"), &ResourceCompatLoader::to_text, DEFVAL(0), DEFVAL("")); - ClassDB::bind_static_method(get_class_static(), D_METHOD("to_binary", "path", "dst", "flags"), &ResourceCompatLoader::to_binary, DEFVAL(0)); - ClassDB::bind_static_method(get_class_static(), D_METHOD("make_globally_available"), &ResourceCompatLoader::make_globally_available); - ClassDB::bind_static_method(get_class_static(), D_METHOD("unmake_globally_available"), &ResourceCompatLoader::unmake_globally_available); - ClassDB::bind_static_method(get_class_static(), D_METHOD("is_globally_available"), &ResourceCompatLoader::is_globally_available); - ClassDB::bind_static_method(get_class_static(), D_METHOD("set_default_gltf_load", "enable"), &ResourceCompatLoader::set_default_gltf_load); - ClassDB::bind_static_method(get_class_static(), D_METHOD("is_default_gltf_load"), &ResourceCompatLoader::is_default_gltf_load); - ClassDB::bind_static_method(get_class_static(), D_METHOD("handles_resource", "path", "type_hint"), &ResourceCompatLoader::handles_resource, DEFVAL("")); - ClassDB::bind_static_method(get_class_static(), D_METHOD("get_resource_script_class", "path"), &ResourceCompatLoader::get_resource_script_class); - ClassDB::bind_static_method(get_class_static(), D_METHOD("get_resource_type", "path"), &ResourceCompatLoader::get_resource_type); - ClassDB::bind_static_method(get_class_static(), D_METHOD("save_custom", "resource", "path", "ver_major", "ver_minor"), &ResourceCompatLoader::save_custom); - ClassDB::bind_static_method(get_class_static(), D_METHOD("resource_to_string", "path", "skip_cr"), &ResourceCompatLoader::resource_to_string, DEFVAL(true)); + ClassDB::bind_static_method(get_class_static(), D_METHOD("to_text", "path", "dst", "flags", "original_path"), &::ResourceCompatLoader::to_text, DEFVAL(0), DEFVAL("")); + ClassDB::bind_static_method(get_class_static(), D_METHOD("to_binary", "path", "dst", "flags"), &::ResourceCompatLoader::to_binary, DEFVAL(0)); + ClassDB::bind_static_method(get_class_static(), D_METHOD("make_globally_available"), &::ResourceCompatLoader::make_globally_available); + ClassDB::bind_static_method(get_class_static(), D_METHOD("unmake_globally_available"), &::ResourceCompatLoader::unmake_globally_available); + ClassDB::bind_static_method(get_class_static(), D_METHOD("is_globally_available"), &::ResourceCompatLoader::is_globally_available); + ClassDB::bind_static_method(get_class_static(), D_METHOD("set_default_gltf_load", "enable"), &::ResourceCompatLoader::set_default_gltf_load); + ClassDB::bind_static_method(get_class_static(), D_METHOD("is_default_gltf_load"), &::ResourceCompatLoader::is_default_gltf_load); + ClassDB::bind_static_method(get_class_static(), D_METHOD("handles_resource", "path", "type_hint"), &::ResourceCompatLoader::handles_resource, DEFVAL("")); + ClassDB::bind_static_method(get_class_static(), D_METHOD("get_resource_script_class", "path"), &::ResourceCompatLoader::get_resource_script_class); + ClassDB::bind_static_method(get_class_static(), D_METHOD("get_resource_type", "path"), &::ResourceCompatLoader::get_resource_type); + ClassDB::bind_static_method(get_class_static(), D_METHOD("save_custom", "resource", "path", "ver_major", "ver_minor"), &::ResourceCompatLoader::save_custom); + ClassDB::bind_static_method(get_class_static(), D_METHOD("resource_to_string", "path", "skip_cr"), &::ResourceCompatLoader::resource_to_string, DEFVAL(true)); ClassDB::bind_integer_constant(get_class_static(), "LoadType", "FAKE_LOAD", ResourceInfo::FAKE_LOAD); ClassDB::bind_integer_constant(get_class_static(), "LoadType", "NON_GLOBAL_LOAD", ResourceInfo::NON_GLOBAL_LOAD); ClassDB::bind_integer_constant(get_class_static(), "LoadType", "GLTF_LOAD", ResourceInfo::GLTF_LOAD); ClassDB::bind_integer_constant(get_class_static(), "LoadType", "REAL_LOAD", ResourceInfo::REAL_LOAD); } - -Ref CompatFormatLoader::custom_load(const String &p_path, const String &p_original_path, ResourceInfo::LoadType p_type, Error *r_error, bool use_threads, ResourceFormatLoader::CacheMode p_cache_mode) { +} //namespace CoreBind +Ref CompatFormatLoader::custom_load(const String &p_path, const String &p_original_path, ResourceInfo::LoadType p_type, Error *r_error, bool use_threads, ResourceCompatLoader::CacheMode p_cache_mode) { if (r_error) { *r_error = ERR_UNAVAILABLE; } diff --git a/compat/resource_loader_compat.h b/compat/resource_loader_compat.h index 5286d5ae6..b4d6e1001 100644 --- a/compat/resource_loader_compat.h +++ b/compat/resource_loader_compat.h @@ -9,9 +9,7 @@ class CompatFormatLoader; class CompatFormatSaver; class ResourceCompatConverter; -class ResourceCompatLoader : public Object { - GDCLASS(ResourceCompatLoader, Object); - +class ResourceCompatLoader { enum { MAX_LOADERS = 64, MAX_CONVERTERS = 8192, @@ -25,26 +23,18 @@ class ResourceCompatLoader : public Object { static bool globally_available; protected: - static Ref _fake_load(const String &p_path, const String &p_type_hint = ""); - static Ref _non_global_load(const String &p_path, const String &p_type_hint = ""); - static Ref _gltf_load(const String &p_path, const String &p_type_hint = ""); - static Ref _real_load(const String &p_path, const String &p_type_hint = "", ResourceFormatLoader::CacheMode p_cache_mode = ResourceFormatLoader::CACHE_MODE_REUSE); - static Dictionary _get_resource_info(const String &p_path, const String &p_type_hint = ""); - static Vector _get_dependencies(const String &p_path, bool p_add_types); - - static void _bind_methods(); - static Ref _load_for_text_conversion(const String &p_path, const String &original_path = "", Error *r_error = nullptr); public: + using CacheMode = ResourceLoaderConstants::CacheMode; static ResourceInfo::LoadType get_default_load_type(); static Ref fake_load(const String &p_path, const String &p_type_hint = "", Error *r_error = nullptr); static Ref non_global_load(const String &p_path, const String &p_type_hint = "", Error *r_error = nullptr); static Ref gltf_load(const String &p_path, const String &p_type_hint = "", Error *r_error = nullptr); - static Ref real_load(const String &p_path, const String &p_type_hint = "", Error *r_error = nullptr, ResourceFormatLoader::CacheMode p_cache_mode = ResourceFormatLoader::CACHE_MODE_REUSE); - static Ref custom_load(const String &p_path, const String &p_type_hint = "", ResourceInfo::LoadType p_type = ResourceInfo::LoadType::REAL_LOAD, Error *r_error = nullptr, bool use_threads = true, ResourceFormatLoader::CacheMode p_cache_mode = ResourceFormatLoader::CACHE_MODE_REUSE); - static Ref load_with_real_resource_loader(const String &p_path, const String &p_type_hint = "", Error *r_error = nullptr, bool use_threads = true, ResourceFormatLoader::CacheMode p_cache_mode = ResourceFormatLoader::CACHE_MODE_REUSE); + static Ref real_load(const String &p_path, const String &p_type_hint = "", Error *r_error = nullptr, CacheMode p_cache_mode = ResourceFormatLoader::CACHE_MODE_REUSE); + static Ref custom_load(const String &p_path, const String &p_type_hint = "", ResourceInfo::LoadType p_type = ResourceInfo::LoadType::REAL_LOAD, Error *r_error = nullptr, bool use_threads = true, CacheMode p_cache_mode = ResourceFormatLoader::CACHE_MODE_REUSE); + static Ref load_with_real_resource_loader(const String &p_path, const String &p_type_hint = "", Error *r_error = nullptr, bool use_threads = true, CacheMode p_cache_mode = ResourceFormatLoader::CACHE_MODE_REUSE); static void add_resource_format_loader(Ref p_format_loader, bool p_at_front = false); static void remove_resource_format_loader(Ref p_format_loader); static void add_resource_object_converter(Ref p_converter, bool p_at_front = false); @@ -78,7 +68,7 @@ class CompatFormatLoader : public ResourceFormatLoader { GDCLASS(CompatFormatLoader, ResourceFormatLoader); public: - virtual Ref custom_load(const String &p_path, const String &p_original_path, ResourceInfo::LoadType p_type, Error *r_error = nullptr, bool use_threads = true, ResourceFormatLoader::CacheMode p_cache_mode = CACHE_MODE_REUSE); + virtual Ref custom_load(const String &p_path, const String &p_original_path, ResourceInfo::LoadType p_type, Error *r_error = nullptr, bool use_threads = true, ResourceCompatLoader::CacheMode p_cache_mode = CACHE_MODE_REUSE); virtual Ref get_resource_info(const String &p_path, Error *r_error) const; virtual bool handles_fake_load() const; @@ -215,3 +205,28 @@ class ResourceCompatConverter : public RefCounted { static Ref set_real_from_missing_resource(Ref mr, Ref res, ResourceInfo::LoadType load_type, const HashMap &prop_map = {}); static bool is_external_resource(Ref mr); }; + +namespace CoreBind { +class ResourceCompatLoader : public Object { + GDCLASS(ResourceCompatLoader, Object); + enum CacheMode { + CACHE_MODE_IGNORE, + CACHE_MODE_REUSE, + CACHE_MODE_REPLACE, + CACHE_MODE_IGNORE_DEEP, + CACHE_MODE_REPLACE_DEEP, + }; + +protected: + static void _bind_methods(); + +public: + static Ref _fake_load(const String &p_path, const String &p_type_hint = ""); + static Ref _non_global_load(const String &p_path, const String &p_type_hint = ""); + static Ref _gltf_load(const String &p_path, const String &p_type_hint = ""); + static Dictionary _get_resource_info(const String &p_path, const String &p_type_hint = ""); + static Vector _get_dependencies(const String &p_path, bool p_add_types); + static Ref _real_load(const String &p_path, const String &p_type_hint = "", CacheMode p_cache_mode = CACHE_MODE_REUSE); +}; +} //namespace CoreBind +VARIANT_ENUM_CAST(CoreBind::ResourceCompatLoader::CacheMode); diff --git a/register_types.cpp b/register_types.cpp index c30cf379a..ab38c41cc 100644 --- a/register_types.cpp +++ b/register_types.cpp @@ -443,7 +443,7 @@ void initialize_gdsdecomp_module(ModuleInitializationLevel p_level) { ClassDB::register_class(); ClassDB::register_class(); ClassDB::register_class(); - ClassDB::register_class(); + ClassDB::register_class(); ClassDB::register_class(); ClassDB::register_class(); ClassDB::register_class(); diff --git a/standalone/gdre_media_player.gd b/standalone/gdre_media_player.gd index c0cf66ca0..6b0268fa3 100644 --- a/standalone/gdre_media_player.gd +++ b/standalone/gdre_media_player.gd @@ -248,7 +248,7 @@ func load_media(path): func load_video(path): if not is_supported_video_format(path): return false - var video_stream: VideoStream = ResourceCompatLoader.real_load(path, "", ResourceFormatLoader.CACHE_MODE_IGNORE_DEEP) + var video_stream: VideoStream = ResourceCompatLoader.real_load(path, "", ResourceCompatLoader.CACHE_MODE_IGNORE_DEEP) if (video_stream == null): return false reset() @@ -269,7 +269,7 @@ func load_sample(path): var audio_stream: AudioStream = null var ext = path.get_extension().to_lower() if not is_non_resource_smp(ext): - audio_stream = ResourceCompatLoader.real_load(path, "", ResourceFormatLoader.CACHE_MODE_IGNORE_DEEP) + audio_stream = ResourceCompatLoader.real_load(path, "", ResourceCompatLoader.CACHE_MODE_IGNORE_DEEP) else: if ext == "wav": audio_stream = AudioStreamWAV.load_from_file(path) diff --git a/standalone/gdre_resource_preview.gd b/standalone/gdre_resource_preview.gd index 64a77e1ee..f72fefcda 100644 --- a/standalone/gdre_resource_preview.gd +++ b/standalone/gdre_resource_preview.gd @@ -89,11 +89,11 @@ var previous_res_info_size = Vector2(0, 0) func load_texture(path): var ext = path.get_extension().to_lower() if (ext == "image"): - %TextureRect.texture = ImageTexture.create_from_image(ResourceCompatLoader.real_load(path, "", ResourceFormatLoader.CACHE_MODE_IGNORE_DEEP)) + %TextureRect.texture = ImageTexture.create_from_image(ResourceCompatLoader.real_load(path, "", ResourceCompatLoader.CACHE_MODE_IGNORE_DEEP)) elif (is_image(ext)): %TextureRect.texture = ImageTexture.create_from_image(Image.load_from_file(path)) else: - %TextureRect.texture = ResourceCompatLoader.real_load(path, "", ResourceFormatLoader.CACHE_MODE_IGNORE_DEEP) # TODO: handle other texture types + %TextureRect.texture = ResourceCompatLoader.real_load(path, "", ResourceCompatLoader.CACHE_MODE_IGNORE_DEEP) # TODO: handle other texture types if (%TextureRect.texture == null): return false %TextureRect.expand_mode = TextureRect.EXPAND_IGNORE_SIZE @@ -153,7 +153,7 @@ func is_scene(ext, type: String): return ext == "tscn" || ext == "scn" || type == "PackedScene" func load_mesh(path): - var res = ResourceCompatLoader.real_load(path, "", ResourceFormatLoader.CACHE_MODE_IGNORE_DEEP) + var res = ResourceCompatLoader.real_load(path, "", ResourceCompatLoader.CACHE_MODE_IGNORE_DEEP) %SwitchViewButton.text = SWITCH_TO_TEXT_TEXT %SwitchViewButton.visible = true if not res: @@ -175,7 +175,7 @@ func load_scene(path): break var start_time = Time.get_ticks_msec() if not res: - res = ResourceCompatLoader.real_load(path, "", ResourceFormatLoader.CACHE_MODE_REUSE) + res = ResourceCompatLoader.real_load(path, "", ResourceCompatLoader.CACHE_MODE_REUSE) %SwitchViewButton.text = SWITCH_TO_TEXT_TEXT %SwitchViewButton.visible = true if not res: From 87b4a9a814ca455f7ddfab1141d0df73477933c9 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Fri, 17 Apr 2026 06:37:18 -0700 Subject: [PATCH 008/134] Fix loading empty scripts --- compat/fake_gdscript.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/compat/fake_gdscript.cpp b/compat/fake_gdscript.cpp index 8ac437c98..296de4317 100644 --- a/compat/fake_gdscript.cpp +++ b/compat/fake_gdscript.cpp @@ -51,8 +51,11 @@ Error FakeGDScript::_reload_from_file() { FAKEGDSCRIPT_FAIL_COND_V_MSG(err != OK, err, "Error reading file: " + script_path); is_binary = binary_buffer.size() >= 4 && binary_buffer[0] == 'G' && binary_buffer[1] == 'D' && binary_buffer[2] == 'S' && binary_buffer[3] == 'C'; if (!is_binary) { - err = source.append_utf8(reinterpret_cast(binary_buffer.ptr()), binary_buffer.size()); - FAKEGDSCRIPT_FAIL_COND_V_MSG(err != OK, err, "Error reading file: " + script_path); + // Empty text files are valid gdscripts (treated as `extends RefCounted`). + if (binary_buffer.size() > 0) { + err = source.append_utf8(reinterpret_cast(binary_buffer.ptr()), binary_buffer.size()); + FAKEGDSCRIPT_FAIL_COND_V_MSG(err != OK, err, "Error reading file: " + script_path); + } binary_buffer.clear(); } } From ca8958c051bb6fc4e1f3837887e6306ca4047d4a Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Fri, 17 Apr 2026 06:37:18 -0700 Subject: [PATCH 009/134] fix progress bar not redrawing --- gui/gdre_progress.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gui/gdre_progress.cpp b/gui/gdre_progress.cpp index bd4e196f7..2ab433163 100644 --- a/gui/gdre_progress.cpp +++ b/gui/gdre_progress.cpp @@ -228,7 +228,7 @@ bool GDREProgressDialog::Task::update() { constexpr uint64_t REDRAW_THRESHOLD = 1000000 / 60; auto curr_time_us = OS::get_singleton()->get_ticks_usec(); if (!should_redraw(curr_time_us)) { - if (!is_headless && !(curr_time_us - last_progress_tick >= REDRAW_THRESHOLD && (progress->get_value() != current_step.step || state->get_text() != current_step.state || indeterminate != progress->is_indeterminate() || steps != progress->get_max()))) { + if (!is_headless && !(curr_time_us - last_progress_tick >= REDRAW_THRESHOLD && (indeterminate || (progress->get_value() != current_step.step || state->get_text() != current_step.state || indeterminate != progress->is_indeterminate() || steps != progress->get_max())))) { return false; } } @@ -245,6 +245,9 @@ bool GDREProgressDialog::Task::update() { if (progress->get_value() != current_step.step) { progress->set_value(current_step.step); } + } else { + // this forces the indeterminate progress bar to redraw + progress->notification(NOTIFICATION_INTERNAL_PROCESS); } if (state->get_text() != current_step.state) { state->set_text(current_step.state); From f6c27488b3121347257d1ae185651abbac231a76 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Fri, 17 Apr 2026 06:37:18 -0700 Subject: [PATCH 010/134] update dispatch_to_main_thread to take in variadic args --- utility/task_manager.h | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/utility/task_manager.h b/utility/task_manager.h index 6265ba412..6e59254a1 100644 --- a/utility/task_manager.h +++ b/utility/task_manager.h @@ -8,7 +8,11 @@ #include "utility/gd_parallel_queue.h" #include "utility/gdre_config.h" +#include #include +#include +#include +#include struct TaskRunnerStruct { virtual bool pre_run() { return true; } @@ -82,18 +86,18 @@ class TaskManager : public Object { }; // This is a helper class to dispatch a callback to the main thread. - template + template class FunctionMainThreadDispatchData : public BaseMainThreadDispatchData { - std::function function; - U userdata; + std::function function; + std::tuple...> userdata; public: - FunctionMainThreadDispatchData(TaskManager::TaskManagerID p_calling_task_id, std::function p_function, U p_userdata) : + FunctionMainThreadDispatchData(TaskManager::TaskManagerID p_calling_task_id, std::function p_function, std::decay_t... p_userdata) : BaseMainThreadDispatchData(p_calling_task_id), - function(p_function), - userdata(p_userdata) {} + function(std::move(p_function)), + userdata(std::move(p_userdata)...) {} void callback_internal() override { - function(userdata); + std::apply(function, userdata); } }; @@ -601,9 +605,9 @@ class TaskManager : public Object { return _dispatch_to_main_thread(data); } - template - bool dispatch_to_main_thread(std::function func, U userdata) { - std::shared_ptr data = std::make_shared>(get_thread_task_id(), func, userdata); + template + bool dispatch_to_main_thread(std::function func, std::decay_t... userdata) { + std::shared_ptr data = std::make_shared>(get_thread_task_id(), std::move(func), std::move(userdata)...); return _dispatch_to_main_thread(data); } From b7a0f846d5491fb519d89278a17bbc1c9ff35907 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Fri, 17 Apr 2026 06:37:18 -0700 Subject: [PATCH 011/134] allow return values in main thread dispatches --- utility/task_manager.h | 90 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 78 insertions(+), 12 deletions(-) diff --git a/utility/task_manager.h b/utility/task_manager.h index 6e59254a1..dda994c41 100644 --- a/utility/task_manager.h +++ b/utility/task_manager.h @@ -10,6 +10,7 @@ #include #include +#include #include #include #include @@ -67,12 +68,45 @@ class TaskManager : public Object { } }; + template > + class MainThreadDispatchResultStorage; + + template + class MainThreadDispatchResultStorage { + std::optional> result; + + public: + template + void execute(F &&p_callback) { + result = std::forward(p_callback)(); + } + + std::optional> take_result() { + return std::move(result); + } + }; + + template + class MainThreadDispatchResultStorage { + public: + template + void execute(F &&p_callback) { + std::forward(p_callback)(); + } + }; + + template + using MainThreadDispatchReturn = std::optional, bool, std::decay_t>>; + // This is a helper class to dispatch a callback to the main thread. template class TemplateMainThreadDispatchData : public BaseMainThreadDispatchData { + using ReturnType = std::invoke_result_t; + C *instance; M method; U userdata; + MainThreadDispatchResultStorage result_storage; public: TemplateMainThreadDispatchData(TaskManager::TaskManagerID p_calling_task_id, C *p_instance, M p_method, U p_userdata) : @@ -81,23 +115,38 @@ class TaskManager : public Object { method(p_method), userdata(p_userdata) {} void callback_internal() override { - (instance->*method)(userdata); + result_storage.execute([this]() -> ReturnType { + return std::invoke(method, instance, userdata); + }); + } + + template >> + std::optional> take_result() { + return result_storage.take_result(); } }; // This is a helper class to dispatch a callback to the main thread. - template + template class FunctionMainThreadDispatchData : public BaseMainThreadDispatchData { - std::function function; + std::function function; std::tuple...> userdata; + MainThreadDispatchResultStorage result_storage; public: - FunctionMainThreadDispatchData(TaskManager::TaskManagerID p_calling_task_id, std::function p_function, std::decay_t... p_userdata) : + FunctionMainThreadDispatchData(TaskManager::TaskManagerID p_calling_task_id, std::function p_function, std::decay_t... p_userdata) : BaseMainThreadDispatchData(p_calling_task_id), function(std::move(p_function)), userdata(std::move(p_userdata)...) {} void callback_internal() override { - std::apply(function, userdata); + result_storage.execute([this]() -> R { + return std::apply(function, userdata); + }); + } + + template >> + std::optional> take_result() { + return result_storage.take_result(); } }; @@ -600,15 +649,32 @@ class TaskManager : public Object { } template - bool dispatch_to_main_thread(C *p_instance, M p_method, U p_userdata) { - std::shared_ptr data = std::make_shared>(get_thread_task_id(), p_instance, p_method, p_userdata); - return _dispatch_to_main_thread(data); + MainThreadDispatchReturn> dispatch_to_main_thread(C *p_instance, M p_method, U p_userdata) { + using ReturnType = std::invoke_result_t; + auto data = std::make_shared>(get_thread_task_id(), p_instance, p_method, p_userdata); + bool canceled = _dispatch_to_main_thread(data); + if (canceled) { + return std::nullopt; + } + if constexpr (std::is_void_v) { + return true; + } else { + return data->take_result(); + } } - template - bool dispatch_to_main_thread(std::function func, std::decay_t... userdata) { - std::shared_ptr data = std::make_shared>(get_thread_task_id(), std::move(func), std::move(userdata)...); - return _dispatch_to_main_thread(data); + template + MainThreadDispatchReturn dispatch_to_main_thread(std::function func, std::decay_t... userdata) { + auto data = std::make_shared>(get_thread_task_id(), std::move(func), std::move(userdata)...); + bool canceled = _dispatch_to_main_thread(data); + if (canceled) { + return std::nullopt; + } + if constexpr (std::is_void_v) { + return true; + } else { + return data->take_result(); + } } DownloadTaskID add_download_task(const String &p_download_url, const String &p_save_path, bool silent = false); From 4c8949b4619e753a04f81cc096ce60a643693ac1 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Fri, 17 Apr 2026 06:37:18 -0700 Subject: [PATCH 012/134] fix errors when converting packedscene to text --- compat/resource_compat_text.cpp | 44 +++++++++++++++++++++++---------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/compat/resource_compat_text.cpp b/compat/resource_compat_text.cpp index cfe2b4a2a..5886ac1fc 100644 --- a/compat/resource_compat_text.cpp +++ b/compat/resource_compat_text.cpp @@ -2159,6 +2159,8 @@ Error ResourceFormatSaverCompatTextInstance::save_to_file(const Ref } #endif + ERR_FAIL_COND_V_MSG(p_resource.is_null(), ERR_INVALID_PARAMETER, "Resource is null"); + if (p_path.ends_with(".tscn") || p_path.ends_with(".escn")) { // If this is a MissingResource holder for a PackedScene, we need to instance it for reals packed_scene = ensure_packed_scenes(p_resource); @@ -3120,9 +3122,9 @@ bool is_packed_scene(const Ref &p_resource) { return p_resource.is_valid() && _resource_get_class(p_resource) == "PackedScene"; } -Ref _ensure_resource_is_packed_scene(const Ref &p_resource, HashMap> &p_seen_resources, HashSet &seen_objects, int recursion_depth); +Ref _ensure_resource_is_packed_scene(const Ref &p_resource, HashMap> &p_seen_resources, HashSet &seen_objects, int recursion_depth, bool *changed_something); -Variant scan_variant(const Variant &v, HashMap> &p_seen_resources, HashSet &seen_objects, int recursion_depth) { +Variant scan_variant(const Variant &v, HashMap> &p_seen_resources, HashSet &seen_objects, int recursion_depth, bool *p_changed_something) { ERR_FAIL_COND_V_MSG(recursion_depth > 1024, Variant(), "Recursion depth exceeded"); recursion_depth++; Variant result = v; @@ -3133,7 +3135,10 @@ Variant scan_variant(const Variant &v, HashMap> &p_seen_re Ref sub_scene; if (!p_seen_resources.has(res->get_path())) { if (_resource_get_class(res) == "PackedScene") { - sub_scene = _ensure_resource_is_packed_scene(res, p_seen_resources, seen_objects, recursion_depth); + sub_scene = _ensure_resource_is_packed_scene(res, p_seen_resources, seen_objects, recursion_depth, p_changed_something); + if (res->get_class() == "MissingResource") { + *p_changed_something = true; + } } } else { sub_scene = p_seen_resources.get(res->get_path()); @@ -3150,8 +3155,11 @@ Variant scan_variant(const Variant &v, HashMap> &p_seen_re List property_list; res->get_property_list(&property_list); for (const PropertyInfo &pi : property_list) { - Variant v = scan_variant(res->get(pi.name), p_seen_resources, seen_objects, recursion_depth); - res->set(pi.name, v); + bool changed_something = false; + Variant v = scan_variant(res->get(pi.name), p_seen_resources, seen_objects, recursion_depth, &changed_something); + if (changed_something) { + res->set(pi.name, v); + } } p_seen_resources[res->get_path()] = res; } @@ -3161,8 +3169,11 @@ Variant scan_variant(const Variant &v, HashMap> &p_seen_re obj->get_property_list(&property_list); for (const PropertyInfo &pi : property_list) { if (pi.usage & PROPERTY_USAGE_STORAGE) { - Variant v = scan_variant(obj->get(pi.name), p_seen_resources, seen_objects, recursion_depth); - obj->set(pi.name, v); + bool changed_something = false; + Variant v = scan_variant(obj->get(pi.name), p_seen_resources, seen_objects, recursion_depth, &changed_something); + if (changed_something) { + obj->set(pi.name, v); + } } } } @@ -3170,15 +3181,15 @@ Variant scan_variant(const Variant &v, HashMap> &p_seen_re case Variant::ARRAY: { Array a = v; for (int j = 0; j < a.size(); j++) { - a[j] = scan_variant(a[j], p_seen_resources, seen_objects, recursion_depth); + a[j] = scan_variant(a[j], p_seen_resources, seen_objects, recursion_depth, p_changed_something); } result = a; } break; case Variant::DICTIONARY: { Dictionary d; for (const KeyValue &kv : v.operator Dictionary()) { - Variant key = scan_variant(kv.key, p_seen_resources, seen_objects, recursion_depth); - Variant value = scan_variant(kv.value, p_seen_resources, seen_objects, recursion_depth); + Variant key = scan_variant(kv.key, p_seen_resources, seen_objects, recursion_depth, p_changed_something); + Variant value = scan_variant(kv.value, p_seen_resources, seen_objects, recursion_depth, p_changed_something); d[key] = value; } v.operator Dictionary().clear(); @@ -3191,10 +3202,10 @@ Variant scan_variant(const Variant &v, HashMap> &p_seen_re return result; } -Ref _ensure_resource_is_packed_scene(const Ref &p_resource, HashMap> &p_seen_resources, HashSet &seen_objects, int recursion_depth) { +Ref _ensure_resource_is_packed_scene(const Ref &p_resource, HashMap> &p_seen_resources, HashSet &seen_objects, int recursion_depth, bool *changed_something) { Ref r_packed_scene = p_resource; if (_resource_get_class(p_resource) == "PackedScene") { - Dictionary bundle = scan_variant(p_resource->get("_bundled"), p_seen_resources, seen_objects, recursion_depth); + Dictionary bundle = scan_variant(p_resource->get("_bundled"), p_seen_resources, seen_objects, recursion_depth, changed_something); // we need to go through the variants in the bundle and ensure that any MissingResources that are PackedScenes are also replaced with instantiated packed scenes // this is because of PackedScene::SceneState::get_node_instance, which returns a Ref, which will be null if it's not an instance of a PackedScene if (r_packed_scene.is_null()) { // MissingResource @@ -3210,7 +3221,14 @@ Ref _ensure_resource_is_packed_scene(const Ref &p_resourc } Ref ResourceFormatSaverCompatTextInstance::ensure_packed_scenes(const Ref &p_resource) { + auto info = ResourceInfo::get_info_from_resource(p_resource); + if (info.is_valid()) { + if (info->load_type == ResourceInfo::REAL_LOAD && info->load_type == ResourceInfo::GLTF_LOAD && p_resource->get_class() == "PackedScene") { + return p_resource; + } + } HashMap> seen_resources; HashSet seen_objects; - return _ensure_resource_is_packed_scene(p_resource, seen_resources, seen_objects, 0); + bool changed_something = false; + return _ensure_resource_is_packed_scene(p_resource, seen_resources, seen_objects, 0, &changed_something); } From 81939c04b02a8581cf685b19c4f9b523d216da39 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Fri, 17 Apr 2026 06:37:18 -0700 Subject: [PATCH 013/134] scene: remove unused mesh counting --- exporters/scene_exporter.cpp | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/exporters/scene_exporter.cpp b/exporters/scene_exporter.cpp index 4a0809ee0..6cd72bddc 100644 --- a/exporters/scene_exporter.cpp +++ b/exporters/scene_exporter.cpp @@ -1642,7 +1642,7 @@ Node *GLBExporterInstance::_set_stuff_from_instanced_scene(Node *root) { } } Node *original_node = nullptr; - auto replace_with_mi = [&](Ref mesh) { + std::function)> replace_with_mi = [&](Ref mesh) { auto mesh_instance = generate_mesh_instance(node, mesh, node); mesh_instance->set_name(node->get_name()); process_mesh_instance(mesh_instance); @@ -3581,15 +3581,6 @@ struct BatchExportToken : public TaskRunnerStruct { } } - constexpr uint64_t MAX_MESHES_ON_WORKER_THREAD = 15; - if (instance.is_batch_export) { - auto meshes = get_meshes(_scene, ver_major, true); - if (meshes.size() > MAX_MESHES_ON_WORKER_THREAD) { - // disabling for now; can't figure out what the x factor is for why certain scenes take forever - // do_on_main_thread = true; - } - surface_count = get_total_surface_count(meshes); - } // print_line("Preloaded scene " + p_src_path); after_preload(); return do_on_main_thread; From 107373491ef50a41e1061ee38e0d519e2a68c19c Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Fri, 17 Apr 2026 06:37:18 -0700 Subject: [PATCH 014/134] scene: split off recompute_animation_tracks_for_library --- exporters/scene_exporter.cpp | 121 ++++++++++++++++++++++------------- exporters/scene_exporter.h | 2 + 2 files changed, 78 insertions(+), 45 deletions(-) diff --git a/exporters/scene_exporter.cpp b/exporters/scene_exporter.cpp index 6cd72bddc..d7ee682ed 100644 --- a/exporters/scene_exporter.cpp +++ b/exporters/scene_exporter.cpp @@ -1470,6 +1470,68 @@ Ref get_nav_array_debug_mesh(const Ref navmesh) { return debug_mesh; } +void GLBExporterInstance::recompute_animation_tracks_for_library(AnimationPlayer *p_player, const Ref &p_anim_lib, const LocalVector &p_anim_names) { + if (ver_major > 3 || p_anim_names.is_empty()) { + return; + } + auto current_animation = p_player->get_current_animation(); + auto current_pos = current_animation.is_empty() ? 0 : p_player->get_current_animation_position(); + + Ref mesh_surface_re = RegEx::create_from_string(":mesh:surface_(\\d+)"); + // Force re-compute animation tracks. + for (auto &anim_name : p_anim_names) { + Ref anim = p_anim_lib->get_animation(anim_name); + auto info = ResourceInfo::get_info_from_resource(anim); + ERR_CONTINUE(!info.is_valid()); + constexpr const char *converted_paths_from_3_x = "converted_paths_from_3.x"; + if (info->extra.get(converted_paths_from_3_x, false)) { + continue; + } + + size_t num_tracks = anim->get_track_count(); + for (size_t i = 0; i < num_tracks; i++) { + String str_path = String(anim->track_get_path(i)); + if (str_path.contains(":mesh:surface_")) { + // Surface properties are 1-indexed in 3.x, but 0-indexed in 4.x. + Ref match = mesh_surface_re->search(str_path); + if (match.is_valid()) { + int surface_index = match->get_string(1).to_int(); + surface_index--; + str_path = mesh_surface_re->sub(str_path, ":mesh:surface_" + String::num_int64(surface_index)); + } + } + if (str_path.contains(":material/")) { + str_path = str_path.replace(":material/", ":surface_material_override/"); + } + if (str_path.contains(":shader_param/")) { + str_path = str_path.replace(":shader_param/", ":shader_parameter/"); + } + anim->track_set_path(i, str_path); + } + info->extra.set(converted_paths_from_3_x, true); + } + + p_player->set_current_animation(*p_anim_names.begin()); + p_player->advance(0); + p_player->set_current_animation(current_animation); + if (!current_animation.is_empty()) { + p_player->seek(current_pos); + } +} + +void GLBExporterInstance::convert_animation_tracks_to_v4_for_player(AnimationPlayer *p_player) { + LocalVector anim_lib_names; + p_player->get_animation_library_list(&anim_lib_names); + for (auto &anim_lib_name : anim_lib_names) { + Ref anim_lib = p_player->get_animation_library(anim_lib_name); + if (anim_lib.is_valid()) { + LocalVector anim_names; + anim_lib->get_animation_list(&anim_names); + recompute_animation_tracks_for_library(p_player, anim_lib, anim_names); + } + } +} + Node *GLBExporterInstance::_set_stuff_from_instanced_scene(Node *root) { root_type = root->get_class(); root_name = root->get_name(); @@ -1756,11 +1818,24 @@ Node *GLBExporterInstance::_set_stuff_from_instanced_scene(Node *root) { } Vector fps_values; + + if (ver_major <= 3) { + std::function convert_all_players_to_v4 = [&]() { + for (int32_t node_i = 0; node_i < animation_player_nodes.size(); node_i++) { + AnimationPlayer *player = Object::cast_to(animation_player_nodes[node_i]); + ERR_CONTINUE(!player); + convert_animation_tracks_to_v4_for_player(player); + } + }; + convert_all_players_to_v4(); + } + for (int32_t node_i = 0; node_i < animation_player_nodes.size(); node_i++) { + AnimationPlayer *player = Object::cast_to(animation_player_nodes[node_i]); + ERR_CONTINUE(!player); bool any_compressed = false; // Force re-compute animation tracks. Vector> anim_libs; - AnimationPlayer *player = Object::cast_to(animation_player_nodes[node_i]); LocalVector anim_lib_names; player->get_animation_library_list(&anim_lib_names); for (auto &lib_name : anim_lib_names) { @@ -1769,54 +1844,10 @@ Node *GLBExporterInstance::_set_stuff_from_instanced_scene(Node *root) { anim_libs.push_back(lib); } } - ERR_CONTINUE(!player); - auto current_anmation = player->get_current_animation(); - auto current_pos = current_anmation.is_empty() ? 0 : player->get_current_animation_position(); int64_t max_fps = -1; for (auto &anim_lib : anim_libs) { LocalVector anim_names; anim_lib->get_animation_list(&anim_names); - if (ver_major <= 3 && anim_names.size() > 0) { - Ref mesh_surface_re = RegEx::create_from_string(":mesh:surface_(\\d+)"); - // force re-compute animation tracks. - for (auto &anim_name : anim_names) { - Ref anim = anim_lib->get_animation(anim_name); - auto info = ResourceInfo::get_info_from_resource(anim); - ERR_CONTINUE(!info.is_valid()); - constexpr const char *converted_paths_from_3_x = "converted_paths_from_3.x"; - if (info->extra.get(converted_paths_from_3_x, false)) { - continue; - } - size_t num_tracks = anim->get_track_count(); - for (size_t i = 0; i < num_tracks; i++) { - String str_path = String(anim->track_get_path(i)); - if (str_path.contains(":mesh:surface_")) { - // replace the number after surface_ with one lower (surface properties are 1-indexed in 3.x, but 0-indexed in 4.0) - Ref match = mesh_surface_re->search(str_path); - if (match.is_valid()) { - int surface_index = match->get_string(1).to_int(); - surface_index--; - str_path = mesh_surface_re->sub(str_path, ":mesh:surface_" + String::num_int64(surface_index)); - } - } - if (str_path.contains(":material/")) { - str_path = str_path.replace(":material/", ":surface_material_override/"); - } - if (str_path.contains(":shader_param/")) { - str_path = str_path.replace(":shader_param/", ":shader_parameter/"); - } - anim->track_set_path(i, str_path); - } - info->extra.set(converted_paths_from_3_x, true); - } - - player->set_current_animation(*anim_names.begin()); - player->advance(0); - player->set_current_animation(current_anmation); - if (!current_anmation.is_empty()) { - player->seek(current_pos); - } - } for (auto &anim_name : anim_names) { double shortest_frame_duration = 1000.0; diff --git a/exporters/scene_exporter.h b/exporters/scene_exporter.h index 8e6fc08a6..404032da9 100644 --- a/exporters/scene_exporter.h +++ b/exporters/scene_exporter.h @@ -182,6 +182,8 @@ class GLBExporterInstance { Error _load_deps(); Error _load_scene_and_deps(Ref &r_scene); Error _load_scene(Ref &r_scene); + void recompute_animation_tracks_for_library(AnimationPlayer *p_player, const Ref &p_anim_lib, const LocalVector &p_anim_names); + void convert_animation_tracks_to_v4_for_player(AnimationPlayer *p_player); void _unload_deps(); Error _get_return_error(); From aaaf995e263f8d8d615040b94a981714a405d73f Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Fri, 17 Apr 2026 06:37:18 -0700 Subject: [PATCH 015/134] scene: minor fix --- exporters/scene_exporter.cpp | 75 ++++++++++++++++++++---------------- 1 file changed, 42 insertions(+), 33 deletions(-) diff --git a/exporters/scene_exporter.cpp b/exporters/scene_exporter.cpp index d7ee682ed..49265fed7 100644 --- a/exporters/scene_exporter.cpp +++ b/exporters/scene_exporter.cpp @@ -3,6 +3,7 @@ #include "compat/resource_loader_compat.h" #include "core/io/file_access.h" #include "core/io/resource_loader.h" +#include "core/os/thread_safe.h" #include "core/variant/variant.h" #include "core/version_generated.gen.h" #include "exporters/export_report.h" @@ -1533,6 +1534,8 @@ void GLBExporterInstance::convert_animation_tracks_to_v4_for_player(AnimationPla } Node *GLBExporterInstance::_set_stuff_from_instanced_scene(Node *root) { + bool current_thread_safe_for_nodes = is_current_thread_safe_for_nodes(); + set_current_thread_safe_for_nodes(true); root_type = root->get_class(); root_name = root->get_name(); @@ -1660,47 +1663,52 @@ Node *GLBExporterInstance::_set_stuff_from_instanced_scene(Node *root) { // create a new mesh instance auto mesh_instance = generate_mesh_instance(node, mesh); mis_generated_from_scripts.insert(mesh_instance); - node->add_child(mesh_instance); } } } - for (auto &mesh_instance : mis_generated_from_scripts) { - bool set_skin = false; - bool set_skeleton = false; - for (auto &prop : properties) { - Variant value; - if (si->get(prop.name, value) && !skins.has(value) && !skeleton_paths.has(value)) { - Ref skin = value; - if (skin.is_valid() && !set_skin) { - mesh_instance->set_skin(skin); - set_skin = true; - skins.insert(skin); - } else if (!set_skeleton) { - NodePath skeleton_path = value; - if (!skeleton_path.is_empty()) { - // we need to check to see if this is actually a skeleton - Node *skeleton = node->get_node(skeleton_path); - Skeleton3D *skeleton3d = skeleton ? Object::cast_to(skeleton) : nullptr; - if (skeleton3d) { - NodePath actual_path = skeleton_path; - if (!skeleton_path.is_absolute()) { - actual_path = mesh_instance->get_path_to(skeleton3d); + if (!mis_generated_from_scripts.is_empty()) { + std::function add_mesh_instances = [&]() { + for (auto &mesh_instance : mis_generated_from_scripts) { + node->add_child(mesh_instance); + bool set_skin = false; + bool set_skeleton = false; + for (auto &prop : properties) { + Variant value; + if (si->get(prop.name, value) && !skins.has(value) && !skeleton_paths.has(value)) { + Ref skin = value; + if (skin.is_valid() && !set_skin) { + mesh_instance->set_skin(skin); + set_skin = true; + skins.insert(skin); + } else if (!set_skeleton) { + NodePath skeleton_path = value; + if (!skeleton_path.is_empty()) { + // we need to check to see if this is actually a skeleton + Node *skeleton = node->get_node(skeleton_path); + Skeleton3D *skeleton3d = skeleton ? Object::cast_to(skeleton) : nullptr; + if (skeleton3d) { + NodePath actual_path = skeleton_path; + if (!skeleton_path.is_absolute()) { + actual_path = mesh_instance->get_path_to(skeleton3d); + } + mesh_instance->set_skeleton_path(actual_path); + set_skeleton = true; + skeleton_paths.insert(skeleton_path); + } } - mesh_instance->set_skeleton_path(actual_path); - set_skeleton = true; - skeleton_paths.insert(skeleton_path); } } + if (set_skin && set_skeleton) { + break; + } + } + process_mesh_instance(mesh_instance); + if (updating_import_info) { + node_options[get_node_path(mesh_instance).operator String()] = { { "import/skip_import", true } }; } } - if (set_skin && set_skeleton) { - break; - } - } - process_mesh_instance(mesh_instance); - if (updating_import_info) { - node_options[get_node_path(mesh_instance).operator String()] = { { "import/skip_import", true } }; - } + }; + add_mesh_instances(); } } Node *original_node = nullptr; @@ -1929,6 +1937,7 @@ Node *GLBExporterInstance::_set_stuff_from_instanced_scene(Node *root) { baked_fps = MIN(max_fps, 120); } } + set_current_thread_safe_for_nodes(current_thread_safe_for_nodes); return root; } From 77dd143fc37c825daa9fd7682a808057ea47e5b8 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Fri, 17 Apr 2026 06:37:18 -0700 Subject: [PATCH 016/134] scene: ensure functions run on main thread --- exporters/scene_exporter.cpp | 92 ++++++++++++++++++++++++++---------- 1 file changed, 67 insertions(+), 25 deletions(-) diff --git a/exporters/scene_exporter.cpp b/exporters/scene_exporter.cpp index 49265fed7..ec4fe0c05 100644 --- a/exporters/scene_exporter.cpp +++ b/exporters/scene_exporter.cpp @@ -996,12 +996,18 @@ Error GLBExporterInstance::_load_deps() { if (our_path != info.remap) { WARN_PRINT(vformat("Dependency %s:%s is not mapped to the same path: %s", info.dep, info.remap, our_path)); if (!ResourceCache::has(info.dep)) { - auto texture = ResourceCompatLoader::custom_load( - info.remap, "", - ResourceCompatLoader::get_default_load_type(), - &err, - using_threaded_load(), - ResourceFormatLoader::CACHE_MODE_IGNORE); // not ignore deep, we want to reuse dependencies if they exist + auto result = TaskManager::get_singleton()->dispatch_to_main_thread((std::function()>)[&]() -> Ref { + return ResourceCompatLoader::custom_load( + info.remap, "", + ResourceCompatLoader::get_default_load_type(), + &err, + using_threaded_load(), + ResourceFormatLoader::CACHE_MODE_IGNORE); // not ignore deep, we want to reuse dependencies if they exist + }); + if (!result.has_value()) { + return ERR_SKIP; // cancelled + } + auto texture = result.value(); if (err || texture.is_null()) { if (ignore_missing_dependencies) { missing_dependencies.push_back(info.dep); @@ -1644,7 +1650,7 @@ Node *GLBExporterInstance::_set_stuff_from_instanced_scene(Node *root) { meshes_in_mesh_instances.insert(mesh); return mesh_instance; }; - std::function process_node = [&](Node *node) { + std::function process_node = [&](Node *node) -> bool { ScriptInstance *si = node->get_script_instance(); List properties; HashSet mis_generated_from_scripts; @@ -1708,7 +1714,9 @@ Node *GLBExporterInstance::_set_stuff_from_instanced_scene(Node *root) { } } }; - add_mesh_instances(); + if (TaskManager::get_singleton()->dispatch_to_main_thread(add_mesh_instances).has_value()) { + return false; // cancelled + } } } Node *original_node = nullptr; @@ -1754,18 +1762,17 @@ Node *GLBExporterInstance::_set_stuff_from_instanced_scene(Node *root) { // replace NavMesh/Occluder/Area3D-only nodes with mesh instances // e.g. the original mesh will have been replaced by a NavMesh/Occluder/Area3D node by the importer, so we have to put it back. if (!is_auto_generated_node(node) && node != root) { + Ref replacement_mesh; if (auto navigation_region = Object::cast_to(node); navigation_region) { if (Ref navmesh = navigation_region->get_navigation_mesh(); navmesh.is_valid()) { - auto debug_mesh = get_nav_array_debug_mesh(navmesh); - replace_with_mi(debug_mesh); + replacement_mesh = get_nav_array_debug_mesh(navmesh); } } else if (auto occluder_instance = Object::cast_to(node); occluder_instance) { if (Ref occluder = occluder_instance->get_occluder(); occluder.is_valid()) { - replace_with_mi(occluder->get_debug_mesh()); + replacement_mesh = occluder->get_debug_mesh(); } } else if (auto area_3d = Object::cast_to(node); area_3d) { Vector> meshes; - Ref mesh; for (auto &E : area_3d->get_children()) { if (auto collision_shape = Object::cast_to(E.operator Object *()); collision_shape && collision_shape->get_shape().is_valid()) { meshes.push_back(collision_shape->get_shape()->get_debug_mesh()); @@ -1782,12 +1789,14 @@ Node *GLBExporterInstance::_set_stuff_from_instanced_scene(Node *root) { } } } - mesh = surface_tool.commit(); + replacement_mesh = surface_tool.commit(); } else if (meshes.size() == 1) { - mesh = meshes[0]; + replacement_mesh = meshes[0]; } - if (mesh.is_valid()) { - replace_with_mi(mesh); + } + if (replacement_mesh.is_valid()) { + if (!TaskManager::get_singleton()->dispatch_to_main_thread(replace_with_mi, replacement_mesh).has_value()) { + return false; // cancelled } } } @@ -1806,11 +1815,16 @@ Node *GLBExporterInstance::_set_stuff_from_instanced_scene(Node *root) { for (auto &E : node->get_children()) { auto child = Object::cast_to(E.operator Object *()); if (!mis_generated_from_scripts.has(Object::cast_to(child))) { - process_node(child); + if (!process_node(child)) { + return false; // cancelled + } } } + return true; }; - process_node(root); + if (!process_node(root)) { + return nullptr; // cancelled + } if (replaced_node_names.size() > 0) { auto thingy = String(", ").join(replaced_node_names); print_line(vformat("%s: replaced nodes with mesh instances: %s", scene_name, thingy)); @@ -3172,7 +3186,13 @@ Node *GLBExporterInstance::_instantiate_scene(Ref scene) { _silence_errors(true); } #endif - Node *root = scene->instantiate(); + auto result = TaskManager::get_singleton()->dispatch_to_main_thread((std::function)[&scene]() -> Node * { + return scene->instantiate(); + }); + if (!result.has_value()) { + return nullptr; // cancelled + } + Node *root = result.value(); #ifndef DEBUG_ENABLED if (ver_major <= 3) { _silence_errors(false); @@ -3205,12 +3225,21 @@ Error GLBExporterInstance::_load_scene(Ref &r_scene) { _silence_errors(true); } #endif + std::optional> result; // For some reason, scenes with meshes fail to load without the load done by ResourceLoader::load, possibly due to notification shenanigans. if (ResourceCompatLoader::is_globally_available() && using_threaded_load()) { - r_scene = ResourceLoader::load(source_path, "PackedScene", ResourceFormatLoader::CACHE_MODE_REUSE, &err); + result = TaskManager::get_singleton()->dispatch_to_main_thread((std::function()>)[&]() -> Ref { + return ResourceLoader::load(source_path, "PackedScene", ResourceFormatLoader::CACHE_MODE_REUSE, &err); + }); } else { - r_scene = ResourceCompatLoader::custom_load(source_path, "PackedScene", mode_type, &err, using_threaded_load(), ResourceFormatLoader::CACHE_MODE_REUSE); + result = TaskManager::get_singleton()->dispatch_to_main_thread((std::function()>)[&]() -> Ref { + return ResourceCompatLoader::custom_load(source_path, "PackedScene", mode_type, &err, using_threaded_load(), ResourceFormatLoader::CACHE_MODE_REUSE); + }); + } + if (!result.has_value()) { + return ERR_SKIP; } + r_scene = result.value(); #ifndef DEBUG_ENABLED if (ver_major <= 3) { _silence_errors(false); @@ -3337,10 +3366,18 @@ Error SceneExporter::export_file_to_obj(const String &p_dest_path, const String return ERR_UNAVAILABLE; } // For some reason, scenes with meshes fail to load without the load done by ResourceLoader::load, possibly due to notification shenanigans. + std::optional> result; if (ResourceCompatLoader::is_globally_available() && using_threaded_load) { - scene = ResourceLoader::load(p_src_path, "PackedScene", ResourceFormatLoader::CACHE_MODE_REUSE, &err); + result = TaskManager::get_singleton()->dispatch_to_main_thread((std::function()>)[&]() -> Ref { + return ResourceLoader::load(p_src_path, "PackedScene", ResourceFormatLoader::CACHE_MODE_REUSE, &err); + }); } else { - scene = ResourceCompatLoader::custom_load(p_src_path, "PackedScene", ResourceCompatLoader::get_default_load_type(), &err, !using_threaded_load, ResourceFormatLoader::CACHE_MODE_REUSE); + result = TaskManager::get_singleton()->dispatch_to_main_thread((std::function()>)[&]() -> Ref { + return ResourceCompatLoader::custom_load(p_src_path, "PackedScene", ResourceCompatLoader::get_default_load_type(), &err, !using_threaded_load, ResourceFormatLoader::CACHE_MODE_REUSE); + }); + } + if (!result.has_value()) { + return ERR_SKIP; // cancelled } ERR_FAIL_COND_V_MSG(err, ERR_FILE_CANT_READ, "Failed to load scene " + p_src_path); return export_scene_to_obj(scene, p_dest_path, iinfo, ver_major); @@ -3610,14 +3647,19 @@ struct BatchExportToken : public TaskRunnerStruct { _scene = scene; if (!is_obj_output()) { root = instance._instantiate_scene(scene); + if (root) { + root = instance._set_stuff_from_instanced_scene(root); + } if (!root) { _scene = nullptr; - err = ERR_CANT_ACQUIRE_RESOURCE; + err = _check_cancelled(); + if (err == OK) { + err = ERR_CANT_ACQUIRE_RESOURCE; + } report->set_error(err); after_preload(); return false; } - root = instance._set_stuff_from_instanced_scene(root); } } From cdea230502dac5e746cb9ecefe8d6f0921071e54 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Fri, 17 Apr 2026 06:37:18 -0700 Subject: [PATCH 017/134] make scene exporter truly multithreaded, don't use batch --- exporters/resource_exporter.cpp | 6 ++++++ exporters/resource_exporter.h | 2 ++ exporters/scene_exporter.cpp | 23 ++++++++++++++++++----- exporters/scene_exporter.h | 4 +++- utility/import_exporter.cpp | 24 +++++++++++++++++++----- 5 files changed, 48 insertions(+), 11 deletions(-) diff --git a/exporters/resource_exporter.cpp b/exporters/resource_exporter.cpp index a82ad7c61..a9fc42afb 100644 --- a/exporters/resource_exporter.cpp +++ b/exporters/resource_exporter.cpp @@ -92,6 +92,12 @@ Error ResourceExporter::test_export(const Ref &export_report, cons return ERR_UNAVAILABLE; } +void ResourceExporter::prebatch_export() { +} + +void ResourceExporter::postbatch_export() { +} + void ResourceExporter::_bind_methods() { ClassDB::bind_method(D_METHOD("get_name"), &ResourceExporter::get_name); ClassDB::bind_method(D_METHOD("supports_nonpack_export"), &ResourceExporter::supports_nonpack_export); diff --git a/exporters/resource_exporter.h b/exporters/resource_exporter.h index 2fa97ffae..caf5523fb 100644 --- a/exporters/resource_exporter.h +++ b/exporters/resource_exporter.h @@ -24,6 +24,8 @@ class ResourceExporter : public RefCounted { virtual String get_default_export_extension(const String &res_path) const; virtual Vector get_export_extensions(const String &res_path) const; virtual Error test_export(const Ref &export_report, const String &original_project_dir) const; + virtual void prebatch_export(); + virtual void postbatch_export(); static Ref _check_for_existing_resources(const Ref &iinfo); }; diff --git a/exporters/scene_exporter.cpp b/exporters/scene_exporter.cpp index ec4fe0c05..0645ed520 100644 --- a/exporters/scene_exporter.cpp +++ b/exporters/scene_exporter.cpp @@ -356,7 +356,7 @@ void get_deps_recursive(const String &p_path, HashMap &r_deps, bool GLBExporterInstance::using_threaded_load() const { // If the scenes are being exported using the worker task pool, we can't use threaded load - return !supports_multithread(); + return true; } Error load_model(const String &p_filename, tinygltf::Model &model, String &r_error) { @@ -3227,7 +3227,7 @@ Error GLBExporterInstance::_load_scene(Ref &r_scene) { #endif std::optional> result; // For some reason, scenes with meshes fail to load without the load done by ResourceLoader::load, possibly due to notification shenanigans. - if (ResourceCompatLoader::is_globally_available() && using_threaded_load()) { + if (ResourceCompatLoader::is_globally_available()) { result = TaskManager::get_singleton()->dispatch_to_main_thread((std::function()>)[&]() -> Ref { return ResourceLoader::load(source_path, "PackedScene", ResourceFormatLoader::CACHE_MODE_REUSE, &err); }); @@ -3360,20 +3360,19 @@ Error SceneExporter::export_file(const String &p_dest_path, const String &p_src_ Error SceneExporter::export_file_to_obj(const String &p_dest_path, const String &p_src_path, Ref iinfo) { Error err; Ref scene; - bool using_threaded_load = !SceneExporter::can_multithread; int ver_major = iinfo.is_valid() ? iinfo->get_ver_major() : get_ver_major(p_src_path); if (ver_major < MINIMUM_GODOT_VER_SUPPORTED) { return ERR_UNAVAILABLE; } // For some reason, scenes with meshes fail to load without the load done by ResourceLoader::load, possibly due to notification shenanigans. std::optional> result; - if (ResourceCompatLoader::is_globally_available() && using_threaded_load) { + if (ResourceCompatLoader::is_globally_available()) { result = TaskManager::get_singleton()->dispatch_to_main_thread((std::function()>)[&]() -> Ref { return ResourceLoader::load(p_src_path, "PackedScene", ResourceFormatLoader::CACHE_MODE_REUSE, &err); }); } else { result = TaskManager::get_singleton()->dispatch_to_main_thread((std::function()>)[&]() -> Ref { - return ResourceCompatLoader::custom_load(p_src_path, "PackedScene", ResourceCompatLoader::get_default_load_type(), &err, !using_threaded_load, ResourceFormatLoader::CACHE_MODE_REUSE); + return ResourceCompatLoader::custom_load(p_src_path, "PackedScene", ResourceCompatLoader::get_default_load_type(), &err, true, ResourceFormatLoader::CACHE_MODE_REUSE); }); } if (!result.has_value()) { @@ -3870,6 +3869,20 @@ Ref SceneExporter::export_file_with_options(const String &out_path return token->report; } +void SceneExporter::prebatch_export() { + bool remove_physics_bodies = GDREConfig::get_singleton()->get_setting("Exporter/Scene/GLTF/remove_physics_bodies", false); + if (remove_physics_bodies) { + unregister_physics_extension(); + } +} + +void SceneExporter::postbatch_export() { + bool remove_physics_bodies = GDREConfig::get_singleton()->get_setting("Exporter/Scene/GLTF/remove_physics_bodies", false); + if (remove_physics_bodies) { + register_physics_extension(); + } +} + Ref SceneExporter::export_resource(const String &output_dir, Ref iinfo) { BatchExportToken token(output_dir, iinfo); token.batch_preload(); diff --git a/exporters/scene_exporter.h b/exporters/scene_exporter.h index 404032da9..e87f3e689 100644 --- a/exporters/scene_exporter.h +++ b/exporters/scene_exporter.h @@ -51,7 +51,7 @@ class SceneExporter : public ResourceExporter { }; static Error export_file_to_non_glb(const String &p_src_path, const String &p_dest_path, Ref iinfo); - static constexpr bool can_multithread = false; + static constexpr bool can_multithread = true; static SceneExporter *get_singleton(); SceneExporter(); @@ -66,6 +66,8 @@ class SceneExporter : public ResourceExporter { virtual bool supports_nonpack_export() const override { return false; } virtual String get_default_export_extension(const String &res_path) const override; virtual Vector get_export_extensions(const String &res_path) const override; + virtual void prebatch_export() override; + virtual void postbatch_export() override; static Ref export_file_with_options(const String &out_path, const String &res_path, const Dictionary &options); static size_t get_vram_usage(); diff --git a/utility/import_exporter.cpp b/utility/import_exporter.cpp index ccb8436a4..50ee7538d 100644 --- a/utility/import_exporter.cpp +++ b/utility/import_exporter.cpp @@ -1008,7 +1008,15 @@ Error ImportExporter::export_imports(const String &p_out_dir, const Vectorpostbatch_export(); + } + ran_prebatch_export = false; + } if (cancelled) { print_line("Export cancelled!"); } @@ -1151,11 +1159,12 @@ Error ImportExporter::export_imports(const String &p_out_dir, const Vectorget_name() == "PackedScene") { - scene_tokens.push_back(iinfo); - export_dest_to_iinfo.insert(iinfo->get_export_dest(), Vector>({ iinfo })); - continue; - } else if (!exporter->supports_multithread()) { + // if (exporter->get_name() == "PackedScene") { + // scene_tokens.push_back(iinfo); + // export_dest_to_iinfo.insert(iinfo->get_export_dest(), Vector>({ iinfo })); + // continue; + // } else + if (!exporter->supports_multithread()) { supports_multithreading = false; } } else { @@ -1300,6 +1309,11 @@ Error ImportExporter::export_imports(const String &p_out_dir, const Vectorprebatch_export(); + } + ran_prebatch_export = true; GDRELogger::clear_error_queues(); if (tokens.size() > 0) { err = TaskManager::get_singleton()->run_multithreaded_group_task( From 9741512e6f9db934ad1ebeafed4730bfb7101e29 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Fri, 17 Apr 2026 06:37:18 -0700 Subject: [PATCH 018/134] cleanup scene multithread implementation --- exporters/scene_exporter.cpp | 297 +---------------------------------- exporters/scene_exporter.h | 7 - utility/import_exporter.cpp | 65 ++++---- 3 files changed, 36 insertions(+), 333 deletions(-) diff --git a/exporters/scene_exporter.cpp b/exporters/scene_exporter.cpp index 0645ed520..18e290849 100644 --- a/exporters/scene_exporter.cpp +++ b/exporters/scene_exporter.cpp @@ -3619,6 +3619,7 @@ struct BatchExportToken : public TaskRunnerStruct { } if (is_text_output()) { + after_preload(); return false; } err = _check_cancelled(); @@ -3884,7 +3885,7 @@ void SceneExporter::postbatch_export() { } Ref SceneExporter::export_resource(const String &output_dir, Ref iinfo) { - BatchExportToken token(output_dir, iinfo); + BatchExportToken token(output_dir, iinfo, {}, true); token.batch_preload(); token.batch_export_instanced_scene(); token.post_export(); @@ -3970,297 +3971,3 @@ Error GLBExporterInstance::_batch_export_instanced_scene(Node *root, const Strin return err; } -// void do_batch_export_instanced_scene(int i, BatchExportToken *tokens); -void SceneExporter::do_batch_export_instanced_scene(int i, std::shared_ptr *tokens) { - std::shared_ptr token = tokens[i]; - token->batch_export_instanced_scene(); -} - -void SceneExporter::do_single_threaded_batch_export_instanced_scene(int i, std::shared_ptr *tokens) { - std::shared_ptr token = tokens[i]; - token->batch_preload(); - token->batch_export_instanced_scene(); -} - -String SceneExporter::get_batch_export_description(int i, std::shared_ptr *tokens) const { - return "Exporting scene " + tokens[i]->p_src_path; -} - -struct BatchExportTokenSort { - bool operator()(const std::shared_ptr &a, const std::shared_ptr &b) const { - return a->export_end_time - a->export_start_time > b->export_end_time - b->export_start_time; - } -}; - -size_t SceneExporter::get_vram_usage() { - List mesh_info; - RS::get_singleton()->mesh_debug_usage(&mesh_info); - List tinfo; - RS::get_singleton()->texture_debug_usage(&tinfo); - - size_t total_vmem = 0; - - for (const RenderingServerTypes::MeshInfo &E : mesh_info) { - total_vmem += E.vertex_buffer_size + E.attribute_buffer_size + E.skin_buffer_size + E.index_buffer_size + E.blend_shape_buffer_size + E.lod_index_buffers_size; - } - for (const RenderingServerTypes::TextureInfo &E : tinfo) { - total_vmem += E.bytes; - } - return total_vmem; -} - -inline uint64_t get_average_delta(const Vector &deltas) { - uint64_t total_delta = 0; - for (auto &delta : deltas) { - total_delta += delta; - } - return total_delta / (deltas.size() > 0 ? deltas.size() : 1); -} - -#ifdef DEBUG_ENABLED -#define perf_print(...) print_line(__VA_ARGS__) -#define perf_print_verbose(...) print_verbose(__VA_ARGS__) -#else -#define perf_print(...) print_verbose(__VA_ARGS__) -#define perf_print_verbose(...) (void)(0) -#endif - -Vector> SceneExporter::batch_export_files(const String &output_dir, const Vector> &scenes) { - Vector> tokens; - HashMap export_dest_to_iinfo; - Vector> reports; - for (auto &scene : scenes) { - Ref report = ResourceExporter::_check_for_existing_resources(scene); - if (report.is_valid()) { - // No extant resources to export - report->set_exporter(get_name()); - reports.push_back(report); - continue; - } - String ext = scene->get_export_dest().get_extension().to_lower(); - if (_check_unsupported(scene->get_ver_major(), ext == "escn" || ext == "tscn")) { - // Unsupported version - report = memnew(ExportReport(scene, get_name())); - _set_unsupported(report, scene->get_ver_major(), ext == "obj"); - reports.push_back(report); - continue; - } - - tokens.push_back(std::make_shared(output_dir, scene, Dictionary(), true)); - int idx = tokens.size() - 1; - auto &token = tokens[idx]; - token->instance.image_path_to_data_hash = Dictionary(); - String export_dest = token->get_export_dest(); - if (export_dest_to_iinfo.has(export_dest)) { - int other_i = export_dest_to_iinfo[export_dest]; - auto &other_token = tokens.write[other_i]; - if (other_token->original_export_dest.get_file() != other_token->get_export_dest().get_file()) { - other_token->append_original_ext_to_export_dest(); - export_dest_to_iinfo.erase(export_dest); - } - } - if (export_dest_to_iinfo.has(export_dest)) { - if (token->original_export_dest.get_file() != token->get_export_dest().get_file()) { - token->append_original_ext_to_export_dest(); - } else { - token->set_export_dest(export_dest.get_basename() + "_" + itos(idx) + "." + export_dest.get_extension()); - } - } else { - export_dest_to_iinfo[export_dest] = idx; - } - } - - if (tokens.is_empty()) { - return reports; - } - - bool remove_physics_bodies = GDREConfig::get_singleton()->get_setting("Exporter/Scene/GLTF/remove_physics_bodies", false); - if (remove_physics_bodies) { - unregister_physics_extension(); - } - - static constexpr int64_t ONE_MB = 1024LL * 1024LL; - static constexpr int64_t ONE_GB = 1024LL * ONE_MB; - static constexpr int64_t EIGHT_GB = 8LL * ONE_GB; - - BatchExportToken::in_progress = 0; - Dictionary mem_info = OS::get_singleton()->get_memory_info(); - int64_t current_memory_usage = OS::get_singleton()->get_static_memory_usage(); - // available memory does not include disk cache, so we should take the larger of this or peak memory usage (i.e. the most we've allocated) - int64_t total_mem_available = MAX((int64_t)mem_info["available"], (int64_t)OS::get_singleton()->get_static_memory_peak_usage() - current_memory_usage); - int64_t max_usage = TaskManager::maximum_memory_usage; - size_t current_vram_usage = get_vram_usage(); - size_t peak_vram_usage = current_vram_usage; - // TODO: get the real available VRAM; right now we assume 4GB - const int64_t MAX_VRAM = EIGHT_GB / 2; - // 75% of the default thread pool size; we can't saturate the thread pool because some loaders may make use of them and we'll cause a deadlock. - const int64_t default_num_threads = OS::get_singleton()->get_default_thread_pool_size() * 0.75; - // The smaller of either the above value, or the amount of memory available to us, divided by 256MB (conservative estimate of 256MB per scene) - const int64_t number_of_threads = MIN(default_num_threads, max_usage / (ONE_GB / 4)); - - print_line("\nExporting scenes..."); - perf_print(vformat("Current memory usage: %.02fMB, Total memory available: %.02fMB", (double)current_memory_usage / (double)ONE_MB, (double)total_mem_available / (double)ONE_MB)); - perf_print(vformat("VRAM usage: %.02fMB", (double)current_vram_usage / (double)ONE_MB)); - perf_print(vformat("Default thread pool size: %d", OS::get_singleton()->get_default_thread_pool_size())); - perf_print(vformat("Number of threads to use for scene export: %d\n", (int64_t)number_of_threads)); - - Vector deltas; - Vector abnormal_deltas; - Vector resync_deltas; - constexpr size_t MAX_DELTA_COUNT = 1000000; - constexpr uint64_t DRAW_DELTA_THRESHOLD = 50000; - resync_deltas.reserve(MAX_DELTA_COUNT); - - auto average_out_delta = [&](Vector &deltas) { - if (deltas.size() >= MAX_DELTA_COUNT) { - auto avg = get_average_delta(deltas); - deltas.clear(); - deltas.resize(100); - deltas.fill(avg); - deltas.reserve(MAX_DELTA_COUNT); - } - }; - auto resync_rendering_server = [&]() { - auto start_tick = OS::get_singleton()->get_ticks_usec(); - RS::get_singleton()->sync(); - auto current_tick = OS::get_singleton()->get_ticks_usec(); - auto delta = current_tick - start_tick; - resync_deltas.push_back(delta); - if (delta > 1) { - abnormal_deltas.push_back(delta); - } - average_out_delta(resync_deltas); - }; - auto _get_vram_usage = [&]() { -#if 0 // RS::get_singleton()->texture_debug_usage() is causing segfaults if we have the export thread pool running, so disabling for now - auto start_tick = OS::get_singleton()->get_ticks_usec(); - current_vram_usage = SceneExporter::get_vram_usage(); - peak_vram_usage = MAX(peak_vram_usage, current_vram_usage); - auto current_tick = OS::get_singleton()->get_ticks_usec(); - if (current_tick - last_print_time > 1000000) { - perf_print_verbose(vformat("VRAM usage: %.02fMB", (double)current_vram_usage / (double)ONE_MB)); - last_print_time = current_tick; - } - auto delta = current_tick - start_tick; - deltas.push_back(delta); - if (delta > 10000 && current_tick - last_warn_time > 1000000) { - perf_print("Took " + itos(delta / 1000) + "ms to get VRAM usage!!"); - last_warn_time = current_tick; - } - average_out_delta(deltas); -#endif - return current_vram_usage; - }; - - Error err = OK; - auto export_start_time = OS::get_singleton()->get_ticks_msec(); - auto last_update_time = export_start_time; - - auto ensure_progress = [&]() { - auto current_time = OS::get_singleton()->get_ticks_usec(); - if (current_time - last_update_time >= DRAW_DELTA_THRESHOLD) { - last_update_time = current_time; - return TaskManager::get_singleton()->update_progress_bg(true); - } - resync_rendering_server(); - return false; - }; - if (number_of_threads <= 1 || GDREConfig::get_singleton()->get_setting("force_single_threaded", false)) { - print_line("Forcing single-threaded scene export..."); - err = TaskManager::get_singleton()->run_group_task_on_current_thread( - this, - &SceneExporter::do_single_threaded_batch_export_instanced_scene, - tokens.ptrw(), - tokens.size(), - &SceneExporter::get_batch_export_description, - "Exporting scenes", - "Exporting scenes", - true); - } else { - auto task_id = TaskManager::get_singleton()->add_group_task( - this, - &SceneExporter::do_batch_export_instanced_scene, - tokens.ptrw(), - tokens.size(), - &SceneExporter::get_batch_export_description, - "Exporting scenes", - "Exporting scenes", - true, - number_of_threads, - true); - - int starve_count = 0; - for (int64_t i = 0; i < tokens.size(); i++) { - auto &token = tokens[i]; - if (token->batch_preload()) { - // we need to do the export on the main thread - token->batch_export_instanced_scene(); - } - // Don't load more than the current number of tasks being processed - _get_vram_usage(); - auto start_wait = OS::get_singleton()->get_ticks_msec(); - auto last_warn_time = 0; - while (BatchExportToken::in_progress >= number_of_threads || - (BatchExportToken::in_progress > 0 && - (TaskManager::is_memory_usage_too_high() || - current_vram_usage > MAX_VRAM))) { - if (ensure_progress()) { - break; - } - _get_vram_usage(); - if (start_wait + 10000 < OS::get_singleton()->get_ticks_msec()) { - if (last_warn_time == 0) { - perf_print(vformat("\n*** STALL WARNING %d ***", ++starve_count)); - } - if (last_warn_time + 1000 < OS::get_singleton()->get_ticks_msec()) { - perf_print(vformat("Scene export stalled: %d/%d threads running, memory starved: %s", BatchExportToken::in_progress + 0, number_of_threads, TaskManager::is_memory_usage_too_high() ? "yes" : "no")); - last_warn_time = OS::get_singleton()->get_ticks_msec(); - } - } - OS::get_singleton()->delay_usec(1000); - } - // calling update_progress_bg serves three purposes: - // 1) updating the progress bar - // 2) checking if the task was cancelled - // 3) allowing the main loop to iterate so that the command queue is flushed - // Without flushing the command queue, GLTFDocument::append_from_scene will hang - if (TaskManager::get_singleton()->update_progress_bg(true)) { - break; - } - } - while (!TaskManager::get_singleton()->is_current_task_completed(task_id)) { - _get_vram_usage(); - if (ensure_progress()) { - break; - } - OS::get_singleton()->delay_usec(1000); - } - err = TaskManager::get_singleton()->wait_for_task_completion(task_id); - - // get the average delta - uint64_t average_delta = get_average_delta(deltas); - uint64_t average_resync_delta = get_average_delta(resync_deltas); - perf_print(vformat("Average time to resync rendering server: %.02fms", (double)average_resync_delta / 1000.0)); - perf_print(vformat("Average time to get VRAM usage: %.02fms", (double)average_delta / 1000.0)); - perf_print(vformat("Peak VRAM usage: %.02fMB", (double)peak_vram_usage / (double)ONE_MB)); - } - perf_print("\n"); - auto export_end_time = OS::get_singleton()->get_ticks_msec(); - tokens.sort_custom(); -#if PRINT_PERF_CSV - perf_print("scene,time,surface_count"); - for (auto &token : tokens) { - perf_print(vformat("%s,%.02f,%d", token->get_export_dest(), (double)(token->export_end_time - token->export_start_time) / 1000.0, (int64_t)token->surface_count)); - } -#endif - for (auto &token : tokens) { - token->post_export(err); - reports.push_back(token->report); - } - - print_line(vformat("*** Exporting %d scenes took %.02fs\n", tokens.size(), (double)(export_end_time - export_start_time) / 1000.0)); - if (remove_physics_bodies) { - register_physics_extension(); - } - return reports; -} diff --git a/exporters/scene_exporter.h b/exporters/scene_exporter.h index e87f3e689..649277efd 100644 --- a/exporters/scene_exporter.h +++ b/exporters/scene_exporter.h @@ -27,11 +27,6 @@ class SceneExporter : public ResourceExporter { static SceneExporter *singleton; - void do_batch_export_instanced_scene(int i, std::shared_ptr *tokens); - void do_single_threaded_batch_export_instanced_scene(int i, std::shared_ptr *tokens); - - String get_batch_export_description(int i, std::shared_ptr *tokens) const; - protected: static void _bind_methods(); @@ -70,8 +65,6 @@ class SceneExporter : public ResourceExporter { virtual void postbatch_export() override; static Ref export_file_with_options(const String &out_path, const String &res_path, const Dictionary &options); - static size_t get_vram_usage(); - Vector> batch_export_files(const String &output_dir, const Vector> &scenes); static constexpr int get_minimum_godot_ver_supported() { return MINIMUM_GODOT_VER_SUPPORTED; diff --git a/utility/import_exporter.cpp b/utility/import_exporter.cpp index 50ee7538d..aa1b342c2 100644 --- a/utility/import_exporter.cpp +++ b/utility/import_exporter.cpp @@ -1097,9 +1097,10 @@ Error ImportExporter::export_imports(const String &p_out_dir, const Vector non_high_priority_tokens; Vector tokens; Vector non_multithreaded_tokens; - Vector> scene_tokens; + Vector scene_tokens; HashMap>> export_dest_to_iinfo; HashSet dupes; + bool force_single_threaded = GDREConfig::get_singleton()->get_setting("force_single_threaded", false); for (int i = 0; i < _files.size(); i++) { Ref iinfo = _files[i]; if (partial_export && !hashset_intersects_vector(files_to_export_set, iinfo->get_dest_files())) { @@ -1155,18 +1156,15 @@ Error ImportExporter::export_imports(const String &p_out_dir, const Vectorget_setting("force_single_threaded", false); + bool supports_multithreading = !force_single_threaded; bool is_high_priority = importer == "gdextension" || importer == "gdnative"; + bool is_scene = false; if (exporter_map.has(importer)) { auto &exporter = exporter_map.get(importer); - // if (exporter->get_name() == "PackedScene") { - // scene_tokens.push_back(iinfo); - // export_dest_to_iinfo.insert(iinfo->get_export_dest(), Vector>({ iinfo })); - // continue; - // } else if (!exporter->supports_multithread()) { supports_multithreading = false; } + is_scene = exporter->get_name() == "PackedScene"; } else { // Non-exportable resource that wasn't imported or auto-converted, don't report it if (iinfo->get_ver_major() <= 2 && !iinfo->is_import() && !iinfo->is_auto_converted()) { @@ -1174,15 +1172,17 @@ Error ImportExporter::export_imports(const String &p_out_dir, const Vector= int64_t(scene_idx + 1) * int64_t(non_scene_count)) { + tokens.push_back(scene_tokens[scene_idx]); + scene_idx++; + } + } + while (scene_idx < scene_count) { + tokens.push_back(scene_tokens[scene_idx]); + scene_idx++; + } + } pr->set_progress_length(false, tokens.size() + non_multithreaded_tokens.size()); @@ -1347,24 +1368,6 @@ Error ImportExporter::export_imports(const String &p_out_dir, const Vector 0) { - if (GDRESettings::get_singleton()->is_headless()) { - print_line("WARNING: Some scenes can fail to export in headless mode. This is due to a limitation of the Godot engine.\n" - "If some scenes fail to export, try running in GUI mode."); - } - // intentionally not checking for cancellation here; scene export can sometimes hang on certain scenes, and we don't want to cancel the entire export - auto reports = SceneExporter::get_singleton()->batch_export_files(output_dir, scene_tokens); - for (int i = 0; i < reports.size(); i++) { - ExportToken token = { reports[i]->get_import_info(), reports[i], false }; - rewrite_metadata(token); - non_multithreaded_tokens.push_back(token); - } - } - - if (err != OK) { - reset_before_return(true); - return err; - } tokens.append_array(non_multithreaded_tokens); pr->step("Finalizing...", tokens.size() - 1, false); pr->set_progress_length(true); From f22f46f3b70984efc84b42e1b7c10b2244eea3fc Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Fri, 17 Apr 2026 06:37:18 -0700 Subject: [PATCH 019/134] simplify ResourceFormatSaverCompatTextInstance::ensure_packed_scenes --- compat/resource_compat_text.cpp | 102 ++++++-------------------------- 1 file changed, 17 insertions(+), 85 deletions(-) diff --git a/compat/resource_compat_text.cpp b/compat/resource_compat_text.cpp index 5886ac1fc..c95b32e83 100644 --- a/compat/resource_compat_text.cpp +++ b/compat/resource_compat_text.cpp @@ -3122,92 +3122,23 @@ bool is_packed_scene(const Ref &p_resource) { return p_resource.is_valid() && _resource_get_class(p_resource) == "PackedScene"; } -Ref _ensure_resource_is_packed_scene(const Ref &p_resource, HashMap> &p_seen_resources, HashSet &seen_objects, int recursion_depth, bool *changed_something); - -Variant scan_variant(const Variant &v, HashMap> &p_seen_resources, HashSet &seen_objects, int recursion_depth, bool *p_changed_something) { - ERR_FAIL_COND_V_MSG(recursion_depth > 1024, Variant(), "Recursion depth exceeded"); - recursion_depth++; - Variant result = v; - switch (v.get_type()) { - case Variant::OBJECT: { - Ref res = v; - if (is_packed_scene(res)) { - Ref sub_scene; - if (!p_seen_resources.has(res->get_path())) { - if (_resource_get_class(res) == "PackedScene") { - sub_scene = _ensure_resource_is_packed_scene(res, p_seen_resources, seen_objects, recursion_depth, p_changed_something); - if (res->get_class() == "MissingResource") { - *p_changed_something = true; - } - } - } else { - sub_scene = p_seen_resources.get(res->get_path()); - } - if (sub_scene.is_valid()) { - p_seen_resources[res->get_path()] = sub_scene; - result = sub_scene; - } else { - p_seen_resources[res->get_path()] = res; - result = res; - } - } else if (res.is_valid()) { - if (!p_seen_resources.has(res->get_path())) { - List property_list; - res->get_property_list(&property_list); - for (const PropertyInfo &pi : property_list) { - bool changed_something = false; - Variant v = scan_variant(res->get(pi.name), p_seen_resources, seen_objects, recursion_depth, &changed_something); - if (changed_something) { - res->set(pi.name, v); - } - } - p_seen_resources[res->get_path()] = res; - } - } else if (Object *obj = v.operator Object *(); obj != nullptr && !seen_objects.has(obj)) { - seen_objects.insert(obj); - List property_list; - obj->get_property_list(&property_list); - for (const PropertyInfo &pi : property_list) { - if (pi.usage & PROPERTY_USAGE_STORAGE) { - bool changed_something = false; - Variant v = scan_variant(obj->get(pi.name), p_seen_resources, seen_objects, recursion_depth, &changed_something); - if (changed_something) { - obj->set(pi.name, v); - } - } - } - } - } break; - case Variant::ARRAY: { - Array a = v; - for (int j = 0; j < a.size(); j++) { - a[j] = scan_variant(a[j], p_seen_resources, seen_objects, recursion_depth, p_changed_something); - } - result = a; - } break; - case Variant::DICTIONARY: { - Dictionary d; - for (const KeyValue &kv : v.operator Dictionary()) { - Variant key = scan_variant(kv.key, p_seen_resources, seen_objects, recursion_depth, p_changed_something); - Variant value = scan_variant(kv.value, p_seen_resources, seen_objects, recursion_depth, p_changed_something); - d[key] = value; - } - v.operator Dictionary().clear(); - v.operator Dictionary().assign(d); - result = v; - } break; - default: { - } - } - return result; -} - -Ref _ensure_resource_is_packed_scene(const Ref &p_resource, HashMap> &p_seen_resources, HashSet &seen_objects, int recursion_depth, bool *changed_something) { +Ref _ensure_resource_is_packed_scene(const Ref &p_resource, HashMap> &p_seen_resources, HashSet &seen_objects, int recursion_depth) { Ref r_packed_scene = p_resource; if (_resource_get_class(p_resource) == "PackedScene") { - Dictionary bundle = scan_variant(p_resource->get("_bundled"), p_seen_resources, seen_objects, recursion_depth, changed_something); + Dictionary bundle = p_resource->get("_bundled"); // we need to go through the variants in the bundle and ensure that any MissingResources that are PackedScenes are also replaced with instantiated packed scenes // this is because of PackedScene::SceneState::get_node_instance, which returns a Ref, which will be null if it's not an instance of a PackedScene + // (We do not have to worry about PackedScenes embedded in other resources/dictionaries/arrays because this only matters for SceneState::get_node_instance) + Array arr = bundle.get("variants", Array()); + if (arr.size() > 0) { + for (int i = 0; i < arr.size(); i++) { + Ref res = arr[i]; + if (res.is_valid() && res->get_save_class() == "PackedScene") { + arr[i] = _ensure_resource_is_packed_scene(res, p_seen_resources, seen_objects, recursion_depth); + } + } + bundle.set("variants", arr); + } if (r_packed_scene.is_null()) { // MissingResource r_packed_scene = Ref(memnew(PackedScene)); if (!bundle.is_empty()) { @@ -3223,12 +3154,13 @@ Ref _ensure_resource_is_packed_scene(const Ref &p_resourc Ref ResourceFormatSaverCompatTextInstance::ensure_packed_scenes(const Ref &p_resource) { auto info = ResourceInfo::get_info_from_resource(p_resource); if (info.is_valid()) { - if (info->load_type == ResourceInfo::REAL_LOAD && info->load_type == ResourceInfo::GLTF_LOAD && p_resource->get_class() == "PackedScene") { + if ((info->load_type == ResourceInfo::REAL_LOAD || info->load_type == ResourceInfo::GLTF_LOAD) && p_resource->get_class() == "PackedScene") { return p_resource; } + } else if (p_resource->get_class() == "PackedScene") { + return p_resource; } HashMap> seen_resources; HashSet seen_objects; - bool changed_something = false; - return _ensure_resource_is_packed_scene(p_resource, seen_resources, seen_objects, 0, &changed_something); + return _ensure_resource_is_packed_scene(p_resource, seen_resources, seen_objects, 0); } From da2cc01fec75db1bea6eb0c9de1bbcadb1169bc9 Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Fri, 17 Apr 2026 06:37:18 -0700 Subject: [PATCH 020/134] scene: make debug copy behavior configurable --- exporters/scene_exporter.cpp | 23 ++++------------------- utility/gdre_config.cpp | 6 ++++++ 2 files changed, 10 insertions(+), 19 deletions(-) diff --git a/exporters/scene_exporter.cpp b/exporters/scene_exporter.cpp index 18e290849..7821d3275 100644 --- a/exporters/scene_exporter.cpp +++ b/exporters/scene_exporter.cpp @@ -42,13 +42,6 @@ #include "scene/resources/packed_scene.h" #include "utility/task_manager.h" -#ifndef MAKE_GLTF_COPY -#ifdef DEBUG_ENABLED -#define MAKE_GLTF_COPY 1 -#else -#define MAKE_GLTF_COPY 0 -#endif -#endif #ifndef PRINT_PERF_CSV #define PRINT_PERF_CSV 0 #endif @@ -1828,15 +1821,13 @@ Node *GLBExporterInstance::_set_stuff_from_instanced_scene(Node *root) { if (replaced_node_names.size() > 0) { auto thingy = String(", ").join(replaced_node_names); print_line(vformat("%s: replaced nodes with mesh instances: %s", scene_name, thingy)); -#ifdef DEBUG_ENABLED - if (true) { + if (GDREConfig::get_singleton()->get_setting("Exporter/Scene/GLTF/debug_copies", false)) { auto new_dest = "res://.tscn_manip/" + report->get_import_info()->get_export_dest().trim_prefix("res://.assets/").trim_prefix("res://").get_basename() + ".tscn"; auto new_dest_path = output_dir.path_join(new_dest.replace_first("res://", "")); Ref scene = memnew(PackedScene); scene->pack(root); ResourceCompatLoader::save_custom(scene, new_dest_path, GODOT_VERSION_MAJOR, GODOT_VERSION_MINOR); } -#endif } Vector fps_values; @@ -2607,8 +2598,7 @@ Error GLBExporterInstance::_export_instanced_scene(Node *root, const String &p_d GDRE_SCN_EXP_FAIL_V_MSG(ERR_FILE_CANT_WRITE, "Failed to serialize glTF document"); } -#if MAKE_GLTF_COPY - { + if (GDREConfig::get_singleton()->get_setting("Exporter/Scene/GLTF/debug_copies", false)) { // save a gltf copy for debugging Dictionary gltf_asset = state->get_json().get("asset", Dictionary()); gltf_asset["generator"] = "GDRE Tools"; @@ -2620,7 +2610,6 @@ Error GLBExporterInstance::_export_instanced_scene(Node *root, const String &p_d _serialize_file(state, gltf_path, buffer_paths, !use_double_precision); } GDRE_SCN_EXP_CHECK_CANCEL(); -#endif auto check_unique = [&](String &name, HashSet &image_map) { if (name.is_empty()) { return; @@ -2846,9 +2835,8 @@ Error GLBExporterInstance::_export_instanced_scene(Node *root, const String &p_d json["asset"] = gltf_asset; } -#if MAKE_GLTF_COPY GDRE_SCN_EXP_CHECK_CANCEL(); - if (p_dest_path.get_extension() == "glb") { + if (p_dest_path.get_extension() == "glb" && GDREConfig::get_singleton()->get_setting("Exporter/Scene/GLTF/debug_copies", false)) { // save a gltf copy for debugging auto rel_path = p_dest_path.begins_with(output_dir) ? p_dest_path.trim_prefix(output_dir).simplify_path().trim_prefix("/") : p_dest_path.get_file(); if (iinfo.is_valid()) { @@ -2863,7 +2851,6 @@ Error GLBExporterInstance::_export_instanced_scene(Node *root, const String &p_d Vector buffer_paths; _serialize_file(state, gltf_path, buffer_paths, !use_double_precision); } -#endif GDRE_SCN_EXP_CHECK_CANCEL(); Vector buffer_paths; err = _serialize_file(state, p_dest_path, buffer_paths, !use_double_precision); @@ -3721,13 +3708,11 @@ struct BatchExportToken : public TaskRunnerStruct { if (err == OK) { report->set_saved_path(p_dest_path); } -#ifdef DEBUG_ENABLED // export a text copy so we can see what went wrong - if (true) { + if (GDREConfig::get_singleton()->get_setting("Exporter/Scene/GLTF/debug_copies", false)) { auto new_dest = "res://.tscn_copy/" + report->get_import_info()->get_export_dest().trim_prefix("res://.assets/").trim_prefix("res://").get_basename() + ".tscn"; auto new_dest_path = output_dir.path_join(new_dest.replace_first("res://", "")); ResourceCompatLoader::to_text(p_src_path, new_dest_path); } -#endif } _scene = nullptr; // print_line("Finished exporting scene " + p_src_path); diff --git a/utility/gdre_config.cpp b/utility/gdre_config.cpp index 9839b4edf..a9cca6c8e 100644 --- a/utility/gdre_config.cpp +++ b/utility/gdre_config.cpp @@ -357,6 +357,12 @@ Vector> GDREConfig::_init_default_settings() { "Replace shader materials", "Replaces shader materials with generated standard materials when exporting the scene.\nSolves issues with exported scenes not having any textures.\nWARNING: This is experimental and may result in inaccurate exports.", false)), + memnew(GDREConfigSetting( + "Exporter/Scene/GLTF/debug_copies", + "Create debug copies", + "For development.", + false, + true)), memnew(GDREConfigSetting( "Exporter/Texture/create_lossless_copy", "Create lossless copy", From a3102f1a8658dd76e3cd6d3e2eee7d8b486696aa Mon Sep 17 00:00:00 2001 From: nikitalita <69168929+nikitalita@users.noreply.github.com> Date: Fri, 17 Apr 2026 06:37:18 -0700 Subject: [PATCH 021/134] Better steam detection --- utility/gdre_settings.cpp | 17 +++++++++++++++++ utility/gdre_settings.h | 4 ++++ utility/import_exporter.cpp | 31 +++++++++++++++---------------- 3 files changed, 36 insertions(+), 16 deletions(-) diff --git a/utility/gdre_settings.cpp b/utility/gdre_settings.cpp index eddf707a1..55d6221c7 100644 --- a/utility/gdre_settings.cpp +++ b/utility/gdre_settings.cpp @@ -2095,11 +2095,13 @@ Error GDRESettings::load_pack_gdscript_cache(bool p_reset) { } namespace { struct ScriptCacheTask { + Ref steam_plugin_regex; struct ScriptCacheTaskToken { String orig_path; bool is_gdscript; Dictionary d; Ref