Add processed assets and improve browser rendering and runtime support

CMakeLists.txt

@@ -36,6 +36,7 @@ option(DAS_IMGUI_DISABLED "Disable dasImgui (Dear ImGui bindings + widget DSL; n
 option(DAS_VULKAN_DISABLED "Disable dasVulkan (Vulkan bindings generated from vk.xml; headers + volk vendored, no SDK needed)" OFF)
 option(DAS_AUDIO_DISABLED "Disable dasAudio (Miniaudio sound library)" OFF)
 option(DAS_STDDLG_DISABLED "Disable dasStdDlg (File new,open,save etc dialogs)" OFF)
+option(DAS_MESHOPTIMIZER_DISABLED "Disable dasMeshOptimizer (mesh processing and buffer codecs)" OFF)
 option(DAS_STBIMAGE_DISABLED "Disable dasStbImage (StbImage bindings, image loading and saving)" OFF)
 option(DAS_METAL_DISABLED "Disable dasMetal (native Metal compute binding, Apple only)" OFF)
 option(DAS_ACCELERATE_DISABLED "Disable dasAccelerate (Accelerate BLAS binding — AMX/SME, Apple only)" OFF)

daslib/ARCHITECTURE.md

@@ -285,3 +285,11 @@ Five companions carry a concern each; a section number is unique across all six
   hook by field name, so a C++ side back on `const` leaves daslang writing through a const
   reference with every suite green. `progArgs` (the program's `options`), `simulate` and the
   `aot*` hooks stay const on both sides.
+
+## 40. Package release symbols {#package-release-symbols}
+
+`daspkg.das:release_include_symbols` sets `ReleaseSpec.include_symbols`.
+`utils/daspkg/commands.das` consumes that field: the native release copies Windows
+PDBs, while `cmd_release_wasm` passes it to `release_one_wasm_app` to retain DWARF,
+logical frames and the emcc symbol map. The manifest contract and these release
+implementations describe the same option.

daslib/daspkg.das

@@ -159,9 +159,7 @@ def release_requires_jit() {
     _release_spec.requires_jit = true
 }
 
-//! Ship debug symbols for every shipped binary into `<bundle>/symbols/`, so a minidump from the
-//! deployed tree resolves (`cdb -y <bundle>\symbols`). Opt-in: symbols dwarf the payload (daslang's
-//! runtime PDBs are ~320 MB). Windows `.pdb` only; see `skills/daspkg.md`.
+[arch(at="ARCHITECTURE.md#package-release-symbols")]
 def release_include_symbols() {
     _release_spec.include_symbols = true
 }

daslib/fio.das

@@ -69,12 +69,12 @@ def fload(file : file; size : int64; blk : block<(data : array<uint8>) : void>)
 
 [generic]
 def fload(f : file; buf : auto(BufType) -const) {
-    var dfh : df_header
+    var dfh : df_header // nolint:LINT003,LINT019 - In generic instantiations, _builtin_read writes this raw output buffer.
     let r1 = _builtin_read(f, dfh, typeinfo sizeof(type<df_header>))
     if (r1 < 0 || dfh.magic != df_magic || dfh.size < 0) return false
     var loaded = false
     _builtin_load(f, int64(dfh.size)) $(data : array<uint8>) {
-        if (length(data) != 0) {
+        if (!empty(data)) {
             binary_load(buf, data)
             loaded = true
         }

doc/reflections/das2rst.das

@@ -113,6 +113,7 @@ require daslib/remove_call_args
 require daslib/stringify
 require peg/peg
 require gltf/gltf_boost
+require gltf/gltf_processed
 require daslib/temp_strings
 require daslib/type_traits
 require daslib/typemacro_boost
@@ -130,6 +131,7 @@ require openai/openai_vision
 require pugixml/PUGIXML_boost
 require stbimage/stbimage_boost
 require stbimage/stbimage_ttf
+require meshoptimizer
 require audio/audio_boost
 require audio/audio_wav
 require audio/audio_record
@@ -1574,11 +1576,23 @@ def document_module_stbimage(_root : string) {
         group_by_regex("Write to memory", mod, %regex~(stbi_write_png_to_memory|stbi_write_bmp_to_memory|stbi_write_tga_to_memory|stbi_write_jpg_to_memory)$%%),
         group_by_regex("Write settings", mod, %regex~(stbi_flip_vertically_on_write|stbi_write_set_.*|stbi_write_get_.*)$%%),
         group_by_regex("Image resizing", mod, %regex~(stbir_resize|stbir_resize_uint8_srgb|stbir_resize_uint8_linear|stbir_resize_float_linear)$%%),
+        group_by_regex("Texture block compression", mod, %regex~stb_compress_blocks$%%),
         group_by_regex("Animated PNG (APNG) writer", mod, %regex~stbi_apng_(begin|frame|end|dropped)$%%)
     )
     document("Image loading, writing, and resizing (stb_image)", mod, "stbimage.rst", groups)
 }
 
+def document_module_meshoptimizer(_root : string) {
+    var mod = get_module("meshoptimizer")
+    var groups <- array<DocGroup>(
+        group_by_regex("Vertex remapping", mod, %regex~meshopt_(generate_remap|remap_vertices|remap_indices)$%%),
+        group_by_regex("Mesh optimization", mod, %regex~meshopt_optimize_(cache|fetch)$%%),
+        group_by_regex("Vertex codec", mod, %regex~meshopt_(encode|decode)_vertices$%%),
+        group_by_regex("Index codec", mod, %regex~meshopt_(encode|decode)_indices$%%)
+    )
+    document("Mesh remapping, optimization, and buffer codecs (meshoptimizer)", mod, "meshoptimizer.rst", groups)
+}
+
 def document_module_raster(_root : string) {
     var mod = get_module("raster")
     var groups <- array<DocGroup>(
@@ -1915,6 +1929,26 @@ def document_module_gltf(_root : string) {
     documents("glTF 2.0 loader (dasGLTF): GLB/.gltf parse, accessor decode, node hierarchy, skinning and keyframe animation into a backend-neutral scene", mod, "gltf.rst", groups)
 }
 
+def document_module_texture_blocks(_root : string) {
+    var mod = find_module("texture_blocks")
+    var groups <- array<DocGroup>(
+        group_by_regex("Texture types", mod, %regex~.*(BlockMip|BlockTexture)$%%),
+        group_by_regex("Compression and caching", mod, %regex~.*(block_texture_key|compress_block_texture|cached_block_texture)$%%),
+        group_by_regex("Validation and decoding", mod, %regex~.*(block_texture_valid|decode_block_mip)$%%)
+    )
+    document("Portable BC texture compression, mip generation, caching, validation, and decoding", mod, "texture_blocks.rst", groups)
+}
+
+def document_module_gltf_processed(_root : string) {
+    var mod = find_module("gltf_processed")
+    var groups <- array<DocGroup>(
+        group_by_regex("Processing types", mod, %regex~.*(ProcessingProfile|ProcessedPrimitive|ProcessedMesh|ProcessedAsset|ProcessedCatalogEntry|ProcessedCatalog|ProcessedManifest)$%%),
+        group_by_regex("Validation and persistence", mod, %regex~.*(processed_valid|save_processed|load_processed|load_and_process_gltf)$%%),
+        group_by_regex("Geometry processing", mod, %regex~.*(processed_stride|processed_vertex_bytes|processed_indices|process_primitive|process_gltf|unpack_processed_geometry)$%%)
+    )
+    document("Backend-neutral glTF geometry and texture preprocessing, validation, and persistence", mod, "gltf_processed.rst", groups)
+}
+
 // das2rst compiles tuned libraries to inspect their source API. The explicit
 // documentation context keeps all [tune] transforms and policy machinery inert.
 [export]
@@ -1935,6 +1969,7 @@ def main {  // nolint:STYLE038 — flat one-call-per-documented-module dispatch
     document_module_dashv(root)
     document_module_openai(root)
     document_module_stbimage(root)
+    document_module_meshoptimizer(root)
     document_module_raster(root)
     document_module_stbtruetype(root)
     document_module_uriparser(root)
@@ -1962,6 +1997,8 @@ def main {  // nolint:STYLE038 — flat one-call-per-documented-module dispatch
     // document dasPEG module
     document_module_peg(root)
     document_module_gltf(root)
+    document_module_texture_blocks(root)
+    document_module_gltf_processed(root)
     // document dasSpirv reflection module
     document_module_spirv(root)
     // document the dasLLAMA facade module (the engine modules behind it stay undocumented by design)

doc/source/stdlib/handmade/function-meshoptimizer-meshopt_decode_indices-0xc16e60c000f53bf4.rst

@@ -0,0 +1 @@
+Decode exactly ``count`` triangle-list indices into ``output`` and return true on success. ``count`` must be divisible by three; invalid codec data returns false and clears ``output``.

doc/source/stdlib/handmade/function-meshoptimizer-meshopt_decode_vertices-0x25019ef686d88461.rst

@@ -0,0 +1 @@
+Decode ``count`` packed vertex records of ``stride`` bytes into ``output`` and return true on success. Invalid codec data returns false and clears ``output``; invalid stride, oversized output, or aliasing ``encoded`` raises an error.

doc/source/stdlib/handmade/function-meshoptimizer-meshopt_encode_indices-0xcebc111796f37b7c.rst

@@ -0,0 +1 @@
+Encode triangle-list ``indices`` with the meshoptimizer index codec and resize ``output`` to the encoded byte count. Every index must be less than the supplied vertex ``count``. Triangle order and winding are preserved, but the three indices within each triangle may be cyclically rotated.

doc/source/stdlib/handmade/function-meshoptimizer-meshopt_encode_vertices-0xe1ad9380a82a899d.rst

@@ -0,0 +1 @@
+Encode packed vertex records into the meshoptimizer vertex codec and resize ``output`` to the encoded byte count. ``stride`` must be nonzero, 4-byte aligned, at most 256 bytes, and divide the input size; ``output`` must not alias ``vertices``.

doc/source/stdlib/handmade/function-meshoptimizer-meshopt_generate_remap-0xdbd2de7ee479a3c2.rst

@@ -0,0 +1 @@
+Build a vertex remap table from triangle-list ``indices`` and packed ``vertices``, returning the number of unique vertices. ``stride`` must be nonzero, 4-byte aligned, at most 256 bytes, and divide the vertex byte count; every index must be in range, and ``remap`` must not alias ``indices``.

doc/source/stdlib/handmade/function-meshoptimizer-meshopt_optimize_cache-0x9f5637f748a14874.rst

@@ -0,0 +1 @@
+Reorder triangle-list ``indices`` in place to improve post-transform vertex-cache locality while preserving each triangle's orientation. Every index must be less than ``count``.

doc/source/stdlib/handmade/function-meshoptimizer-meshopt_optimize_fetch-0xcd80344cb34f875f.rst

@@ -0,0 +1 @@
+Reorder ``indices`` and packed ``vertices`` in place for sequential vertex fetch, discard unreferenced vertices, and return the resulting vertex count. The vertex stride and all triangle-list indices are validated before either array is modified.

doc/source/stdlib/handmade/function-meshoptimizer-meshopt_remap_indices-0x69706c78732a40b2.rst

@@ -0,0 +1 @@
+Apply a vertex remap table to triangle-list ``indices`` and resize ``output`` to the index count. Every input index must address the remap table, and ``output`` must not alias ``remap``.

doc/source/stdlib/handmade/function-meshoptimizer-meshopt_remap_vertices-0xe33b3a97b651e4e8.rst

@@ -0,0 +1 @@
+Apply ``remap`` to packed ``vertices`` and resize ``output`` to ``count * stride`` bytes. The remap table must contain one entry per input vertex, each entry must be less than ``count`` or the unused value ``0xffffffff``, and ``output`` must not alias ``vertices``.

doc/source/stdlib/handmade/function-stbimage-stb_compress_blocks-0xcc27189d5d081eae.rst

@@ -0,0 +1 @@
+Compress one tightly packed RGBA8 image into GPU texture blocks, replicating edge pixels when a dimension is not divisible by four. ``format`` selects BC1 (0), BC3 (1), BC4 from red (2), or BC5 from red and green (3); dimensions must be 1 through 32768, ``pixels`` must contain exactly ``width * height * 4`` bytes, and ``output`` must not alias ``pixels``.

doc/source/stdlib/handmade/module-gltf_processed.rst

@@ -0,0 +1 @@
+Backend-neutral glTF preprocessing that packs and optionally encodes geometry, compresses textures, and persists validated monolithic or split-file assets.

doc/source/stdlib/handmade/module-meshoptimizer.rst

@@ -0,0 +1 @@
+Checked array bindings for meshoptimizer 1.2 vertex remapping, vertex-cache and vertex-fetch optimization, and vertex and index buffer codecs. The API accepts triangle-list ``uint`` indices and packed vertex bytes with a nonzero, 4-byte-aligned stride of at most 256 bytes.

doc/source/stdlib/handmade/module-texture_blocks.rst

@@ -0,0 +1 @@
+Portable BC1, BC3, BC4, and BC5 texture compression with complete mip chains, deterministic cache keys, serialized validation, and RGBA8 decoding.

doc/source/stdlib/sec_media.rst

@@ -14,3 +14,6 @@ Image loading, saving, resizing, pixel manipulation, TrueType font rendering, an
    generated/stbtruetype.rst
    generated/stbimage_ttf.rst
    generated/gltf.rst
+   generated/meshoptimizer.rst
+   generated/texture_blocks.rst
+   generated/gltf_processed.rst

modules/dasAudio/ARCHITECTURE.md

@@ -3,6 +3,25 @@
 **Who reads this: me.** Durable facts about why the audio system is shaped the way it is -
 readable cold, no history, no PR numbers.
 
+## Threaded WebAssembly output
+
+The AudioWorklet only consumes float PCM from a preallocated single-producer,
+single-consumer ring and zero-fills an underrun. A regular pthread runs `mix_audio`,
+including command-stream draining, decoder allocation and daslang context locking.
+The worker produces 128-frame blocks into a 512-frame ring (about 10.7 ms at 48 kHz).
+It sleeps briefly when the ring is full; the worklet never waits for it.
+
+This separation is required for correctness: an Emscripten AudioWorklet is a Wasm
+Worker, and the hybrid runtime can initialize it with no pthread pointer. C++ mutex
+ownership then sees thread ID zero, so a mutex acquired by that worklet need not
+exclude a main-thread producer. Draining `Stream` there can race with `Stream::push`.
+Do not put the command queue, allocator, context locks or script execution back in
+that callback even when a particular browser/toolchain appears to tolerate it.
+
+Start the ring/producer before starting the device. On teardown, stop/join the
+producer, stop the device, then free the ring and mixer context. Native, null-device
+and single-threaded WASM backends retain the direct callback path.
+
 ## The audio callback must never wait on a game thread
 
 `data_callback` (`src/dasAudio.cpp`) runs on miniaudio's realtime thread and has a buffer

modules/dasAudio/CMakeLists.txt

@@ -66,7 +66,7 @@ IF ((NOT DAS_AUDIO_INCLUDED) AND ((NOT ${DAS_AUDIO_DISABLED}) OR (NOT DEFINED DA
 	SETUP_AUDIO(dasModuleAudio)
 
 	# Threaded web build: drive miniaudio's emscripten backend via AudioWorklet
-	# (mixer callback on the dedicated audio rendering thread) instead of the
+	# (PCM consumption on the audio rendering thread, mixing on a pthread) instead of the
 	# main-thread ScriptProcessorNode. Needs the worklet emcc link flags
 	# (-sAUDIO_WORKLET=1 -sWASM_WORKERS=1, set in web/CMakeLists.txt) + the
 	# non-blocking ma_device_init patch (patches/miniaudio_memory64.cmake), so it

modules/dasAudio/src/dasAudio.cpp

@@ -6,6 +6,8 @@
 #include "daScript/simulate/bind_enum.h"
 
 #include <atomic>
+#include <thread>
+#include <chrono>
 
 // include vorbis extras before miniaudio
 #define STB_VORBIS_HEADER_ONLY
@@ -308,7 +310,7 @@ void on_error_log ( void * , ma_uint32 level, const char * message ) {
     }
 }
 
-void data_callback(ma_device*, void* pOutput, const void*, ma_uint32 frameCount) {
+static void mix_audio(void* pOutput, ma_uint32 frameCount) {
     float fdt = 1.0f / float(g_rate);
     Array buffer;
     array_mark_locked(buffer, pOutput, frameCount * g_channels);
@@ -324,6 +326,70 @@ void data_callback(ma_device*, void* pOutput, const void*, ma_uint32 frameCount)
     daScriptEnvironment::setBound(saved);
 }
 
+#if defined(__EMSCRIPTEN__) && defined(__EMSCRIPTEN_PTHREADS__)
+static ma_pcm_rb g_playback_rb;
+static std::thread g_playback_thread;
+static std::atomic<bool> g_playback_running { false };
+static bool g_playback_buffered = false;
+
+static bool start_playback_worker() {
+    if (ma_pcm_rb_init(ma_format_f32, (ma_uint32)g_channels, 512, nullptr, nullptr, &g_playback_rb) != MA_SUCCESS) return false;
+    g_playback_running.store(true, std::memory_order_release);
+    try {
+        g_playback_thread = std::thread([] {
+            while (g_playback_running.load(std::memory_order_acquire)) {
+                ma_uint32 count = 128;
+                void * output = nullptr;
+                if (ma_pcm_rb_acquire_write(&g_playback_rb, &count, &output) != MA_SUCCESS || !count) {
+                    std::this_thread::sleep_for(std::chrono::milliseconds(1));
+                    continue;
+                }
+                memset(output, 0, (size_t)count * g_channels * sizeof(float));
+                mix_audio(output, count);
+                ma_pcm_rb_commit_write(&g_playback_rb, count);
+            }
+        });
+    } catch (...) {
+        g_playback_running.store(false, std::memory_order_release);
+        ma_pcm_rb_uninit(&g_playback_rb);
+        return false;
+    }
+    g_playback_buffered = true;
+    return true;
+}
+
+static void stop_playback_worker() {
+    g_playback_running.store(false, std::memory_order_release);
+    if (g_playback_thread.joinable()) g_playback_thread.join();
+}
+
+static void read_playback_ring(void * output, ma_uint32 frameCount) {
+    float * destination = (float *)output;
+    while (frameCount) {
+        ma_uint32 count = frameCount;
+        void * input = nullptr;
+        if (ma_pcm_rb_acquire_read(&g_playback_rb, &count, &input) != MA_SUCCESS || !count) {
+            memset(destination, 0, (size_t)frameCount * g_channels * sizeof(float));
+            return;
+        }
+        memcpy(destination, input, (size_t)count * g_channels * sizeof(float));
+        ma_pcm_rb_commit_read(&g_playback_rb, count);
+        destination += (size_t)count * g_channels;
+        frameCount -= count;
+    }
+}
+#endif
+
+void data_callback(ma_device*, void* pOutput, const void*, ma_uint32 frameCount) {
+#if defined(__EMSCRIPTEN__) && defined(__EMSCRIPTEN_PTHREADS__)
+    if (g_playback_buffered) {
+        read_playback_ring(pOutput, frameCount);
+        return;
+    }
+#endif
+    mix_audio(pOutput, frameCount);
+}
+
 Context & dasAudio_mixerContext ( Context * context, LineInfoArg * at ) {
     if ( !g_mixer_context ) context->throw_error_at(at,"sound mixer is not initialized");
     return *g_mixer_context;
@@ -371,9 +437,22 @@ bool dasAudio_init ( TFunc<void,TTemporary<TArray<float>>,int32_t,int32_t,float>
     g_mixer_context->verySafeContext = false;
     g_mixer_function = mixer;
     g_mixer_env = daScriptEnvironment::getBound();
+#if defined(__EMSCRIPTEN__) && defined(__EMSCRIPTEN_PTHREADS__)
+    if (g_device.pContext->backend == ma_backend_webaudio && !start_playback_worker()) {
+        ma_device_uninit(&g_device);
+        g_mixer_context.reset();
+        return false;
+    }
+#endif
     if ( ma_device_start(&g_device) != MA_SUCCESS ) {
+#if defined(__EMSCRIPTEN__) && defined(__EMSCRIPTEN_PTHREADS__)
+        stop_playback_worker();
+#endif
         ma_device_uninit(&g_device);
         if ( g_null_context_inited ) { ma_context_uninit(&g_null_context); g_null_context_inited = false; }
+#if defined(__EMSCRIPTEN__) && defined(__EMSCRIPTEN_PTHREADS__)
+        if (g_playback_buffered) { ma_pcm_rb_uninit(&g_playback_rb); g_playback_buffered = false; }
+#endif
         g_mixer_context.reset();
         return false;
     }
@@ -413,7 +492,13 @@ static bool ensure_capture_context () {
 
 void dasAudio_finalize ( void ) {
     if ( g_mixer_initialized ) {
+#if defined(__EMSCRIPTEN__) && defined(__EMSCRIPTEN_PTHREADS__)
+        stop_playback_worker();
+#endif
         ma_device_uninit(&g_device);
+#if defined(__EMSCRIPTEN__) && defined(__EMSCRIPTEN_PTHREADS__)
+        if (g_playback_buffered) { ma_pcm_rb_uninit(&g_playback_rb); g_playback_buffered = false; }
+#endif
         g_mixer_context.reset();
         g_mixer_initialized = false;
     }

modules/dasGLTF/.das_module

@@ -6,7 +6,7 @@ def initialize(project_path : string) {
     // gltf_gl / gltf_pbr are the opt-in OpenGL adapters (they require dasOpenGL); registering their
     // paths here only makes `require gltf/gltf_gl` resolvable — the neutral core never pulls them in.
     let gltf_paths = ["gltf_types", "gltf_accessor", "gltf_parse", "gltf_scene", "gltf_boost",
-        "gltf_gl", "gltf_pbr", "gltf_pbr_common"]
+        "gltf_gl", "gltf_atmosphere", "gltf_pbr", "gltf_pbr_common", "gltf_processed", "gltf_processed_gl"]
     for (path in gltf_paths) {
         register_native_path("gltf", "{path}", "{project_path}/gltf/{path}.das")
     }

modules/dasGLTF/ARCHITECTURE.md

@@ -0,0 +1,26 @@
+# dasGLTF architecture
+
+## 1. Atmospheric transmittance {#atmosphere-transmittance}
+
+The height-density integral uses a primitive continuous at the layer top, where
+constant density meets exponential falloff. Rays crossing that boundary therefore
+retain continuous transmittance.
+
+## 2. Directional shadow filtering {#directional-shadow-filtering}
+
+Directional shadow gradients are evaluated before divergent bounds checks. Shadow
+coordinates reconstruct the geometric receiver plane independently of shading normals.
+Filtering interpolates depth-comparison results, rather than raw depths: a 4-by-4
+nearest-sample footprint combines the weights of a 3-by-3 bilinear comparison kernel.
+
+## 3. Processed vertex portability {#processed-vertex-portability}
+
+UV packing uses scalar float16 conversion so CPU processing does not depend on
+native half-vector casts.
+
+## 4. Compressed texture capability {#compressed-texture-capability}
+
+When the reported compressed-format list omits S3TC formats, upload checks S3TC
+extension support. Its sRGB variants also require core or extension sRGB support.
+Each mip dimension is block-aligned or one or two pixels; other dimensions cause
+CPU decoding and uncompressed upload for the entire texture.