From ffdfe7b7a93e0899b9444209d431d6284cf2e113 Mon Sep 17 00:00:00 2001 From: Boris Batkin Date: Wed, 16 Sep 2026 10:38:51 -0700 Subject: [PATCH] the pre-release sweep: C++ cites architecture by anchor and lint reads it, the arch tools locate instead of quote, the worker heap test measures collection, and the install lists catch up A C or C++ source cites an architecture section the way a .das declaration does now. The spelling is `// #` inside a `//` comment - the kept-set pointer the comment rulebook already named, with the anchor in place of `sec.N`, a number nothing could check. One scanner (utils/common/arch_citations.das) reads both spellings: a .das file for its [arch(at=...)] annotations, a C++ file for its pointers, whose path is root-relative and whose root is the first folder up from the citing file the path exists under, so a header in include/ reaches the src/ document that owns its mechanism and a planted fixture tree resolves inside itself; the folder-tree rule does not bind C++ for the same reason. A URL to a document on the web is not a pointer - a scheme's walk-back leaves a leading slash, a bare host a dotted first segment - which imgui's vendored sources proved on the first run. LINT026 walks C++ in both directions, and preflight's lint gate hands changed .cpp and headers to it; arch-extract reads the pointers, so a section cited only from C++ lands in the SDK excerpt (src/misc sec.8 has no other citer); the sixty-one `sec.N` pointers are rewritten to anchors and the eleven sections they name carry `{#anchor}` headings. The fixture plants a C++ source with a resolving pointer, a dangling one, an #include, a block comment and two URLs, a header under include/deep reaching src/ through the root above both, and a negative control that swaps the anchor away and back. `arch_of` and `arch_sites` answered a question nobody asked: eight calls in their whole life, against thousands for lint and compile_check, because `arch_of` dumped the section's text and lost the coordinates the next step - a Read or an Edit of that section - needs, while `arch_sites` was a grep. `arch_of` is a locator now, one line per citation, `: -> :- #`, both spellings; `arch_sites` gives each anchor its section's line range and reads the C++ homes (src/, include/, every module's src/) beyond the document's own subtree, since a C++ citer may sit anywhere in them - the census of src/ast's manifest section grew from 11 sites to 22 in 10 files. CLAUDE.md sends a reader to the document first and to the tools for where. tests/strudel_device/test_worker_heap.das flaked at 10242 KB against a 10 MB bar, on darwin Debug. The number the strudel worker publishes is its heap BEFORE the tick's collect, so it reads live set plus that tick's garbage, and a tick's garbage runs from hundreds of kilobytes to ten megabytes: on an idle 64-core box the published number spans 9.7 MB while its floor moves 0 KB. `peak - base` measured the burstiness of a pattern query, not collection, and the bar was widened 2 -> 5 -> 10 MB in one day chasing it. The test now asserts the two things only a collect produces - a published number lower than the one before it (28 per six worker-seconds here; zero without collection, which test_worker_no_gc.das pins) and a floor that holds across two stretches (uncollected grows ~10 MB a stretch) - with warm-up and the stretches on the worker's own clock, so a slow box warms up before it is measured. The install lists: modules/dasGLTF/tools/convert.das, the asset converter the README documents, and utils/aot/main_llvm_aot.das, the LLVM object AOT driver, ship; the twelve utils/* install GLOBs carry CONFIGURE_DEPENDS, so a file added after a box's last configure lands in its bundle; MESHOPTIMIZER.LICENSE joins the bundle smoke's roster (the module is default-on and its notice installed, and the gate that exists to notice a silently skipped install rule did not list it). CMake's file(WRITE) emits the platform newline, so on Windows the xxd step rewrote every tracked .das.inc with CRLF on every build - its "not modified, skipped" branch could never fire there - and `git add -A` swept two of them into the last PR twice; the step pins LF through configure_file, and .gitattributes pins *.das.inc so a checkout cannot flip it back. CHANGELIST.md carries this and the previous PR. Co-Authored-By: Claude Fable 5.1 --- .gitattributes | 5 + ARCHITECTURE_COMMON.md | 9 +- CHANGELIST.md | 6 + CLAUDE.md | 10 +- CMakeLists.txt | 28 +++-- CMakeXxdImpl.txt | 7 +- REVIEW_COMMON.md | 9 +- ci/smoke_test_bundle.sh | 2 +- include/daScript/ast/ast.h | 6 +- include/daScript/ast/ast_handle.h | 2 +- include/daScript/ast/ast_serializer.h | 2 +- include/daScript/ast/dyn_modules.h | 2 +- include/daScript/simulate/aot_builtin_ast.h | 2 +- modules/dasGLTF/CMakeLists.txt | 5 + modules/dasImgui/src/module_imgui_app.cpp | 6 +- .../dasLLVM/ARCHITECTURE_TARGET_FEATURES.md | 4 +- skills/comment_style_hygiene.md | 8 +- skills/mcp_tools.md | 6 +- src/ast/ARCHITECTURE.md | 8 +- src/ast/ast_module.cpp | 4 +- src/ast/ast_parse.cpp | 20 +-- src/ast/ast_simulate.cpp | 6 +- src/ast/dyn_modules.cpp | 12 +- src/builtin/ARCHITECTURE.md | 12 +- src/builtin/REVIEW.md | 8 +- src/builtin/module_builtin_ast.cpp | 8 +- src/builtin/module_builtin_ast_serialize.cpp | 24 ++-- src/builtin/module_builtin_dasbind.cpp | 6 +- src/builtin/module_builtin_debugger.cpp | 2 +- src/builtin/module_builtin_fio.cpp | 8 +- src/builtin/module_builtin_rtti.cpp | 2 +- src/misc/ARCHITECTURE.md | 2 +- src/misc/job_que.cpp | 4 +- src/parser/parser_impl.cpp | 6 +- tests/strudel_device/test_worker_heap.das | 86 ++++++++++--- utils/common/arch_citations.das | 117 ++++++++++++++++-- utils/internal/arch-extract/main.das | 4 +- utils/internal/preflight/main.das | 21 ++-- utils/lint/main.das | 7 +- utils/lint/tests/lint026_arch_citations.das | 71 +++++++++++ utils/mcp/registry_das.das | 6 +- utils/mcp/test_tools.das | 54 +++++--- utils/mcp/tools/arch_common.das | 31 ++++- utils/mcp/tools/arch_of.das | 36 ++---- utils/mcp/tools/arch_sites.das | 30 ++++- 45 files changed, 515 insertions(+), 199 deletions(-) diff --git a/.gitattributes b/.gitattributes index 4c74eaa5c2..e3e6d0388b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -27,6 +27,11 @@ tree-sitter-daslang/src/*.c text eol=lf tree-sitter-daslang/src/*.json text eol=lf tree-sitter-daslang/src/tree_sitter/*.h text eol=lf +# The xxd step (CMakeXxdImpl.txt) regenerates a tracked .das.inc beside its source on every +# build; the generator writes LF, and this keeps a checkout under autocrlf=true from reading +# every line as changed and `git add -A` from committing a line-ending flip. +*.das.inc text eol=lf + # The v1 -> gen2 converter preserves whatever line endings its input had, and # its suite proves that by converting an LF fixture and a CRLF twin it builds # from that same fixture. Under autocrlf=true the fixtures check out as CRLF, diff --git a/ARCHITECTURE_COMMON.md b/ARCHITECTURE_COMMON.md index f6d32af523..646f6c3cf2 100644 --- a/ARCHITECTURE_COMMON.md +++ b/ARCHITECTURE_COMMON.md @@ -31,9 +31,12 @@ and the exception ledger. Nothing else.** for different content: append new sections, never renumber. **A section that code implements carries a `{#anchor}` on its heading, and every anchor is -cited by an `[arch(at="#")]` on a function in the document's own folder tree.** -One anchor per heading; the anchor name is stable across rewording, like the section number. -An anchor no function cites, a citation naming no anchor, and a citation reaching a document +cited by an `[arch(at="#")]` on a function in the document's own folder tree, or +by a `// #` pointer in a C or C++ source.** One anchor per heading; the anchor +name is stable across rewording, like the section number. A C++ pointer's path is root-relative +(`src/ast/ARCHITECTURE.md#...`), found by walking up from the citing file, and the folder-tree +rule does not bind it - a header under `include/` answers for a mechanism `src/` documents. +An anchor no code cites, a citation naming no anchor, and a `.das` citation reaching a document outside the citer's folder tree are all lint findings (LINT026), in every folder. A mechanism another folder's document states is restated here in prose - a paragraph, not a resolved link - and the code cites this document. A section no function implements is narrative and carries no diff --git a/CHANGELIST.md b/CHANGELIST.md index 30633e9be7..1a41a07bd4 100644 --- a/CHANGELIST.md +++ b/CHANGELIST.md @@ -152,6 +152,7 @@ A native glTF 2.0 loader with an OpenGL PBR renderer, viewer, and capstone tutor - **Real threads on the web** (#3293, #3309, #3331) - `DAS_WASM_PTHREADS` with an AudioWorklet audio thread, real core count via `emscripten_num_logical_cores`, and the worklet futex + persistent-jobque fixes - **Pure-das Dear ImGui renderer** (#3283) and its compile-time WebGL2 draw path + `get_running_platform_name` (#3294); strudel audio in the playground (#3272); embedded fonts (#3292); canvas-CSS-size/DPR + `glfw_toggle_fullscreen` externs for responsive, fullscreen-capable cards (#3323, #3326) - **Static-link gates** (#3351, #3358) - `examples/fatman` links every external C++ module into one host with a CI lane (the ABI canary for daslang-vs-externals drift), and its web sibling builds the imgui family from source into the threaded wasm playground +- **The browser tier's K-quant dot, FMA, spinning workers, and a mixer off the host clock** (#4045) - `dot_k4q8` rides the `idot` lattice op (both nibbles into a signed `byte16`, one body, no target branch; 1.1x -> 1.7x real time for Pocket TTS on the parrot page, 1.90x on x64, bit-exact), `+relaxed-simd` joins the wasm feature string so a contracted multiply-add becomes `f32x4.relaxed_madd` instead of splitting (8728 contractions across the module), the job queue's spin window reads the clock once per stride and pauses with no JS in it (pool clock time 25 s -> 5 s per 15 s window), and a browser's workers take the same 30 ms spin window as every other target (1.7x -> 2.3x); dasAudio's browser mixer parks on a drain counter the worklet bumps instead of a sleep, primes a 20 ms ring before the device starts, and counts underruns (`sound_playback_underrun_frames`) #### dasTerminal: Terminal Emulator in daslang (#3510, #3513, #3514, #3516) @@ -264,6 +265,7 @@ Z3 SMT solver bindings as a dynamic module, dasLLVM-style. - **`daslib/build_const` + `[EnvConfig]`** (#3643, #3701) - compile-time constants that fold pre-infer, feed `static_if`, and land in the semantic hash; clargs environment-variable twins; and the shared `daslib/env_registry` knob rail (dasLLAMA's 146 env vars in eleven structs, drift-tested docs) - **Review rails** (#3711, #3723, #3743, #3750) - folder-scoped `REVIEW.md` checklists under one shared contract with tool-driven discovery, the constitutional branch-test rule with a per-PR TDD audit, and review-comment triage + `pr-babysit` watch tooling - **MCP stability** (#3729) - the `run_with_progress` channel-scope race fixed (`join()` is the fence) +- **The pre-release sweep** (#4047) - `modules/dasGLTF/tools/convert.das` (the asset converter its README documents) and `utils/aot/main_llvm_aot.das` (the LLVM object AOT driver) install with the SDK; the twelve `utils/*` install GLOBs carry `CONFIGURE_DEPENDS`, so a file added after a box's last configure lands in its bundle; the xxd step writes its tracked `.das.inc` with LF (CMake's `file(WRITE)` emits CRLF on Windows, so the "unchanged, skipped" branch never fired there and every build rewrote every `.inc`), with `.gitattributes` pinning `*.das.inc` to LF; MCP `arch_of` becomes a locator (`file:line symbol -> doc:first-last #anchor`, one line per citation, no text dump) and `arch_sites` carries each anchor's line range - the shape a Read or Edit of the section follows from #### Lint @@ -278,6 +280,7 @@ Z3 SMT solver bindings as a dynamic module, dasLLVM-style. - **The out-param and sizing wave** (#3783) - LINT023 a by-value `var` parameter written but never read (the error message that never reaches its caller), LINT024 a 32-bit product with a call among the factors wrapped in a 64-bit cast, and PERF032 `@exact_size`: on an annotated array every grow must follow a `reserve` of the same receiver - the input-sized-buffer panic class, annotated across dasLLAMA; the whole-tree sweep is clean on both the Linux and macOS module renderings, and it surfaced two compiler fixes (a location-less `delete super.self` cast, `for_each_pass_macro` skipping three macro lists) - **The full sweep + the nightly lane** (#3733, #3741) - the whole tree lints to zero: ~3900 findings retired across 841 files (net -290 lines), three rules corrected rather than obeyed, and a nightly whole-tree lint lane that promptly reaped ten stale nolints - **Lint prints the offending source line** under each warning (#3585) +- **LINT026 reads C and C++** (#4047) - a `// #` pointer inside a `//` comment is the C++ spelling of the `[arch]` citation, resolved by the shared scanner exactly as a `.das` annotation (the path root-relative, walked up from the citing file, so a header under `include/` reaches `src/`'s document; the folder-tree rule does not bind C++), checked both ways by lint, read by `arch-extract` for the SDK excerpt and by the MCP `arch` tools; preflight's lint gate hands changed C++ files to the pass. The 61 `// sec.N` pointers - a number nothing could check - are migrated to anchors, and the eleven sections they name carry `{#anchor}` headings ### Bug Fixes @@ -299,6 +302,8 @@ Z3 SMT solver bindings as a dynamic module, dasLLVM-style. - **Nine red-first fixes from the doc-probe triage** (#3662) - `operator []<-` dispatch, `require X as Y` alias registration for all forms, false `[nodiscard]` errors in string interpolation, the `each_ref` crash, transactional qmatch captures, linq `_fold` defer, SQL generated-column visibility, and the temp-string AOT hash desync (162 `error[50101]` -> 0) - **`float2(x) * y` folded to `(x*y, 0)`** (#3776) - the splat-collapse rule installed a scalar operator under a vector-typed node, and its operand swap could reorder observable effects; both arms now require the other operand to be the vector and decline impure operands - one AST-level fix covers interpreter, JIT, and AOT (v0.6.4-RC1 only) - **The RC1 `-exe` toolchain could not run from a bundle** (#3779, #3781) - the prebuilt tools' rpath missed the installed `lib/` on Linux/macOS; the module native-path table carried the build machine's absolute paths, now re-rooted at run time (exe-relative, then das-root, then baked); and `dastest.exe` demanded a JIT it does not link - it runs the interpreter, the `-jit` host is unchanged. The bundle smoke now launches every shipped tool instead of presence-checking it +- **Browser audio corrupted on Windows** (#4045) - the mixer's producer slept 1 ms when its 10.7 ms ring was full, and a sleep resolves to the host's timer quantum: ~1 ms on macOS, 15.6 ms on Windows (measured in a Worker at 16.0 ms median, 16.6 ms worst), so the ring emptied every cycle there while the same build played clean on a Mac. It parks on a drain counter now; `modules/dasAudio/REVIEW.md` bans a sleep-timed producer and a ring deepened to hide underruns. In the same change a zero spin window parked one iteration late after a served team chunk, and `standalone_capi_dasbind` could never pass under a multi-config generator (the fixture DLL landed in `Release/` while the generated bindings looked one level up) +- **`test_worker_heap.das` flaked at 10.002 MB against a 10 MB bar** (#4047) - the number the strudel worker publishes is its heap BEFORE the tick's collect, so it jitters by one tick's garbage (up to ~10 MB on a cycle-boundary tick, 9.7 MB of spread on an idle 64-core box) and a `peak - base` bar measured that burstiness, not collection - widened 2 -> 5 -> 10 MB chasing it. The test now asserts what only a collect produces - a published number lower than the one before it, and a floor that holds across two stretches - on the worker's own clock ### Site / Blog / Docs @@ -324,6 +329,7 @@ Z3 SMT solver bindings as a dynamic module, dasLLVM-style. - **The release audit** (#3773, #3782) - `utils/internal/test-release`: the compile phase proves every shipped example and tutorial against the bundle's own binary (803 targets), and the utils phase exercises every shipped tool from the extracted bundle alone (exit code plus output only a working run prints). The RC1 sweep's structural yield: `examples/` and `tutorials/` install as whole directories, so 31 silently-dropped files ship again; three ship-defects fixed; package examples moved to their package repos - **A hung sweep names the program that hung** (#3949) - dastest prints a timestamped `run N/M: ` before every program of a sequential sweep and `worker K starts: ` before every isolated batch, on stderr, flushed at once and printed under `--failures-only` too; the `run_tests_*` cmake targets carry `USES_TERMINAL`, so Ninja streams the sweep's output instead of holding it until the command ends - a lane cancelled mid-hang keeps its last line instead of losing everything the sweep printed. The first hang those lines named was `tests/debug_agent/test_callback_threadlock.das`: its hook waited on a thread that had yet to create its context, while the tick holds the agent registry every context creation needs - `DapiDebugAgent` carries the contract, and the test now proves the callback/pinvoke serialization from one thread with the new `try_invoke_in_context`, the non-blocking `invoke_in_context` that returns false, running nothing, while another thread holds the context lock - **Bounded apt on ubuntu runners** (#3785) - every workflow apt site goes through `ci/apt_install.sh`: the degrading Azure mirror swapped for the public archive (mirrorlist included), capped fetches, retried update - the worst case becomes a loud failure in minutes instead of a silent hours-long hang +- **Gate rosters catch up with the tree** (#4047) - `MESHOPTIMIZER.LICENSE` joins the bundle smoke's license roster (the module is default-on and its MIT notice installed, but the gate that exists to notice a silently-skipped install rule did not list it); preflight's compile sweep stops trying `examples/dasLLAMA/library/main.das`, whose `dasllama_lib_c` only the opt-in `dasllama_lib_jit` target writes (#4045); `tests-cpp`'s `standalone_ctx` fixture pins its output directory with `$<1:...>` so `standalone_capi_dasbind` passes under Visual Studio and Xcode generators (#4045) ### Examples and Tutorials diff --git a/CLAUDE.md b/CLAUDE.md index 3f7b361b75..917d20c994 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -44,10 +44,12 @@ beside the tree's install rules) so LINT026 stays armed in an installed SDK. The mood test routes misplaced text: must/never binding a diff -> REVIEW.md; present-tense fact -> ARCHITECTURE.md; dated or past-tense -> deleted, git keeps it. -**Before editing a function annotated `[arch(at=".md#")]`, read the section it -cites.** MCP `arch_of` returns each of a file's citations with its section text; `arch_sites` -returns a document's anchors with the code citing them, dead anchors and dangling citations -included. Citation spelling, path resolution, and failure reasons: `skills/mcp_tools.md`. +**Before editing a function annotated `[arch(at=".md#")]`, or C++ under a +`// .md#` pointer, read the section it cites** - open the document at the anchor. +MCP `arch_of` locates every citation a file makes (`file:line symbol -> doc:first-last #anchor`, +both spellings) and `arch_sites` lists a document's anchors with their line ranges and citers, +dead anchors and dangling citations included. Citation spelling, path resolution, and failure +reasons: `skills/mcp_tools.md`. **An architecture-doc heading that code cites carries the `{#anchor}` its citation names, and every anchor is cited.** LINT026 fails a citation naming no such file or anchor, and - in every diff --git a/CMakeLists.txt b/CMakeLists.txt index 86d9b3032b..73ecb040d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1973,7 +1973,9 @@ install(FILES ${PROJECT_SOURCE_DIR}/src/misc/LUAU.LICENSE DESTINATION ${DAS_INST # (machine-checked by utils/REVIEW.das). # Install aot tool (the AOT generation driver the integration scaffolds invoke) -install(FILES ${PROJECT_SOURCE_DIR}/utils/aot/main.das DESTINATION utils/aot) +install(FILES ${PROJECT_SOURCE_DIR}/utils/aot/main.das + ${PROJECT_SOURCE_DIR}/utils/aot/main_llvm_aot.das + DESTINATION utils/aot) # Install fix-lint-errors tool (mechanical lint auto-fixer) install(FILES ${PROJECT_SOURCE_DIR}/utils/fix-lint-errors/main.das DESTINATION utils/fix-lint-errors) @@ -2006,9 +2008,9 @@ install(FILES ${PROJECT_SOURCE_DIR}/utils/mcp/README.md DESTINATION utils/mcp ) -file(GLOB DAS_MCP_TOOLS ${PROJECT_SOURCE_DIR}/utils/mcp/tools/*.das) +file(GLOB DAS_MCP_TOOLS CONFIGURE_DEPENDS ${PROJECT_SOURCE_DIR}/utils/mcp/tools/*.das) install(FILES ${DAS_MCP_TOOLS} DESTINATION utils/mcp/tools) -file(GLOB DAS_MCP_SUBTOOLS ${PROJECT_SOURCE_DIR}/utils/mcp/subtools/*.das) +file(GLOB DAS_MCP_SUBTOOLS CONFIGURE_DEPENDS ${PROJECT_SOURCE_DIR}/utils/mcp/subtools/*.das) install(FILES ${DAS_MCP_SUBTOOLS} DESTINATION utils/mcp/subtools) install(FILES ${PROJECT_SOURCE_DIR}/utils/mcp/test_tools.das @@ -2037,7 +2039,7 @@ install(FILES ${PROJECT_SOURCE_DIR}/utils/lsp/README.md DESTINATION utils/lsp ) -file(GLOB DAS_LSP_SUBTOOLS ${PROJECT_SOURCE_DIR}/utils/lsp/subtools/*.das) +file(GLOB DAS_LSP_SUBTOOLS CONFIGURE_DEPENDS ${PROJECT_SOURCE_DIR}/utils/lsp/subtools/*.das) install(FILES ${DAS_LSP_SUBTOOLS} DESTINATION utils/lsp/subtools) install(FILES ${PROJECT_SOURCE_DIR}/utils/lsp/plugin/.claude-plugin/plugin.json DESTINATION utils/lsp/plugin/.claude-plugin @@ -2068,11 +2070,11 @@ install(FILES # Install dascov (code coverage) install(FILES ${PROJECT_SOURCE_DIR}/utils/dascov/main.das DESTINATION utils/dascov) install(FILES ${PROJECT_SOURCE_DIR}/utils/dascov/README.md DESTINATION utils/dascov) -file(GLOB DAS_DASCOV_TEST_FILES ${PROJECT_SOURCE_DIR}/utils/dascov/tests/*.das) +file(GLOB DAS_DASCOV_TEST_FILES CONFIGURE_DEPENDS ${PROJECT_SOURCE_DIR}/utils/dascov/tests/*.das) install(FILES ${DAS_DASCOV_TEST_FILES} DESTINATION utils/dascov/tests) # Install benchctl (benchmark analysis) -file(GLOB DAS_BENCHCTL_FILES ${PROJECT_SOURCE_DIR}/utils/benchctl/*.das) +file(GLOB DAS_BENCHCTL_FILES CONFIGURE_DEPENDS ${PROJECT_SOURCE_DIR}/utils/benchctl/*.das) install(FILES ${DAS_BENCHCTL_FILES} DESTINATION utils/benchctl) install(FILES ${PROJECT_SOURCE_DIR}/utils/benchctl/README.md DESTINATION utils/benchctl) install(FILES ${PROJECT_SOURCE_DIR}/utils/benchctl/suite.json DESTINATION utils/benchctl) @@ -2086,7 +2088,7 @@ install(DIRECTORY ${PROJECT_SOURCE_DIR}/utils/lint/tests/ ) # Install detect-dupe (cross-file duplicate-function detector) -file(GLOB DAS_DETECT_DUPE_FILES ${PROJECT_SOURCE_DIR}/utils/detect-dupe/*.das) +file(GLOB DAS_DETECT_DUPE_FILES CONFIGURE_DEPENDS ${PROJECT_SOURCE_DIR}/utils/detect-dupe/*.das) install(FILES ${DAS_DETECT_DUPE_FILES} DESTINATION utils/detect-dupe) install(FILES ${PROJECT_SOURCE_DIR}/utils/detect-dupe/README.md @@ -2100,7 +2102,7 @@ install(DIRECTORY ${PROJECT_SOURCE_DIR}/utils/detect-dupe/fixture/ # Install find-dupe (AI judge for detect-dupe clusters via das-claude). # Not built into all_utils_exe — it depends on the das-claude daspkg # package fetched at runtime, which isn't available at build time. -file(GLOB DAS_FIND_DUPE_FILES ${PROJECT_SOURCE_DIR}/utils/find-dupe/*.das) +file(GLOB DAS_FIND_DUPE_FILES CONFIGURE_DEPENDS ${PROJECT_SOURCE_DIR}/utils/find-dupe/*.das) install(FILES ${DAS_FIND_DUPE_FILES} DESTINATION utils/find-dupe) install(FILES ${PROJECT_SOURCE_DIR}/utils/find-dupe/README.md @@ -2116,7 +2118,7 @@ install(DIRECTORY ${PROJECT_SOURCE_DIR}/utils/find-dupe/tests/ # server startable: it runs interpreted with no -project, and its require graph reaches # utils/common/arch_citations.das (via utils/mcp/tools/arch_common) and # utils/common/git_signature.das (via utils/mcp/tools/cpp_common). -file(GLOB DAS_UTILS_COMMON_FILES ${PROJECT_SOURCE_DIR}/utils/common/*.das) +file(GLOB DAS_UTILS_COMMON_FILES CONFIGURE_DEPENDS ${PROJECT_SOURCE_DIR}/utils/common/*.das) install(FILES ${DAS_UTILS_COMMON_FILES} DESTINATION utils/common) install(DIRECTORY ${PROJECT_SOURCE_DIR}/utils/common/tests/ DESTINATION utils/common/tests @@ -2124,7 +2126,7 @@ install(DIRECTORY ${PROJECT_SOURCE_DIR}/utils/common/tests/ ) # Install daspkg (package manager) -file(GLOB DAS_DASPKG_FILES ${PROJECT_SOURCE_DIR}/utils/daspkg/*.das) +file(GLOB DAS_DASPKG_FILES CONFIGURE_DEPENDS ${PROJECT_SOURCE_DIR}/utils/daspkg/*.das) install(FILES ${DAS_DASPKG_FILES} DESTINATION utils/daspkg) install(FILES ${PROJECT_SOURCE_DIR}/utils/daspkg/README.md @@ -2136,7 +2138,7 @@ install(DIRECTORY ${PROJECT_SOURCE_DIR}/utils/daspkg/fixtures/ ) # Install das-fmt (daslang formatter) -file(GLOB DAS_FMT_FILES ${PROJECT_SOURCE_DIR}/utils/das-fmt/*.das) +file(GLOB DAS_FMT_FILES CONFIGURE_DEPENDS ${PROJECT_SOURCE_DIR}/utils/das-fmt/*.das) install(FILES ${DAS_FMT_FILES} DESTINATION utils/das-fmt) install(FILES ${PROJECT_SOURCE_DIR}/utils/das-fmt/README.md @@ -2147,7 +2149,7 @@ install(FILES # Install dasllama-server (OpenAI-compatible LLM server; JIT-only, runs from the # SDK's live toolchain — see its deploy-jit.ps1 for the standalone bundle path) -file(GLOB DAS_LLAMA_SERVER_FILES ${PROJECT_SOURCE_DIR}/utils/dasllama-server/*.das) +file(GLOB DAS_LLAMA_SERVER_FILES CONFIGURE_DEPENDS ${PROJECT_SOURCE_DIR}/utils/dasllama-server/*.das) list(FILTER DAS_LLAMA_SERVER_FILES EXCLUDE REGEX "/REVIEW\\.das$") install(FILES ${DAS_LLAMA_SERVER_FILES} ${PROJECT_SOURCE_DIR}/utils/dasllama-server/README.md @@ -2175,7 +2177,7 @@ install(FILES ) # Install jobque-timeline (per-lane jobque trace viewer) -file(GLOB DAS_JOBQUE_TIMELINE_FILES ${PROJECT_SOURCE_DIR}/utils/jobque-timeline/*.das) +file(GLOB DAS_JOBQUE_TIMELINE_FILES CONFIGURE_DEPENDS ${PROJECT_SOURCE_DIR}/utils/jobque-timeline/*.das) install(FILES ${DAS_JOBQUE_TIMELINE_FILES} ${PROJECT_SOURCE_DIR}/utils/jobque-timeline/README.md DESTINATION utils/jobque-timeline diff --git a/CMakeXxdImpl.txt b/CMakeXxdImpl.txt index d16061f640..bc0d5b1ef5 100644 --- a/CMakeXxdImpl.txt +++ b/CMakeXxdImpl.txt @@ -32,7 +32,12 @@ endif() string(REGEX MATCHALL "([^,]+,)([^,]+,)?([^,]+,)?([^,]+,)?([^,]+,)?([^,]+,)?([^,]+,)?([^,]+,)?" SEPARATED_LINES "${FORMATTED_HEX}, ") list(JOIN SEPARATED_LINES "\n" FORMATTED_HEX) -file(WRITE ${XXD_TMP_DIR}/${INPUT_NAME}.inc.temp "static unsigned char ${CPP_STR_CONST_NAME}[] = {\n${FORMATTED_HEX}\n};\n") +# file(WRITE) emits the platform newline, CRLF on Windows, so a tracked .inc never matched its +# LF checkout there - the skip below could not fire and every build rewrote it. configure_file +# is the 3.17 way to pin LF (file(CONFIGURE) needs 3.18). +file(WRITE ${XXD_TMP_DIR}/${INPUT_NAME}.inc.stage "static unsigned char ${CPP_STR_CONST_NAME}[] = {\n${FORMATTED_HEX}\n};\n") +configure_file(${XXD_TMP_DIR}/${INPUT_NAME}.inc.stage ${XXD_TMP_DIR}/${INPUT_NAME}.inc.temp @ONLY NEWLINE_STYLE LF) +file(REMOVE ${XXD_TMP_DIR}/${INPUT_NAME}.inc.stage) file(READ ${XXD_TMP_DIR}/${INPUT_NAME}.inc.temp TEMP_CONTENT HEX) if(EXISTS ${INPUT_DIR}/${INPUT_NAME}.inc) file(READ ${INPUT_DIR}/${INPUT_NAME}.inc DEST_CONTENT HEX) diff --git a/REVIEW_COMMON.md b/REVIEW_COMMON.md index bd0cf372f8..0aa0922c64 100644 --- a/REVIEW_COMMON.md +++ b/REVIEW_COMMON.md @@ -27,10 +27,11 @@ reachable branch ships a test that fails without it; a diff that adds a branch n distinguishes is a defect. The audit procedure - including how to settle "would this test fail without the change?" - is `skills/tdd_audit.md`. -**A diff that changes a function carrying `[arch(at="#")]` - beyond comments - -audits the anchor's other citing functions and the cited section: read the section, check it -still describes the code and each citer still conforms, verdict per function.** The citers of -one anchor share one mechanism; MCP `arch_sites` lists them. +**A diff that changes a function carrying `[arch(at="#")]`, or C++ under a +`// #` pointer - beyond comments - audits the anchor's other citing functions and +the cited section: read the section, check it still describes the code and each citer still +conforms, verdict per function.** The citers of one anchor share one mechanism; MCP +`arch_sites` lists them, both spellings. **A diff that adds, removes, or retargets an `[arch(at=...)]` citation audits the cited section against the code - both sections on a retarget.** The citation claims the section diff --git a/ci/smoke_test_bundle.sh b/ci/smoke_test_bundle.sh index 4be8841193..5fd1593ffb 100644 --- a/ci/smoke_test_bundle.sh +++ b/ci/smoke_test_bundle.sh @@ -322,7 +322,7 @@ for lic in URIPARSER DAG_NOISE VEC_MATH FMT FAST_FLOAT LUAU GLTF_SAMPLE_ASSETS \ IMGUI FREETYPE MD4C JETBRAINS_MONO KHRONOS_GL \ VULKAN_HEADERS VOLK TREE_SITTER TREE_SITTER_ICU TREE_SITTER_C \ TREE_SITTER_CPP TREE_SITTER_MARKDOWN CLIP MINFFT SPIRV_HEADERS \ - STB DROID_SANS_MONO; do + STB DROID_SANS_MONO MESHOPTIMIZER; do [[ -f "$BUNDLE/$lic.LICENSE" ]] || MISSING_LICENSES="$MISSING_LICENSES $lic" done [[ -f "$BUNDLE/LICENSE" ]] || MISSING_LICENSES="$MISSING_LICENSES " diff --git a/include/daScript/ast/ast.h b/include/daScript/ast/ast.h index 1ec99d6cc3..360d4b408c 100644 --- a/include/daScript/ast/ast.h +++ b/include/daScript/ast/ast.h @@ -1106,7 +1106,7 @@ namespace das DAS_API bool isValidBuiltinName ( const string & name, bool canPunkt = false ); - // src/ast/ARCHITECTURE.md sec.2 + // src/ast/ARCHITECTURE.md#module-scan-manifest typedef bool (*DeferredModuleLoader) ( const string & name ); DAS_API void setDeferredModuleLoader ( DeferredModuleLoader loader ); DAS_API DeferredModuleLoader getDeferredModuleLoader (); @@ -1678,7 +1678,7 @@ namespace das FileAccessPtr access; // the access this program parses through int totalFunctions = 0; int totalVariables = 0; - // src/ast/ARCHITECTURE.md sec.4 + // src/ast/ARCHITECTURE.md#program-scoped-symbol-state das_hash_set usedFunctions; das_hash_set usedVariables; das_hash_map functionIndices; @@ -1761,7 +1761,7 @@ namespace das TextWriter & logs, ModuleGroup & libGroup, CodeOfPolicies policies = CodeOfPolicies() ); DAS_CC_API ProgramPtr compileDaScriptSerialize ( const string & fileName, const FileAccessPtr & access, TextWriter & logs, ModuleGroup & libGroup, CodeOfPolicies policies = CodeOfPolicies() ); - // src/ast/ARCHITECTURE.md sec.3 + // src/ast/ARCHITECTURE.md#require-after-walk DAS_CC_API Module * requireModuleNow ( const string & requireName, const FileAccessPtr & access, TextWriter & logs, CodeOfPolicies policies = CodeOfPolicies() ); diff --git a/include/daScript/ast/ast_handle.h b/include/daScript/ast/ast_handle.h index 6e7356b58b..9a04f8108b 100644 --- a/include/daScript/ast/ast_handle.h +++ b/include/daScript/ast/ast_handle.h @@ -748,7 +748,7 @@ namespace das } }; - // src/ast/ARCHITECTURE.md sec.2 + // src/ast/ARCHITECTURE.md#module-scan-manifest __forceinline Module * vectorHomeModule ( const TypeDeclPtr & elem, const ModuleLibrary & library ) { auto t = elem; while ( t && t->isPointer() && t->firstType ) t = t->firstType; diff --git a/include/daScript/ast/ast_serializer.h b/include/daScript/ast/ast_serializer.h index 519be813e4..5f54bb3e9a 100644 --- a/include/daScript/ast/ast_serializer.h +++ b/include/daScript/ast/ast_serializer.h @@ -154,7 +154,7 @@ namespace das { // file info clean up vector deleteUponFinish; // these pointers are for builtins (which we don't serialize) and need to be cleaned manually das_hash_set doNotDelete; - // per-record tables, cleared in clearNodeIds (src/builtin/ARCHITECTURE.md sec.6) + // per-record tables, cleared in clearNodeIds (src/builtin/ARCHITECTURE.md#module-cache-record-bytes) das_hash_map writeFileInfos; vector readFileInfos; FileInfo * lastWriteFileInfo = nullptr; diff --git a/include/daScript/ast/dyn_modules.h b/include/daScript/ast/dyn_modules.h index 984da1aa51..2775805476 100644 --- a/include/daScript/ast/dyn_modules.h +++ b/include/daScript/ast/dyn_modules.h @@ -30,7 +30,7 @@ DAS_CC_API bool require_dynamic_modules(smart_ptr file_access, const vector &disabled_modules, TextWriter &tout); -// src/ast/ARCHITECTURE.md sec.2 +// src/ast/ARCHITECTURE.md#module-scan-manifest struct DynModuleManifestRow { bool dynamic = false; // false: native path (a=mod_name, b=src, c=dst); true: dynamic module (a=path, b=cpp class, c=das name) bool group = false; // group member (a=group, b=member, c=guard or empty) diff --git a/include/daScript/simulate/aot_builtin_ast.h b/include/daScript/simulate/aot_builtin_ast.h index 4addc7c498..f1a76946f2 100644 --- a/include/daScript/simulate/aot_builtin_ast.h +++ b/include/daScript/simulate/aot_builtin_ast.h @@ -528,7 +528,7 @@ namespace das { DAS_CC_API TypeDeclPtr parseMangledNameFn ( const char * txt, ModuleGroup & lib, Module * thisModule, Context * context, LineInfoArg * at ); DAS_CC_API void notInferred ( Function * func, Context * context, LineInfoArg * at ); DAS_API void collectDependencies ( FunctionPtr fun, const TBlock,TArray> & block, Context * context, LineInfoArg * line ); - // src/ast/ARCHITECTURE.md sec.4 + // src/ast/ARCHITECTURE.md#program-scoped-symbol-state DAS_CC_API bool ast_is_function_used ( const Program * program, const Function * fn, Context * context, LineInfoArg * at ); DAS_CC_API bool ast_is_variable_used ( const Program * program, const Variable * var, Context * context, LineInfoArg * at ); DAS_CC_API int32_t ast_function_index ( const Program * program, const Function * fn, Context * context, LineInfoArg * at ); diff --git a/modules/dasGLTF/CMakeLists.txt b/modules/dasGLTF/CMakeLists.txt index f8d1148318..29d29c78c4 100644 --- a/modules/dasGLTF/CMakeLists.txt +++ b/modules/dasGLTF/CMakeLists.txt @@ -12,5 +12,10 @@ IF(NOT DAS_GLTF_INCLUDED) install(FILES ${PROJECT_SOURCE_DIR}/modules/dasGLTF/.das_module DESTINATION ${DAS_INSTALL_MODULESDIR}/dasGLTF ) + # the asset converter README.md documents - a shipped tree with no way to make + # a .das_gltf cannot use the processed-asset rail + install(FILES ${PROJECT_SOURCE_DIR}/modules/dasGLTF/tools/convert.das + DESTINATION ${DAS_INSTALL_MODULESDIR}/dasGLTF/tools + ) ENDIF() diff --git a/modules/dasImgui/src/module_imgui_app.cpp b/modules/dasImgui/src/module_imgui_app.cpp index 024bf693fd..effca6b9fb 100644 --- a/modules/dasImgui/src/module_imgui_app.cpp +++ b/modules/dasImgui/src/module_imgui_app.cpp @@ -47,7 +47,7 @@ static void install_imgui_browser_callbacks(GLFWwindow * window) { } #endif -// modules/dasImgui/ARCHITECTURE.md sec.1 +// modules/dasImgui/ARCHITECTURE.md#browser-callback-routing DAS_MOD_API bool das_imgui_init_glfw_for_opengl ( GLFWwindow * window, bool install_callbacks ) { const bool ok = ImGui_ImplGlfw_InitForOpenGL(window, install_callbacks); #ifdef __EMSCRIPTEN__ @@ -57,7 +57,7 @@ DAS_MOD_API bool das_imgui_init_glfw_for_opengl ( GLFWwindow * window, bool inst return ok; } -// modules/dasImgui/ARCHITECTURE.md sec.1 +// modules/dasImgui/ARCHITECTURE.md#browser-callback-routing DAS_MOD_API void das_imgui_shutdown_glfw() { #if defined(__EMSCRIPTEN__) && !defined(EMSCRIPTEN_USE_PORT_CONTRIB_GLFW3) if (g_emscripten_callback_owner == ImGui::GetCurrentContext()) { @@ -106,7 +106,7 @@ DAS_MOD_API void das_imgui_synth_input_char ( uint32_t cp ) { // Caller toggles only on a state change, honoring ImGui_ImplGlfw's // InstalledCallbacks invariant (Install asserts when already installed, and // vice-versa). Must run on the render/main thread — glfwSet*Callback requires it. -// modules/dasImgui/ARCHITECTURE.md sec.1 +// modules/dasImgui/ARCHITECTURE.md#browser-callback-routing DAS_MOD_API void das_imgui_set_real_input_callbacks ( bool enabled ) { GLFWwindow * w = glfwGetCurrentContext(); if ( !w ) return; diff --git a/modules/dasLLVM/ARCHITECTURE_TARGET_FEATURES.md b/modules/dasLLVM/ARCHITECTURE_TARGET_FEATURES.md index 82474cd6f0..2696b5178d 100644 --- a/modules/dasLLVM/ARCHITECTURE_TARGET_FEATURES.md +++ b/modules/dasLLVM/ARCHITECTURE_TARGET_FEATURES.md @@ -1,8 +1,8 @@ # dasLLVM architecture - target feature truth Companion of `ARCHITECTURE.md` (contract: `../../ARCHITECTURE_COMMON.md`). Sections 4, 6 and 9 moved -here with their numbers - a renumber would orphan every `sec.N` pointer to them - and section 10 -continues the numbering, so the index reads in order. Each section is cited by the code +here with their numbers - rules cite sections by number, and a number is never reused - and +section 10 continues the numbering, so the index reads in order. Each section is cited by the code that embodies it. ## 4. Host CPU feature truth on aarch64 {#aarch64-feature-truth} diff --git a/skills/comment_style_hygiene.md b/skills/comment_style_hygiene.md index 3fc6801da5..df3e3870c2 100644 --- a/skills/comment_style_hygiene.md +++ b/skills/comment_style_hygiene.md @@ -89,9 +89,11 @@ it in a name, a shape, or a test, or documents itself in the module's `.md`. Kep is earned: `//!` and `/** */` docs on a contract a CALLER must know - public API always, a non-public helper when callers inside the module need the contract - `NOLINT` / `clang-format` suppressions, the leading header block, and the bare section pointer -`// /ARCHITECTURE.md sec.N` on the function that implements the section (C++ has no -`[arch]` annotation; the pointer is its citation, and nothing checks it - a renumbered section -orphans every pointer to it). Existing comments answer to the +`// /ARCHITECTURE.md#` on the function that implements the section (C++ has no +`[arch]` annotation, so the pointer is its citation; LINT026 resolves it exactly as a .das +citation - the path is root-relative, walked up from the citing file, so a header under +`include/` reaches the `src/` document that owns its mechanism - and a `sec.N` spelling is a +number nothing checks). Existing comments answer to the deletion test; editing one so it stays true is not a new comment, and the edited comment answers to that test as a whole. diff --git a/skills/mcp_tools.md b/skills/mcp_tools.md index 987d99eb23..44a5352190 100644 --- a/skills/mcp_tools.md +++ b/skills/mcp_tools.md @@ -29,8 +29,8 @@ The daslang MCP server (`utils/mcp/main.das`) exposes compiler diagnostics, prog | `describe_type` | Reading source to understand type fields, methods, and values | | `grep_usage` | Built-in Grep across `.das` files (parse-aware via ast-grep + tree-sitter) | | `outline` | Manually scanning files for function/struct/enum declarations | -| `arch_of` | Opening a file's cited architecture document and counting headings to find the section - returns the section text per `[arch]` citation | -| `arch_sites` | Grepping for a document's anchors to learn which code answers for a section - returns the citation census, grouped by anchor | +| `arch_of` | Opening a file's cited architecture document and hunting for the section - returns one locator per citation, `file:line symbol -> doc:first-last #anchor`, for `.das` annotations and C/C++ `// #` pointers alike | +| `arch_sites` | Grepping for a document's anchors under both citation spellings to learn which code answers for a section - returns the citation census grouped by anchor, each with its section's line range | | `cpp_grep_usage` | Built-in Grep across `.cpp/.h/.hpp/.cc` files (parse-aware via ast-grep + tree-sitter-cpp) | | `cpp_find_symbol` | Searching for C++ symbol DECLARATIONS by name + kind (function/class/struct/enum/union/typedef/namespace/macro) | | `cpp_outline` | Manually scanning C++ files for top-level declarations | @@ -67,7 +67,7 @@ The daslang MCP server (`utils/mcp/main.das`) exposes compiler diagnostics, prog **`with_cpp_source` redirect.** `find_symbol` and `goto_definition` accept an optional `with_cpp_source` boolean. When `true`, results that have a C++ implementation (builtin functions, handled types via `addExtern`/`MAKE_TYPE_FACTORY`) get a resolved C++ source location appended via the lazily-built cpp index. First call costs ~2s (one full scan); subsequent calls cost ~150ms (a git-state staleness signature: `rev-parse HEAD` + filtered `git status` + per-file mtimes + `cpp_search_config.das` mtime). The index rebuilds automatically when relevant `.cpp/.cc/.h/.hpp` files change, when HEAD moves, or when the search config is edited. Default off - opt in when the question is "where is X *actually* implemented", not when just enumerating symbols. -**`[arch]` tools.** `arch_of` (code -> document) and `arch_sites` (document -> code) resolve citations exactly as LINT026 does: the path in `[arch(at=".md#")]` resolves against the CITING file's folder, an anchor is a `{#name}` heading suffix, a match with `//` earlier on its line is not a citation, the document must sit in the citing file's own folder tree (its folder is the file's folder or an ancestor - restate a far mechanism in prose in your own tree's document and cite that), and a failure reports lint's own reason (`no such file` / `no such anchor` / `anchor appears N times` / `malformed citation` / `outside this file's folder tree`). Both read source rather than the AST, so they answer for a file this environment cannot compile. `arch_of` returns the cited section - its heading line through the last line before the next heading of the same or a higher level. `arch_sites` searches the subtree of the folder that owns the document, skipping `_`- and `.`-prefixed names the way lint's reverse pass does. +**`[arch]` tools.** `arch_of` (code -> document) and `arch_sites` (document -> code) resolve citations exactly as LINT026 does. A `.das` citation is `[arch(at=".md#")]`: the path resolves against the CITING file's folder, an anchor is a `{#name}` heading suffix, a match with `//` earlier on its line is not a citation, and the document must sit in the citing file's own folder tree (its folder is the file's folder or an ancestor - restate a far mechanism in prose in your own tree's document and cite that). A C/C++ citation is a `// .md#` pointer inside a `//` comment: its path is root-relative, the root found by walking up from the citing file until the path exists, and the folder-tree rule does not bind it. A failure reports lint's own reason (`no such file` / `no such anchor` / `anchor appears N times` / `malformed citation` / `outside this file's folder tree`). Both read source rather than the AST, so they answer for a file this environment cannot compile. `arch_of` returns one locator per citation - `file:line symbol -> doc:first-last #anchor`, the section's heading line through its last line before the next heading of the same or a higher level - so a `Read` of that range or an `Edit` in it follows. `arch_sites` searches the subtree of the folder that owns the document, skipping `_`- and `.`-prefixed names the way lint's reverse pass does, and gives each anchor's line range. **Live tools.** `live_*` interact with a running `daslang-live` instance via its REST API. `live_launch` starts one if not already running (sets working directory to the script's folder). All live tools accept an optional `port` parameter (default 9090). When a compilation error is active, `live_command` and `live_pause` return HTTP 503 with the error - use `live_reload` to fix. Hitting any unknown endpoint returns JSON help with all endpoints + curl examples. diff --git a/src/ast/ARCHITECTURE.md b/src/ast/ARCHITECTURE.md index 03a9e4efa0..cdb5c47f6a 100644 --- a/src/ast/ARCHITECTURE.md +++ b/src/ast/ARCHITECTURE.md @@ -4,7 +4,7 @@ Contract: `ARCHITECTURE_COMMON.md` (repo root). This document carries the mechan folder that a rule cites. The folder's other subsystems - parsing, inference, simulation - are described in `skills/internal/cpp_codebase_notes.md` (repo-only). -## 1. The module-cache read in `trySerializeProgramModule` (`ast_parse.cpp`) +## 1. The module-cache read in `trySerializeProgramModule` (`ast_parse.cpp`) {#module-cache-read} The front end parses modules in require order. When a module cache is installed (`ModuleFileCache`), each module first tries `trySerializeProgramModule`. The reader stands at @@ -56,7 +56,7 @@ every program it read, das reading them back through `ast_serializer_finalize_us `ast_serializer_setup_usec`, and the decode is the read minus both; `-log-compile-time`'s cache line prints decode against finalize, with the macro simulate named inside finalize. -## 2. The module scan and the descriptor manifest (`dyn_modules.cpp`) +## 2. The module scan and the descriptor manifest (`dyn_modules.cpp`) {#module-scan-manifest} `require_dynamic_modules` walks `/modules/`, then the project root's, then each `-load_module` folder, and for every `.das_module` it finds calls `init_dyn_modules`. A @@ -187,7 +187,7 @@ itself first with `[module] a pending module's artifact exists - loading every d A replayed descriptor's time is its manifest read plus its rows, and the second number is the share the `.shared_module` dlopen and module constructor took. -## 3. A require after the walk (`requireModuleNow`, `ast_parse.cpp`) +## 3. A require after the walk (`requireModuleNow`, `ast_parse.cpp`) {#require-after-walk} `requireModuleNow(requireName, access, logs, policies)` is a `require` issued by code that runs after the prerequisite walk - a macro, a simulate macro, an `[init]` - for a module the walk @@ -234,7 +234,7 @@ the requirer's position, or read mid-record; so a nested walk hides the stream f (`LateRequireEnvScope`), parses its modules from source, once per process, and pushes no record. A module whose parse failed pushes no record either, whatever the walk. -## 4. Program-scoped symbol state (`ast.h`, `ast_export.cpp`, `ast_allocate_stack.cpp`) +## 4. Program-scoped symbol state (`ast.h`, `ast_export.cpp`, `ast_allocate_stack.cpp`) {#program-scoped-symbol-state} Whether a program uses a function or a global, and the slot each holds in that program's context, are decided per program - the symbol-use pass (`markSymbolUse` and its variants, diff --git a/src/ast/ast_module.cpp b/src/ast/ast_module.cpp index 460e394ef3..702829e04d 100644 --- a/src/ast/ast_module.cpp +++ b/src/ast/ast_module.cpp @@ -193,7 +193,7 @@ namespace das { return g_deferredModuleLoader && g_deferredModuleLoader(name) && Module::requireEx(name, false); } - // ARCHITECTURE.md sec.2 - process-wide, like the native paths: a descriptor registers once per process + // src/ast/ARCHITECTURE.md#module-scan-manifest - process-wide, like the native paths: a descriptor registers once per process static das_map> g_moduleGroups; static mutex g_moduleGroupsMutex; @@ -229,7 +229,7 @@ namespace das { string notInitialized; if ( !InitializeDependencies(notInitialized) ) { - // a half-warm tree: a descriptor compiled cold loaded its module, its dependency's replayed row waits (ARCHITECTURE.md sec.2) + // a half-warm tree: a descriptor compiled cold loaded its module, its dependency's replayed row waits (src/ast/ARCHITECTURE.md#module-scan-manifest) load_all_deferred_dynamic_modules(); if ( !InitializeDependencies(notInitialized) ) { DAS_FATAL_ERROR("Unable to initialize some modules:%s\n", notInitialized.c_str()); diff --git a/src/ast/ast_parse.cpp b/src/ast/ast_parse.cpp index 4cdbc7c315..d1c489185a 100644 --- a/src/ast/ast_parse.cpp +++ b/src/ast/ast_parse.cpp @@ -251,7 +251,7 @@ namespace das { continue; } } - // ARCHITECTURE.md sec.2 + // src/ast/ARCHITECTURE.md#module-scan-manifest auto guardAvailable = [&](const string & guard) { if ( guard.empty() ) return true; if ( guard.find('/')!=string::npos ) { @@ -270,7 +270,7 @@ namespace das { return src + 6 < src_end && memcmp(src, "public", 6) == 0; }; if ( isReq && src[0]=='[' ) { - // ARCHITECTURE.md sec.2 + // src/ast/ARCHITECTURE.md#module-scan-manifest src ++; while ( src < src_end && isspaceE(src[0]) ) { // the parser reads tokens, so a newline inside the brackets is nothing src ++; @@ -462,7 +462,7 @@ namespace das { } module = Module::requireEx(mod, allowPromoted, modRec.name, info.fileName); // try native with that name AGAIN (promoted?) if ( !module ) { - // ARCHITECTURE.md sec.2 + // src/ast/ARCHITECTURE.md#module-scan-manifest if ( auto loader = getDeferredModuleLoader(); loader && loader(mod) ) { module = Module::requireEx(mod, allowPromoted, modRec.name, info.fileName); if ( log && module ) { @@ -704,7 +704,7 @@ namespace das { } } - // ARCHITECTURE.md sec.1 + // src/ast/ARCHITECTURE.md#module-cache-read static vector collectRequireNames ( FileInfo * fi, const FileAccessPtr & access ) { vector names; if ( fi ) { @@ -828,7 +828,7 @@ namespace das { } return false; } - // ARCHITECTURE.md sec.1 + // src/ast/ARCHITECTURE.md#module-cache-read { auto currentReq = collectRequireNames(access->getFileInfo(fileName), access); sort(savedReq.begin(), savedReq.end()); @@ -1156,7 +1156,7 @@ namespace das { program->inferPassesUsed = 0; // reset once per module; inferTypesDirty accumulates across all inferTypes legs (incl. restartInfer) program->policies = policies; // before the cache read: the reader compares the record's policies against this compile's - // ARCHITECTURE.md sec.1 + // src/ast/ARCHITECTURE.md#module-cache-read auto & serializer_read = daScriptEnvironment::getBound()->serializer_read; uint64_t macroSim0 = serializer_read ? serializer_read->totMacroTime : 0; uint64_t finalize0 = serializer_read ? serializer_read->totFinalizeTime : 0; @@ -1625,7 +1625,7 @@ namespace das { *serializer_write << get<1>(dep); *serializer_write << get<2>(dep); } - // ARCHITECTURE.md sec.1 + // src/ast/ARCHITECTURE.md#module-cache-read uint32_t reqCount = uint32_t(requireNames.size()); *serializer_write << reqCount; for ( auto & req : requireNames ) { @@ -1914,7 +1914,7 @@ namespace das { } } - // ARCHITECTURE.md sec.3 - what a nested compile rebinds on the environment, put back on every exit; + // src/ast/ARCHITECTURE.md#require-after-walk - what a nested compile rebinds on the environment, put back on every exit; // a walk nested in a parse or a record read also hides the stream for its duration struct LateRequireEnvScope { daScriptEnvironment * env; @@ -1946,7 +1946,7 @@ namespace das { } }; - // ARCHITECTURE.md sec.3 + // src/ast/ARCHITECTURE.md#require-after-walk Module * requireModuleNow ( const string & requireName, const FileAccessPtr & access, TextWriter & logs, CodeOfPolicies policies ) { lock_guard guard(g_requireModuleNowMutex); verifyCodeOfPoliciesStamp(policies); @@ -1963,7 +1963,7 @@ namespace das { return nullptr; } LateRequireEnvScope envScope(env); - // ARCHITECTURE.md sec.3 - a walk nested in a parse or a record read leaves the stream alone + // src/ast/ARCHITECTURE.md#require-after-walk - a walk nested in a parse or a record read leaves the stream alone bool nested = (env->g_Program && env->g_Program->isCompiling) || (env->serializer_read && env->serializer_read->readingRecord > 0); if ( nested ) envScope.hideStream(); diff --git a/src/ast/ast_simulate.cpp b/src/ast/ast_simulate.cpp index d3de2f9bdb..18de99ecd2 100644 --- a/src/ast/ast_simulate.cpp +++ b/src/ast/ast_simulate.cpp @@ -27,7 +27,7 @@ namespace das // fusion function pointers (defined here in main lib, set by fusion lib) void (*g_fusionContextFn) ( Context & context, TextWriter & logs, bool enableFusion ) = nullptr; void (*g_resetFusionEngineFn) ( bool orphan ) = nullptr; - // ARCHITECTURE.md sec.4 + // src/ast/ARCHITECTURE.md#program-scoped-symbol-state static __forceinline int32_t programIndexOf ( const Context & context, const Function * fn ) { return context.thisProgram ? context.thisProgram->indexOf(fn) : -1; } @@ -3642,13 +3642,13 @@ namespace das } thisModule->macroContext = get_context(macroStackSize); thisModule->macroContext->category = das::Bitfield(uint32_t(das::ContextCategory::macro_context)); - thisModule->macroContext->contextMutex = new recursive_mutex; // invoke_in_context locks its target (ARCHITECTURE.md sec.3) + thisModule->macroContext->contextMutex = new recursive_mutex; // invoke_in_context locks its target (src/ast/ARCHITECTURE.md#require-after-walk) auto oldAot = policies.aot; auto oldHeap = policies.persistent_heap; policies.aot = false; policies.persistent_heap = policies.macro_context_persistent_heap; simulate(*thisModule->macroContext, logs); - thisModule->macroContext->thisProgram = nullptr; // the context outlives this program (ARCHITECTURE.md sec.4) + thisModule->macroContext->thisProgram = nullptr; // the context outlives this program (src/ast/ARCHITECTURE.md#program-scoped-symbol-state) policies.aot = oldAot; policies.persistent_heap = oldHeap; isCompilingMacros = false; diff --git a/src/ast/dyn_modules.cpp b/src/ast/dyn_modules.cpp index 2b0a22dccd..2e3a89332c 100644 --- a/src/ast/dyn_modules.cpp +++ b/src/ast/dyn_modules.cpp @@ -37,7 +37,7 @@ static Result run_descriptor(smart_ptr fa, const string & mod_filena ModuleGroup dummyGroup; CodeOfPolicies policies; policies.no_init_check = true; - policies.ignore_shared_modules = true; // ARCHITECTURE.md sec.2 + policies.ignore_shared_modules = true; // src/ast/ARCHITECTURE.md#module-scan-manifest auto program = compileDaScript(mod_filename, fa, tout, dummyGroup, policies); if ( program->failed() ) { for ( auto & err : program->errors ) { @@ -89,7 +89,7 @@ static Result run_descriptor(smart_ptr fa, const string & mod_filena return Result::OK; } -static constexpr const char *MANIFEST_SUFFIX = ".das_module.manifest"; // ARCHITECTURE.md sec.2 +static constexpr const char *MANIFEST_SUFFIX = ".das_module.manifest"; // src/ast/ARCHITECTURE.md#module-scan-manifest static constexpr const char *MANIFEST_HEADER = "das_module_manifest\t4"; static bool g_ignore_manifests = false; @@ -355,7 +355,7 @@ static bool write_manifest(const string & file, uint32_t descSize, uint64_t desc #endif } -// src/ast/ARCHITECTURE.md sec.2 - a replayed manifest with a nameless dm row, kept until the scan ends +// src/ast/ARCHITECTURE.md#module-scan-manifest - a replayed manifest with a nameless dm row, kept until the scan ends struct NamelessRowsManifest { string descriptor, file; uint32_t descSize = 0; @@ -424,7 +424,7 @@ static Result init_dyn_modules(smart_ptr fa, string path, TextWriter for ( auto & row : mr.rows ) { nameless |= row.dynamic && row.c.empty(); } - if ( nameless ) { // src/ast/ARCHITECTURE.md sec.2 - named at the end of the scan once the module registered + if ( nameless ) { // src/ast/ARCHITECTURE.md#module-scan-manifest - named at the end of the scan once the module registered g_nameless_manifests.push_back({mod_filename, manifest, len, stamp, key, mr.deps, mr.rows}); } for ( auto & row : mr.rows ) { @@ -633,7 +633,7 @@ static void move_all_nodes(gc_root & from, gc_root & to) { } } -// ARCHITECTURE.md sec.2 +// src/ast/ARCHITECTURE.md#module-scan-manifest static bool load_deferred_module_for_require(const string & name) { static recursive_mutex loadMutex; lock_guard guard(loadMutex); @@ -735,7 +735,7 @@ bool require_dynamic_modules(FileAccessPtr file_access, // before its dependency — its register_dynamic_module dlopen then fails and is // deferred. Retry the deferred set in fixed-point passes so order stops mattering. retry_pending_dynamic_modules(); - // src/ast/ARCHITECTURE.md sec.2 - still pending with its artifact on disk: an import a deferred row holds + // src/ast/ARCHITECTURE.md#module-scan-manifest - still pending with its artifact on disk: an import a deferred row holds if ( pending_dynamic_module_artifact_present() ) { if ( trace_scan() ) { LOG(LogLevel::info) << "[module] a pending module's artifact exists - loading every deferred module\n"; diff --git a/src/builtin/ARCHITECTURE.md b/src/builtin/ARCHITECTURE.md index 4233b5a852..da65026a42 100644 --- a/src/builtin/ARCHITECTURE.md +++ b/src/builtin/ARCHITECTURE.md @@ -36,7 +36,7 @@ flavor by destination module. The gate (`review_nttp.das`) skips them by `cppNam The gate scans the module registry of its own program, so only modules its `require` list pulls in are covered - the require list is the coverage list. -## 2. The default module-cache path +## 2. The default module-cache path {#default-module-cache-path} `ModuleFileCache::defaultPath` (`module_builtin_ast_serialize.cpp`) returns `.jitted_scripts/module_cache/-.dascache` - relative, so the cache follows the @@ -82,7 +82,7 @@ reads, so a record in use is the newest and a stale variant the oldest. Only the directory is pruned - an explicit `-module-cache ` is the user's - and the limit variable is the one `DAS*` name the record key skips, since it decides nothing about a compile. -## 3. The interpreter's `[extern]` call +## 3. The interpreter's `[extern]` call {#interpreter-extern-call} An `[extern]` function (`module_builtin_dasbind.cpp`) is called from the interpreter through a wrapper `vec4f (*)(void * fn, vec4f * args)`: the node evaluates every argument into a @@ -148,7 +148,7 @@ compile, which a served registrar's program takes with it, while the proxy lives process. A failed bind on the on-demand path is the call's transform error, the diagnostic `apply` would have given. -## 4. A message that crosses the panic jump +## 4. A message that crosses the panic jump {#message-crosses-panic-jump} `Context::throw_error_at` formats into a stack buffer and jumps; without C++ exceptions the jump is a `longjmp`, which unwinds nothing, so a heap-owning local alive at the call - a @@ -162,7 +162,7 @@ refusal (`crash_and_burn` in `module_builtin_dasbind.cpp`, thrown at the first c inline. The same discipline is the `FMT_THROW` stash in `include/daScript/das_config.h`, where the temporary dies when the stash statement ends. -## 5. A spawned child's stdout pipe +## 5. A spawned child's stdout pipe {#spawned-child-stdout-pipe} `spawn_process` (`module_builtin_fio.cpp`) hands the child one pipe for stdout and stderr and never blocks on it: `process_drain` takes what the pipe holds and returns, and the caller decides @@ -174,7 +174,7 @@ when asked for the default - a chatty child under the watchdog's tick moves at m second through one of those, the pipe's 4 KB four drains a second - so the Windows pipe is created at the POSIX capacity, and every platform drains the same bursts. -## 6. What a module-cache record's bytes are +## 6. What a module-cache record's bytes are {#module-cache-record-bytes} `module_builtin_ast_serialize.cpp` writes and reads one record per module. Every table below is per record: it clears in `AstSerializer::clearNodeIds` at the end of each program, so no record @@ -240,7 +240,7 @@ bytes written and `serializeProgram` calls it at the end of every program, so `b the stream at a quiescent point; a storage that buffers ahead settles what it holds in its own `flush()`. -## 7. The serialization profile rail +## 7. The serialization profile rail {#serialization-profile-rail} `DAS_SERIALIZE_PROFILE` (`ast_serializer.h`, 0 unless the build defines it) compiles in the stream's size and time breakdown; off, every `DAS_SER_PROFILE` expands to nothing and the diff --git a/src/builtin/REVIEW.md b/src/builtin/REVIEW.md index 389f11757c..0a9324ebc9 100644 --- a/src/builtin/REVIEW.md +++ b/src/builtin/REVIEW.md @@ -82,10 +82,10 @@ cannot represent is misread on the way in and throws on the way out. - **A diff that changes the function or block a citing comment annotates - a comment naming a - section of an architecture document, this folder's (`// src/builtin/ARCHITECTURE.md sec.N`) - or another's (`// src/ast/ARCHITECTURE.md sec.N`) - updates that section in the same - change.** C++ carries no `[arch]` annotation, so nothing but this rule keeps a cited section - true. + section of an architecture document, this folder's (`// src/builtin/ARCHITECTURE.md#`) + or another's (`// src/ast/ARCHITECTURE.md#`) - updates that section in the same + change.** C++ carries no `[arch]` annotation; LINT026 checks that the pointer resolves, and + nothing but this rule keeps the cited section true. - **In a C++ type das binds through an annotation with `addField` in this folder, a member whose size differs between the standard libraries the repo's targets use diff --git a/src/builtin/module_builtin_ast.cpp b/src/builtin/module_builtin_ast.cpp index 0a304edcee..03d4658e74 100644 --- a/src/builtin/module_builtin_ast.cpp +++ b/src/builtin/module_builtin_ast.cpp @@ -792,7 +792,7 @@ namespace das { ok = true; das::Context ctx; auto env = daScriptEnvironment::getBound(); - ctx.thisProgram = env ? env->g_Program.get() : nullptr; // src/ast/ARCHITECTURE.md sec.4 + ctx.thisProgram = env ? env->g_Program.get() : nullptr; // src/ast/ARCHITECTURE.md#program-scoped-symbol-state auto node = simulateExpression(ctx, expr); ctx.restart(); vec4f result = ctx.evalWithCatch(node); @@ -1259,7 +1259,7 @@ namespace das { return structure->aliases.find(aliasName); } - // a null program is a holder nobody filled, never an answer (src/ast/ARCHITECTURE.md sec.4) + // a null program is a holder nobody filled, never an answer (src/ast/ARCHITECTURE.md#program-scoped-symbol-state) static const Program * symbolStateProgram ( const Program * program, const char * what, Context * context, LineInfoArg * at ) { if ( !program ) context->throw_error_at(at, "%s: null program", what); return program; @@ -1422,7 +1422,7 @@ namespace das { #endif #if !DAS_NO_FILEIO - // src/builtin/ARCHITECTURE.md sec.2 + // src/builtin/ARCHITECTURE.md#default-module-cache-path struct ScriptModuleCache { unique_ptr hidden; ModuleFileCache cache; @@ -1479,7 +1479,7 @@ namespace das { } #endif - // src/ast/ARCHITECTURE.md sec.3 + // src/ast/ARCHITECTURE.md#require-after-walk void rtti_builtin_require_module_now ( char * name, smart_ptr access, const CodeOfPolicies & cop, const TBlock & block, Context * context, LineInfoArg * at ) { if ( !name || !name[0] ) context->throw_error_at(at, "require_module_now: empty module name"); diff --git a/src/builtin/module_builtin_ast_serialize.cpp b/src/builtin/module_builtin_ast_serialize.cpp index 0d72916355..a94251135e 100644 --- a/src/builtin/module_builtin_ast_serialize.cpp +++ b/src/builtin/module_builtin_ast_serialize.cpp @@ -266,7 +266,7 @@ namespace das { return buf; } - // src/builtin/ARCHITECTURE.md sec.7 + // src/builtin/ARCHITECTURE.md#serialization-profile-rail void AstSerializer::profReport ( TextWriter & tw ) const { if ( profNodes.empty() ) return; uint64_t total = 0; @@ -371,7 +371,7 @@ namespace das { //////////////////////////////////////////////////////////////////////////// - // src/builtin/ARCHITECTURE.md sec.6 + // src/builtin/ARCHITECTURE.md#module-cache-record-bytes void AstSerializer::serializeAdaptiveSize32 ( uint32_t & size ) { if ( writing ) { uint8_t enc[5]; @@ -454,7 +454,7 @@ namespace das { return *this; } - // src/builtin/ARCHITECTURE.md sec.6 + // src/builtin/ARCHITECTURE.md#module-cache-record-bytes AstSerializer & AstSerializer::serializeString ( string & str, bool temp ) { dtag(HASH_TAG("string")); DAS_SER_PROFILE(*this, "string"); @@ -655,7 +655,7 @@ namespace das { return ptr->module == thisModule; } - // src/builtin/ARCHITECTURE.md sec.6 + // src/builtin/ARCHITECTURE.md#module-cache-record-bytes void AstSerializer::writeIdentifications ( Function * & func ) { auto & name = writeMangledNames[func]; if ( name.empty() ) name = func->getMangledName(); @@ -932,7 +932,7 @@ namespace das { enum TypeRef : uint32_t { TypeRefNull = 0, TypeRefInline = 1, TypeRefFresh = 2, TypeRefFirstEntry = 3 }; - // src/builtin/ARCHITECTURE.md sec.6 + // src/builtin/ARCHITECTURE.md#module-cache-record-bytes AstSerializer & AstSerializer::operator << ( TypeDeclPtr & type ) { dtag(HASH_TAG("TypeDeclPtr")); DAS_SER_PROFILE(*this, "TypeDeclPtr"); @@ -1098,7 +1098,7 @@ namespace das { enum LineShape : uint8_t { LineSameFile = 0x80, LineOneLine = 0x40, LineDeltaMask = 0x3f, LineDeltaBias = 31, LineDeltaEscape = 63 }; - // src/builtin/ARCHITECTURE.md sec.6 + // src/builtin/ARCHITECTURE.md#module-cache-record-bytes AstSerializer & AstSerializer::operator << ( LineInfo & at ) { dtag(HASH_TAG("LineInfo")); DAS_SER_PROFILE(*this, "LineInfo"); @@ -1150,7 +1150,7 @@ namespace das { return *this; } - // src/builtin/ARCHITECTURE.md sec.6 + // src/builtin/ARCHITECTURE.md#module-cache-record-bytes AstSerializer & AstSerializer::operator << ( FileInfo * & info ) { dtag(HASH_TAG("FileInfo *")); DAS_SER_PROFILE(*this, "FileInfo*"); @@ -1383,7 +1383,7 @@ namespace das { return *this; } - // a read no prerequisite walk precedes meets a deferred C++ module here (src/ast/ARCHITECTURE.md sec.2) + // a read no prerequisite walk precedes meets a deferred C++ module here (src/ast/ARCHITECTURE.md#module-scan-manifest) static Module * requireBuiltinModule ( const string & name ) { auto m = Module::require(name); if ( !m ) { @@ -1392,7 +1392,7 @@ namespace das { return m; } - // src/builtin/ARCHITECTURE.md sec.6 + // src/builtin/ARCHITECTURE.md#module-cache-record-bytes AstSerializer & AstSerializer::operator << ( Module * & module ) { DAS_SER_PROFILE(*this, "Module*"); if ( writing ) { @@ -2518,7 +2518,7 @@ namespace das { ser << expr->typeexpr; } - // src/builtin/ARCHITECTURE.md sec.6 + // src/builtin/ARCHITECTURE.md#module-cache-record-bytes AstSerializer & AstSerializer::operator << ( ExpressionPtr & expr ) { dtag(HASH_TAG("ExpressionPtr")); DAS_SER_PROFILE(*this, "ExpressionPtr"); @@ -3297,7 +3297,7 @@ namespace das { // embedders are built without exception handling, so nothing may escape this // rail: contain dasException here (truncated/corrupt cache streams throw from // the stream readers) and report failure through the failed/failToCompile flags - // src/builtin/ARCHITECTURE.md sec.6 + // src/builtin/ARCHITECTURE.md#module-cache-record-bytes void AstSerializer::serializeProgram ( ProgramPtr program, ModuleGroup & libGroup ) noexcept { try { serializeProgramImpl(program, libGroup); @@ -4052,7 +4052,7 @@ namespace das { const TBlock,const string> & block, Context * context, LineInfoArg * at ) { auto prog = make_smart(); - prog->access = access; // the reader's: a stream carries none (src/ast/ARCHITECTURE.md sec.3) + prog->access = access; // the reader's: a stream carries none (src/ast/ARCHITECTURE.md#require-after-walk) { gc_guard deserialize_gc_scope; // same-version streams can still be truncated/corrupt: the stream readers throw diff --git a/src/builtin/module_builtin_dasbind.cpp b/src/builtin/module_builtin_dasbind.cpp index e5662de261..95271510bd 100644 --- a/src/builtin/module_builtin_dasbind.cpp +++ b/src/builtin/module_builtin_dasbind.cpp @@ -33,7 +33,7 @@ #else #define DAS_ARM64_CALL_SYMBOL "das_arm64_call" #endif -// src/builtin/ARCHITECTURE.md sec.3 +// src/builtin/ARCHITECTURE.md#interpreter-extern-call __asm__( ".text\n" ".p2align 2\n" @@ -175,7 +175,7 @@ FastCallWrapper getExtraWrapper ( int nargs, int res, int perm ) { #undef AX #undef AD - // src/builtin/ARCHITECTURE.md sec.3 + // src/builtin/ARCHITECTURE.md#interpreter-extern-call struct Arm64Layout { struct Slot { uint8_t argIndex, bytes; uint16_t offset; }; uint8_t gprArg[8] = {}, fprArg[8] = {}; @@ -411,7 +411,7 @@ FastCallWrapper getExtraWrapper ( int nargs, int res, int perm ) { } }); if ( !crash_and_burn.empty() ) { - char message[8192]; // src/builtin/ARCHITECTURE.md sec.4 + char message[8192]; // src/builtin/ARCHITECTURE.md#message-crosses-panic-jump snprintf(message, sizeof(message), "%s", crash_and_burn.c_str()); string().swap(crash_and_burn); context.throw_error_at(debugInfo, "%s", message); diff --git a/src/builtin/module_builtin_debugger.cpp b/src/builtin/module_builtin_debugger.cpp index d8415f051d..68cda06e41 100644 --- a/src/builtin/module_builtin_debugger.cpp +++ b/src/builtin/module_builtin_debugger.cpp @@ -1140,7 +1140,7 @@ namespace debugger { // pinvoke(context,"function",....) - // src/builtin/ARCHITECTURE.md sec.4 + // src/builtin/ARCHITECTURE.md#message-crosses-panic-jump DAS_NORETURN_PREFIX static void throw_pinvoke_error ( Context & context, const LineInfo & at, string & text ) DAS_NORETURN_SUFFIX; static void throw_pinvoke_error ( Context & context, const LineInfo & at, string & text ) { char message[8192]; diff --git a/src/builtin/module_builtin_fio.cpp b/src/builtin/module_builtin_fio.cpp index 75b3014325..b549c1d8c6 100644 --- a/src/builtin/module_builtin_fio.cpp +++ b/src/builtin/module_builtin_fio.cpp @@ -1865,7 +1865,7 @@ namespace das { sa.bInheritHandle = TRUE; sa.lpSecurityDescriptor = NULL; HANDLE hRead = NULL, hWrite = NULL; - const DWORD spawnPipeBytes = 64 * 1024; // src/builtin/ARCHITECTURE.md sec.5 + const DWORD spawnPipeBytes = 64 * 1024; // src/builtin/ARCHITECTURE.md#spawned-child-stdout-pipe if ( !CreatePipe(&hRead, &hWrite, &sa, spawnPipeBytes) ) { context->throw_error_at(at, "spawn_process: CreatePipe failed"); return nullptr; @@ -2350,7 +2350,7 @@ namespace das { // after the folder scan, so module enumeration order stops mattering. static vector> g_pending_dynamic_modules; // path, cpp_class_name, last dlopen error - // the descriptor manifest recorder (dyn_modules.h, src/ast/ARCHITECTURE.md sec.2) + // the descriptor manifest recorder (dyn_modules.h, src/ast/ARCHITECTURE.md#module-scan-manifest) static thread_local bool g_manifest_recording = false; static thread_local bool g_manifest_opt_out = false; static thread_local vector g_manifest_rows; @@ -2503,7 +2503,7 @@ namespace das { string path, cpp_class, das_name; int on_error = 0; }; - static vector g_deferred_dynamic_modules; // src/ast/ARCHITECTURE.md sec.2 + static vector g_deferred_dynamic_modules; // src/ast/ARCHITECTURE.md#module-scan-manifest static std::recursive_mutex g_deferred_dynamic_modules_mutex; // a run-time has_module reads while another thread's compile loads DAS_API void defer_dynamic_module ( const char * path, const char * cpp_class, int on_error, const char * das_name ) { @@ -2573,7 +2573,7 @@ namespace das { } } - // src/ast/ARCHITECTURE.md sec.2 - a pending row whose artifact is on disk failed on something + // src/ast/ARCHITECTURE.md#module-scan-manifest - a pending row whose artifact is on disk failed on something // other than a missing file: on a platform that resolves imports at load, a sibling the // deferred set still holds DAS_API bool pending_dynamic_module_artifact_present() { diff --git a/src/builtin/module_builtin_rtti.cpp b/src/builtin/module_builtin_rtti.cpp index 4e258db382..0052a6a3b4 100644 --- a/src/builtin/module_builtin_rtti.cpp +++ b/src/builtin/module_builtin_rtti.cpp @@ -1261,7 +1261,7 @@ namespace das { return Module::require(name) != nullptr || is_dynamic_module_deferred(name); } - // a member's guard answers as the require's would (src/ast/ARCHITECTURE.md sec.2) + // a member's guard answers as the require's would (src/ast/ARCHITECTURE.md#module-scan-manifest) static bool moduleGroupMemberAvailable ( const ModuleGroupMember & m ) { if ( m.guard.empty() ) return true; if ( m.guard.find('/') == string::npos ) return guardModuleAvailable(m.guard); diff --git a/src/misc/ARCHITECTURE.md b/src/misc/ARCHITECTURE.md index d73fb1e661..5174ef59a7 100644 --- a/src/misc/ARCHITECTURE.md +++ b/src/misc/ARCHITECTURE.md @@ -110,7 +110,7 @@ one tier the POSIX arm does not: it distrusts a symbol whose offset exceeds `kMaxTrustedSymbolOffset`, because `SymFromAddr` answers with a distant neighbour where `dladdr` answers with nothing. -## 8. The spin window reads the clock once per stride, for both of the things it needs it for +## 8. The spin window reads the clock once per stride, for both of the things it needs it for {#spin-window-clock-stride} A worker in the spin-before-park window (`JobQue::job`, opt-in via `setWorkerSpin`) wants the clock twice: to extend the window when it served a team chunk, and to end the window when it diff --git a/src/misc/job_que.cpp b/src/misc/job_que.cpp index 689e7c8b9f..6c900e6465 100644 --- a/src/misc/job_que.cpp +++ b/src/misc/job_que.cpp @@ -38,7 +38,7 @@ namespace das { } // One short burst of the CPU's spin-wait hint (worker spin-before-park, see JobQue::job). - // src/misc/ARCHITECTURE.md sec.8 + // src/misc/ARCHITECTURE.md#spin-window-clock-stride static inline void jobque_spin_pause() { #if defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) for ( int i = 0; i != 64; ++i ) _mm_pause(); @@ -613,7 +613,7 @@ namespace das { // Team mode extends the same window: poll the team slot too. The window stays BOUNDED // (the ggml hybrid poll/park shape) — after it expires the worker parks, and a team // publish that finds parked workers notifies (see teamParallelFor's wake gate). - // src/misc/ARCHITECTURE.md sec.8 + // src/misc/ARCHITECTURE.md#spin-window-clock-stride int spinUs = mSpinUs.load(std::memory_order_relaxed); bool teamMode = mTeamMode.load(std::memory_order_relaxed) != 0; if ( (spinUs > 0 || teamMode) && !mShutdown.load(std::memory_order_relaxed) ) { diff --git a/src/parser/parser_impl.cpp b/src/parser/parser_impl.cpp index 4deb4fc739..d027fd7c92 100644 --- a/src/parser/parser_impl.cpp +++ b/src/parser/parser_impl.cpp @@ -1237,7 +1237,7 @@ namespace das { } } - // src/ast/ARCHITECTURE.md sec.2 + // src/ast/ARCHITECTURE.md#module-scan-manifest static bool ast_requireGuardAvailable ( yyscan_t scanner, const string & guard ) { if ( guard.empty() ) return true; if ( guard.find('/') != string::npos ) { @@ -1264,7 +1264,7 @@ namespace das { if ( modalias ) delete modalias; } - // src/ast/ARCHITECTURE.md sec.2 + // src/ast/ARCHITECTURE.md#module-scan-manifest void ast_requireModuleGroup ( yyscan_t scanner, string * group, bool pub, const LineInfo & atName, string * guard ) { if ( ast_requireGuardAvailable(scanner, guard) ) { for ( const auto & member : getModuleGroupMembers(*group) ) { @@ -1279,7 +1279,7 @@ namespace das { auto info = yyextra->g_Access->getModuleInfo(name, yyextra->g_FileAccessStack.back()->name); auto mod = yyextra->g_Program->addModule(info.moduleName); if ( !mod ) { - // a parse with no prerequisite walk (compile of a string) meets a deferred module here (src/ast/ARCHITECTURE.md sec.2) + // a parse with no prerequisite walk (compile of a string) meets a deferred module here (src/ast/ARCHITECTURE.md#module-scan-manifest) if ( auto loader = getDeferredModuleLoader(); loader && loader(info.moduleName) ) { mod = yyextra->g_Program->addModule(info.moduleName); } diff --git a/tests/strudel_device/test_worker_heap.das b/tests/strudel_device/test_worker_heap.das index 747841ace0..e7c1406735 100644 --- a/tests/strudel_device/test_worker_heap.das +++ b/tests/strudel_device/test_worker_heap.das @@ -12,10 +12,22 @@ require audio/audio_boost require daslib/fio // The playback worker's context is its own: nothing else collects it, and a bed that queries -// four patterns every tick leaves megabytes of garbage a second behind. The scheduler's pools -// grow for the first few seconds and then hold, so the arm samples after that warm-up and pins -// that the heap stays within one collect window from there, over a stretch that would have -// grown many windows' worth uncollected. Null backend, so it runs headless. +// four patterns every tick leaves megabytes of garbage a second behind. The number the worker +// publishes is its heap BEFORE the tick's collect, so it reads live set plus that tick's garbage +// - a tick's garbage runs from hundreds of kilobytes to ten megabytes, so the number jitters +// by that much between consecutive ticks even on an idle box, and a bar on its spread measures +// the burstiness of a query, not collection. Two things do measure collection, and each is +// zero for a worker that never collects: a published number LOWER than the one before it, +// which only a collect can produce (test_worker_no_gc.das pins the converse), and a floor - +// the least number over a stretch - that holds where an uncollected heap climbs a window's +// worth a second. Warm-up and the two stretches are measured on the worker's own clock, its +// pattern time, so a slow box warms up before it is measured. Null backend, so it runs headless. + +let WARM_UP_SECONDS = 6.0lf +let STRETCH_SECONDS = 3.0lf +let MIN_DROPS = 6 // an idle 64-core box reads ~28 per two stretches +let FLOOR_DRIFT_LIMIT = 4ul * 1024ul * 1024ul // twice the worker's collect window; uncollected grows ~10 MB a stretch +let WALL_CAP_MS = 90000u // a worker that never advances is its own failure def worker_setup() { strudel_set_cps(2.0lf) @@ -26,26 +38,66 @@ def worker_setup() { strudel_play() } +def worker_seconds() : double { + return strudel_get_playback_time()._0 +} + +// Sleeps until the worker's clock reaches `until`, or the wall cap passes; false on the cap. +def wait_worker_until(until : double; var wall_ms : uint&) : bool { + while (worker_seconds() < until) { + return false if (wall_ms >= WALL_CAP_MS) + sleep(20u) + wall_ms += 20u + } + return true +} + +struct Stretch { + samples : int // distinct published numbers seen; a stretch with none measured nothing + lo : uint64 + drops : int +} + +// The least published heap over one stretch of worker time, and how many times a tick published +// less than the tick before it; `prev` carries the last number across stretches. +def sample_stretch(until : double; var prev : uint64&; var wall_ms : uint&) : Stretch { + var st = Stretch(lo = 0xFFFFFFFFFFFFFFFFul) + while (worker_seconds() < until && wall_ms < WALL_CAP_MS) { + sleep(20u) + wall_ms += 20u + let h = strudel_get_worker_heap_bytes() + continue if (h == prev) // the same tick as last time + if (prev != 0ul && h < prev) { st.drops++ } + prev = h + st.lo = min(st.lo, h) + st.samples++ + } + return st +} + [test] def test_worker_heap_stays_flat(t : T?) { - t |> run("the playback worker collects its own heap: a four-track bed holds within a few collect windows past its warm-up") @(t : T?) { + t |> run("the playback worker collects its own heap: past warm-up the published heap drops between ticks and its floor holds") @(t : T?) { sound_set_null_device(true) with_audio_system() { strudel_init(@@worker_setup) - sleep(6000u) - let base = strudel_get_worker_heap_bytes() - t |> success(base > 0ul, "the worker publishes its heap") - var peak = base - for (_i in range(12)) { - sleep(500u) - peak = max(peak, strudel_get_worker_heap_bytes()) + var wall_ms = 0u + let warm = wait_worker_until(WARM_UP_SECONDS, wall_ms) + t |> success(warm, "the worker reached {WARM_UP_SECONDS} s of pattern time inside {WALL_CAP_MS} ms of wall time") + if (warm) { + var prev = 0ul + let w0 = worker_seconds() + let early = sample_stretch(w0 + STRETCH_SECONDS, prev, wall_ms) + let late = sample_stretch(w0 + 2.0lf * STRETCH_SECONDS, prev, wall_ms) + // a stretch that saw no published number leaves `lo` at its sentinel, which would + // pass a bare `lo > 0`; the sample counts are what say the worker published at all + t |> success(early.samples > 0 && late.samples > 0, "the worker publishes its heap: {early.samples} then {late.samples} distinct numbers") + let drops = early.drops + late.drops + t |> success(drops >= MIN_DROPS, "the heap dropped between ticks {drops} times over {2.0lf * STRETCH_SECONDS} s of pattern time - only a collect lowers it") + let drift = int64(late.lo) - int64(early.lo) + t |> success(drift < int64(FLOOR_DRIFT_LIMIT), "the floor moved {drift / 1024l} KB between the stretches (early {int64(early.lo) / 1024l} KB, late {int64(late.lo) / 1024l} KB)") } strudel_shutdown() - // the published number is the heap before the tick's collect, so a peak sits one collect - // window plus one tick's garbage above a base sampled at a trough - and a starved CI tick - // leaves five megabytes, where an uncollected worker grows twenty in the same six seconds - let LIMIT = 10ul * 1024ul * 1024ul - t |> success(peak < base + LIMIT, "the worker heap grew {int64(peak - base) / 1024l} KB in the six seconds past warm-up") } sound_set_null_device(false) // sticky process-wide flag: leave a clean slate } diff --git a/utils/common/arch_citations.das b/utils/common/arch_citations.das index 7cab23d410..09484e1d5c 100644 --- a/utils/common/arch_citations.das +++ b/utils/common/arch_citations.das @@ -1,12 +1,14 @@ // The `[arch]` citation scanner: one implementation of LINT026's resolution semantics, shared by -// the rule that enforces them (`utils/lint/main.das`) and the tools that read them -// (`utils/mcp/tools/arch_of`, `utils/mcp/tools/arch_sites`). The failure strings live here too, -// so a citation lint reports and a citation a tool refuses cannot describe themselves -// differently. +// the rule that enforces them (`utils/lint/main.das`), the install-time excerpt writer +// (arch-extract) and the tools that read them (`utils/mcp/tools/arch_of`, +// `utils/mcp/tools/arch_sites`). The failure strings live here too, so a citation lint reports +// and a citation a tool refuses cannot describe themselves differently. // -// Everything here reads SOURCE rather than the AST: the annotation is an inert builtin marker, so -// citing a document costs a build nothing, and a citation must be checkable in a file the reading -// environment cannot compile. +// Two spellings, one resolution: `[arch(at="#")]` on a .das declaration, and +// `// #` inside a `//` comment of a C or C++ source, which has no annotation to +// carry it. Everything here reads SOURCE rather than the AST: the annotation is an inert builtin +// marker, so citing a document costs a build nothing, and a citation must be checkable in a file +// the reading environment cannot compile. options gen2 @@ -102,9 +104,103 @@ def private positions_of_offsets(text : string; offsets : array) : array#` pointer form. +def public is_cpp_source(name : string) : bool { + for (s in CPP_SOURCE_SUFFIXES) { + return true if (name |> ends_with(s)) + } + return false +} + + +// True for a file an [arch] walk reads for citations: a .das, or a C/C++ source. +def public arch_cites_source(name : string) : bool { + return name |> ends_with(".das") || is_cpp_source(name) +} + + +def private is_path_char(c : int) : bool { + return is_alpha(c) || is_number(c) || c == '_' || c == '.' || c == '/' || c == '-' +} + + +def private is_anchor_char(c : int) : bool { + return is_alpha(c) || is_number(c) || c == '_' || c == '-' +} + + +// A C++ pointer's document is spelled root-relative (`src/ast/ARCHITECTURE.md`), and the root is +// the first folder, from the citing file's own upward, that the path exists under: a header in +// include/ reaches src/'s document, a bare `ARCHITECTURE.md` still means the citer's own folder, +// and a planted fixture tree resolves inside itself. A path found nowhere resolves at the +// citer's folder, where the forward check reports it missing. +def private resolve_pointer_doc(folder, rel : string) : string { + var dir = arch_path(folder) + while (true) { + let candidate = arch_path(path_join(dir, rel)) + let st = stat(candidate) + return candidate if (st.is_valid && st.is_reg) + let up = dir_name(dir) + break if (empty(up) || up == dir) + dir = up + } + return arch_path(path_join(folder, rel)) +} + + +// Every pointer a C or C++ source makes, in file order: `.md#` inside a `//` +// comment, the path the longest run of path characters before `.md`, the anchor the run of +// anchor characters after `#`. The `//` test inverts the .das rule's: here only text INSIDE the +// comment is a citation. A block comment is not read. +def public collect_arch_pointers(file, text : string) : array { + var out : array + let folder = dir_name(file) + var lineno = 0 + for (line in split(text, "\n")) { + lineno ++ + let slashes = find(line, "//") + continue if (slashes < 0) + peek_data(line) $(d) { + let len = length(d) + var from = slashes + 2 + while (true) { + let ext = find(line, ".md#", from) + break if (ext < 0) + let hash = ext + 3 + var start = ext + while (start > from && is_path_char(int(d[start - 1]))) { + start -- + } + var stop = hash + 1 + while (stop < len && is_anchor_char(int(d[stop]))) { + stop ++ + } + from = stop + // `.md#` with no name after it is prose; a URL to a document on the web is not a + // pointer either: the walk back through `https://host/...` stops at the colon and + // leaves a path opening with `/`, and a bare `host.tld/...` opens with a dotted + // first segment - a tree path's first segment carries no dot and never a leading slash + continue if (stop == hash + 1 || int(d[start]) == '/') + let first_slash = find(line, "/", start) + continue if (first_slash > start && first_slash < ext && find(line, ".", start) < first_slash) + out |> push(ArchCitation(line = lineno, raw = slice(line, start, stop), + anchor = slice(line, hash + 1, stop), + doc = resolve_pointer_doc(folder, slice(line, start, hash)))) + } + } + } + return <- out +} + + +// Every citation `text` makes, in file order, in whichever spelling `file`'s language carries. +// `file` is only read for its extension and the folder a relative citation resolves against, so +// it may name a file that is no longer on disk. def public collect_arch_citations(file, text : string) : array { + return <- collect_arch_pointers(file, text) if (is_cpp_source(file)) var out : array var raws : array var offsets : array @@ -302,6 +398,9 @@ def public citation_outside_subtree(file : string; c : ArchCitation) : bool { def public citation_failure_reason(var docs : ArchDocs; file : string; c : ArchCitation) : string { let reason = citation_failure_reason(docs, c) return reason if (!empty(reason)) + // a C++ pointer is exempt from the tree rule: the layout puts a header under include/ far + // from the src/ folder whose document owns the mechanism it implements + return "" if (is_cpp_source(file)) return "cites a document outside this file's folder tree - restate the mechanism in an architecture doc of this folder's tree and cite that" if (citation_outside_subtree(file, c)) return "" } diff --git a/utils/internal/arch-extract/main.das b/utils/internal/arch-extract/main.das index 03cf9d1afe..0ed7146874 100644 --- a/utils/internal/arch-extract/main.das +++ b/utils/internal/arch-extract/main.das @@ -50,7 +50,7 @@ def private scan_das_files(path : string; var out : array) { return if (!st.is_valid) if (st.is_dir) { scan_das_files(full, out) - } elif (st.is_reg && name |> ends_with(".das")) { + } elif (st.is_reg && arch_cites_source(name)) { out |> push(full) } } @@ -76,7 +76,7 @@ def private collect_cited(files : array; var citedDocs, citedAnchors : t var failures = 0 for (f in files) { let text = fread(f) - continue if (empty(text) || find(text, "arch") < 0) + continue if (empty(text) || (find(text, "arch") < 0 && find(text, ".md#") < 0)) for (c in collect_arch_citations(f, text)) { let reason = citation_failure_reason(docs, f, c) if (!empty(reason)) { diff --git a/utils/internal/preflight/main.das b/utils/internal/preflight/main.das index b40af883ea..914511e389 100644 --- a/utils/internal/preflight/main.das +++ b/utils/internal/preflight/main.das @@ -112,7 +112,8 @@ struct PreflightCtx { base : string changed_das : array changed_md : array // the lint gate reads these beside changed_das, as CI does - changed_cpp : array + changed_cpp : array // the translation units the cpp-syntax gate compiles + changed_cxx_other : array // every other changed C-family file - headers, .c, .cxx, .mm - the lint gate's [arch] pass reads; `changed_hdr` counts the core-scope headers changed_all : array // every changed path vs base, any extension, plus untracked files - the reach tests read this changed_hdr : int jobs : int @@ -322,7 +323,7 @@ def in_core_scope(path : string) : bool { return starts_with(g, "src/") || starts_with(g, "include/") || starts_with(g, "tests-cpp/") } -def collect_changed_files(base : string; var das_files, md_files, cpp_files : array&; var hdr_count : int&) { +def collect_changed_files(base : string; var das_files, md_files, cpp_files, cxx_other : array&; var hdr_count : int&) { var seen : table let probe = run_argv(["git", "rev-parse", "--verify", "--quiet", base]) var ranges : array @@ -346,11 +347,12 @@ def collect_changed_files(base : string; var das_files, md_files, cpp_files : ar md_files |> push(f) } elif (ext == ".cpp" || ext == ".cc") { cpp_files |> push(f) - } elif (ext == ".h" || ext == ".hpp" || ext == ".inc") { - // core-scope only — the sole consumer is the cpp-syntax + } elif (ext == ".c" || ext == ".cxx" || ext == ".mm" || ext == ".h" || ext == ".hpp" || ext == ".inc" || ext == ".inl") { + cxx_other |> push(f) + // the COUNT is core-scope headers only — its consumer is the cpp-syntax // full-sweep trigger, and module headers don't ripple into // core TUs (modules depend on core, not the other way) - if (in_core_scope(f)) { + if ((ext == ".h" || ext == ".hpp" || ext == ".inc") && in_core_scope(f)) { hdr_count ++ } } @@ -598,12 +600,13 @@ def gate_format(ctx : PreflightCtx) : GateResult { // outside the PR — staleness belongs to the nightly whole-tree -j 1 pass. def gate_lint(ctx : PreflightCtx) : GateResult { let t0 = ref_time_ticks() - if (empty(ctx.changed_das) && empty(ctx.changed_md)) { + if (empty(ctx.changed_das) && empty(ctx.changed_md) && empty(ctx.changed_cpp) && empty(ctx.changed_cxx_other)) { return GateResult(name = "lint", status = GateStatus.Skip, seconds = seconds_since(t0), - detail = "no .das or .md files changed vs {ctx.base}") + detail = "no .das, .md or C/C++ files changed vs {ctx.base}") } var files <- [for (f in ctx.changed_das); f] - files |> push_from(ctx.changed_md) + // a C/C++ positional compiles nothing; it arms its folder's [arch] pass, whose pointers it may carry + files |> push_from(ctx.changed_md, ctx.changed_cpp, ctx.changed_cxx_other) var host_args <- [ctx.daslang, "utils/lint/main.das", "--"] host_args |> push_from(files) host_args |> push("--quiet") @@ -1782,7 +1785,7 @@ def main() : int { // nolint:STYLE037,STYLE038 — the gate loop + CLI surface; let clang_info = find_clang(cfg.clang, ctx.build_dir) ctx.clang = clang_info.exe ctx.clang_is_cl = clang_info.is_cl - collect_changed_files(cfg.base, ctx.changed_das, ctx.changed_md, ctx.changed_cpp, ctx.changed_hdr) + collect_changed_files(cfg.base, ctx.changed_das, ctx.changed_md, ctx.changed_cpp, ctx.changed_cxx_other, ctx.changed_hdr) ctx.changed_all <- collect_changed_paths(cfg.base) let tier = cfg.full ? "full" : "fast" diff --git a/utils/lint/main.das b/utils/lint/main.das index 3692c6af3b..8b64585ffa 100644 --- a/utils/lint/main.das +++ b/utils/lint/main.das @@ -318,7 +318,8 @@ def private report_arch_citation(file : string; c : ArchCitation; var docs : Arc } // Both halves of the citation check in one walk: the .md files that declare anchors and the -// .das files that may cite them. Same pruning as scan_das_files; `seen` folds nested roots. +// .das and C/C++ sources that may cite them. Same pruning as scan_das_files; `seen` folds +// nested roots. def private scan_arch_folder(path : string; var mds, dases : array; var seen : table) { return if (seen |> key_exists(path)) seen |> insert(path, null) @@ -332,7 +333,7 @@ def private scan_arch_folder(path : string; var mds, dases : array; var scan_arch_folder(full, mds, dases, seen) } elif (fst.is_reg && name |> ends_with(".md")) { mds |> push(full) - } elif (fst.is_reg && name |> ends_with(".das") && !is_skip_file(name)) { + } elif (fst.is_reg && arch_cites_source(name) && !is_skip_file(name)) { dases |> push(full) } } @@ -358,7 +359,7 @@ def run_arch_pass(roots : array) : int { var cited : table for (f in dases) { let text = fread(f) - continue if (empty(text) || find(text, "arch") < 0) + continue if (empty(text) || (find(text, "arch") < 0 && find(text, ".md#") < 0)) for (c in collect_arch_citations(f, text)) { found += report_arch_citation(f, c, docs) continue if (empty(c.anchor)) diff --git a/utils/lint/tests/lint026_arch_citations.das b/utils/lint/tests/lint026_arch_citations.das index a3baa60c6f..8974ced207 100644 --- a/utils/lint/tests/lint026_arch_citations.das +++ b/utils/lint/tests/lint026_arch_citations.das @@ -345,6 +345,77 @@ def test_arch_reverse(t : T?) { } +// A C++ source carrying `pointers`, one per line inside a `//` comment, plus an `#include`, a +// block comment and two web URLs that must read as nothing. Pointer N lands on line 3N + 3. +def private plant_cpp(dir, name : string; pointers : array) { + plant(dir, name, build_string() $(var w) { + w |> write("#include \"ARCHITECTURE.md#not-a-pointer.h\"\n") + w |> write("/* ARCHITECTURE.md#alpha inside a block comment is not read */\n") + w |> write("// see https://github.com/example/repo/blob/main/docs/Usage.md#the-web-not-the-tree\n") + w |> write("// and github.com/example/repo/blob/main/docs/FAQ.md#a-host-first-segment\n") + for (p in pointers) { + w |> write("\n// {p} - the mechanism this function implements\nvoid f() \{\}\n") + } + }) +} + + +[test] +def test_arch_cpp_pointers(t : T?) { + t |> run("a C++ `// #` pointer is a citation: it satisfies the anchor, and a dangling one fires at its line") @(t : T?) { + let dir = path_join(tmp_root(), "cpp") + mkdir_rec(dir) + plant_doc(dir, "ARCHITECTURE.md", ["alpha", "orphan"]) + plant_cpp(dir, "ok.cpp", ["ARCHITECTURE.md#alpha", "ARCHITECTURE.md#nope"]) + var out : string + let rc = run_lint(dir, out) + t |> equal(count_of(out, "\{#alpha\}"), 0, "the pointer counts as alpha's citer\n{out}") + t |> equal(count_of(out, "\{#orphan\}"), 1, "the anchor nothing points at is still owed\n{out}") + t |> success(find(out, "ok.cpp:9: LINT026: [arch] citation does not resolve - no such anchor") >= 0, + "the dangling pointer is named at its own line\n{out}") + t |> equal(count_of(out, "not-a-pointer"), 0, "an #include is not a comment, so not a pointer\n{out}") + t |> equal(count_of(out, "the-web-not-the-tree"), 0, "a URL with a scheme is a document on the web, not a pointer\n{out}") + t |> equal(count_of(out, "a-host-first-segment"), 0, "a bare host.tld/... path is a document on the web, not a pointer\n{out}") + t |> equal(count_of(out, "LINT026"), 2, "the block-comment mention reads as nothing\n{out}") + t |> equal(rc, 2, "a finding sets the issue exit code\n{out}") + } + t |> run("a pointer's path is root-relative, walked up from the citing file, and the folder-tree rule does not bind C++") @(t : T?) { + let root = path_join(tmp_root(), "cpproot") + let inc = path_join(path_join(root, "include"), "deep") + let src = path_join(root, "src") + mkdir_rec(inc) + mkdir_rec(src) + plant_doc(src, "ARCHITECTURE.md", ["far"]) + plant_cpp(inc, "deep.h", ["src/ARCHITECTURE.md#far"]) + var out : string + let rc = run_lint(root, out) + t |> equal(count_of(out, "LINT026"), 0, "include/deep/deep.h reaches src/ARCHITECTURE.md through the root above both\n{out}") + t |> equal(rc, 0, "and the run is clean\n{out}") + } + t |> run("negative control - the pointer follows its anchor") @(t : T?) { + let dir = path_join(tmp_root(), "cppctl") + mkdir_rec(dir) + plant_doc(dir, "ARCHITECTURE.md", ["alpha"]) + plant_cpp(dir, "ok.cpp", ["ARCHITECTURE.md#alpha"]) + var before : string + let rc = run_lint(dir, before) + t |> equal(count_of(before, "LINT026"), 0, "a resolving pointer is silent both ways\n{before}") + t |> equal(rc, 0, "and the run is clean\n{before}") + // mutate: the pointer names a section the document does not declare + plant_cpp(dir, "ok.cpp", ["ARCHITECTURE.md#gone"]) + var swapped : string + run_lint(dir, swapped) + t |> equal(count_of(swapped, "does not resolve"), 1, "the dangling pointer fires\n{swapped}") + t |> equal(count_of(swapped, "\{#alpha\}"), 1, "and the anchor it left is owed\n{swapped}") + // restore + plant_cpp(dir, "ok.cpp", ["ARCHITECTURE.md#alpha"]) + var again : string + run_lint(dir, again) + t |> equal(count_of(again, "LINT026"), 0, "restoring the pointer restores silence\n{again}") + } +} + + [test] def test_arch_modes(t : T?) { t |> run("--disable LINT026 silences the forward direction") @(t : T?) { diff --git a/utils/mcp/registry_das.das b/utils/mcp/registry_das.das index f9f49167ea..a7671f35f9 100644 --- a/utils/mcp/registry_das.das +++ b/utils/mcp/registry_das.das @@ -405,9 +405,9 @@ def build_das_tools(var reg : array) { // nolint:STYLE038 — flat tool reg |> emplace(ToolDef( tool <- make_tool( "arch_of", - "Read direction of the [arch] annotation: the architecture-document citations a .das file makes, each resolved to its document and the SECTION TEXT under the cited anchor heading (heading line through the last line before the next heading of the same or a higher level). A citation that does not resolve reports LINT026's own reason ('no such file', 'no such anchor', 'anchor appears N times', 'malformed citation'). Reads source, not the AST — works on a file that does not compile.", + "Read direction of the [arch] annotation: WHERE each citation a source file makes lands - one line per citation, `: -> :- #`, the section's heading line through its last line, so a Read of that range or an Edit in it follows directly. A .das file is read for its [arch(at=...)] annotations, a C/C++ source for its `// #` pointers. A citation that does not resolve reports LINT026's own reason ('no such file', 'no such anchor', 'anchor appears N times', 'malformed citation'). Reads source, not the AST - works on a file that does not compile.", { - "file" => PropertySchema(_type = "string", description = "Path to the .das file (repo-relative or absolute)"), + "file" => PropertySchema(_type = "string", description = "Path to the .das file or C/C++ source - .c, .cpp, .cc, .cxx, .mm, .h, .hpp, .inl, .inc - repo-relative or absolute"), "symbol" => PropertySchema(_type = "string", description = "Optional function/struct/class name — show only the citations annotating it") }, ["file"]), @@ -417,7 +417,7 @@ def build_das_tools(var reg : array) { // nolint:STYLE038 — flat tool reg |> emplace(ToolDef( tool <- make_tool( "arch_sites", - "Write direction of the [arch] annotation: the census of an architecture document — every .das citing it, with file, line, and the annotated symbol. Searched from the folder that OWNS the document (its own subtree), the scope LINT026's reverse pass walks. Without an anchor, the census is grouped under the document's own headings in document order, uncited anchors included; with one, only that section's citations are listed.", + "Write direction of the [arch] annotation: the census of an architecture document - every source citing it, .das annotations and C/C++ `// #` pointers alike, with file, line and the annotated symbol, each anchor with its section's line range. Searched from the folder that OWNS the document (its own subtree), the scope LINT026's reverse pass walks. Without an anchor, the census is grouped under the document's own headings in document order, uncited anchors and citations naming no declared anchor included; with one, only that section's citations are listed.", { "doc" => PropertySchema(_type = "string", description = "Path to the .md document, optionally with an anchor: 'utils/lint/ARCHITECTURE.md' or 'utils/lint/ARCHITECTURE.md#citations'") }, diff --git a/utils/mcp/test_tools.das b/utils/mcp/test_tools.das index 4951ea3fc5..a2c1c4d6a0 100644 --- a/utils/mcp/test_tools.das +++ b/utils/mcp/test_tools.das @@ -3606,14 +3606,13 @@ def test_arch_of(t : T?) { let ok = parse_result(do_arch_of(path_join(dir, "code.das")), text, is_error) t |> success(ok, "parse result JSON") t |> success(!is_error, "a readable file is not a tool failure -- text={text}") - t |> success(find(text, "code.das:3:") >= 0, "the citation is reported at its own line -- text={text}") - t |> success(find(text, "symbol: scan_tokens") >= 0, "the annotated function is named -- text={text}") - t |> success(find(text, "## Tokenizer \{#tokenizer\}") >= 0, "the section opens at its heading -- text={text}") - t |> success(find(text, "The tokenizer section body.") >= 0, "the section body is returned -- text={text}") - t |> success(find(text, "A subsection stays inside its parent.") >= 0, - "a deeper heading does not end the section -- text={text}") - t |> success(find(text, "The buffers section body.") < 0, - "the next same-level heading ends it -- text={text}") + t |> success(find(text, "code.das:3 scan_tokens -> ") >= 0, + "the citation is a locator: its own line, the annotated function, then where it lands -- text={text}") + t |> success(find(text, "ARCHITECTURE.md:3-9 #tokenizer") >= 0, + "the section is a line range from its heading through its last line - the deeper heading at 7 does not end it, the next same-level one at 11 does -- text={text}") + t |> success(find(text, "The tokenizer section body.") < 0, + "the text itself is the document's to show; the tool gives coordinates -- text={text}") + t |> success(find(text, "#buffers") < 0, "a section nothing cites is not in the answer -- text={text}") rmdir_rec(dir) } t |> run("a dangling citation reports LINT026's reason") <| @(t : T?) { @@ -3628,7 +3627,8 @@ def test_arch_of(t : T?) { parse_result(do_arch_of(path_join(dir, "code.das")), text, is_error) t |> success(find(text, "does not resolve - no such anchor") >= 0, "lint's own wording, so one tool's answer reads as the other's -- text={text}") - t |> success(find(text, "symbol: dangling") >= 0, "the dangling citation still names its symbol -- text={text}") + t |> success(find(text, "code.das:7 dangling -> does not resolve") >= 0, + "the dangling citation still names its line and symbol -- text={text}") rmdir_rec(dir) } t |> run("the symbol filter keeps one citation") <| @(t : T?) { @@ -3641,8 +3641,9 @@ def test_arch_of(t : T?) { var text : string var is_error = false parse_result(do_arch_of(path_join(dir, "code.das"), "scan_tokens"), text, is_error) - t |> success(find(text, "symbol: scan_tokens") >= 0, "the named symbol's citation is kept -- text={text}") - t |> success(find(text, "symbol: dangling") < 0, "the other symbol's citation is dropped -- text={text}") + t |> success(find(text, "scan_tokens -> ") >= 0, "the named symbol's citation is kept -- text={text}") + t |> success(find(text, "dangling") < 0, "the other symbol's citation is dropped -- text={text}") + t |> success(find(text, "1 citation(s) shown.") >= 0, "and the tally says one -- text={text}") rmdir_rec(dir) } t |> run("a missing file is a tool failure") <| @(t : T?) { @@ -3661,6 +3662,29 @@ def test_arch_of(t : T?) { } } +// The C++ arm of arch_of: a `// #` pointer is read from a C++ source and located in +// the same shape a .das citation is, with no symbol - the tool does not parse C++ declarations. +[test] +def test_arch_of_cpp(t : T?) { + t |> run("a C++ source is read for its pointers, and lands in the same shape") <| @(t : T?) { + var terr : string + let dir = plant_arch_fixture("mcp_arch_cpp_test", terr) + if (empty(dir)) { + t |> failure("temp dir: {terr}") + return + } + fwrite(path_join(dir, "code.cpp"), "// ARCHITECTURE.md#tokenizer - the pointer form\nvoid f() \{\}\n") + var text : string + var is_error = false + parse_result(do_arch_of(path_join(dir, "code.cpp")), text, is_error) + t |> success(!is_error, "a C++ file is a readable source -- text={text}") + t |> success(find(text, "code.cpp:1 -> ") >= 0, + "the pointer is located at its line, with no symbol -- text={text}") + t |> success(find(text, "ARCHITECTURE.md:3-9 #tokenizer") >= 0, "and lands on the same section -- text={text}") + rmdir_rec(dir) + } +} + [test] def test_arch_sites(t : T?) { t |> run("the census finds the citations planted in the document's subtree") <| @(t : T?) { @@ -3675,12 +3699,14 @@ def test_arch_sites(t : T?) { let ok = parse_result(do_arch_sites(path_join(dir, "ARCHITECTURE.md")), text, is_error) t |> success(ok, "parse result JSON") t |> success(!is_error, "an existing document is not a tool failure -- text={text}") - t |> success(find(text, "\{#tokenizer\} (line 3) - 1 citation(s)") >= 0, - "the cited anchor is grouped at its heading line -- text={text}") + t |> success(find(text, "\{#tokenizer\} (lines 3-9) - 1 citation(s)") >= 0, + "the cited anchor is grouped with its section's line range -- text={text}") t |> success(find(text, "code.das:3 scan_tokens") >= 0, "the site names file, line and symbol -- text={text}") - t |> success(find(text, "\{#buffers\} (line 11) - no citations") >= 0, + t |> success(find(text, "\{#buffers\} (lines 11-13) - no citations") >= 0, "an uncited anchor is part of the census -- text={text}") + t |> success(find(text, "searched: ") >= 0 && find(text, "searched: ") < find(text, "src"), + "the header names the document's own folder first, then the C++ homes -- text={text}") t |> success(find(text, "\{#gone\}") >= 0, "a citation naming an undeclared anchor is surfaced, not dropped -- text={text}") t |> success(find(text, "2 citation(s) in 1 file(s).") >= 0, "the tally counts both -- text={text}") diff --git a/utils/mcp/tools/arch_common.das b/utils/mcp/tools/arch_common.das index d8acbd7f90..5f8c94719b 100644 --- a/utils/mcp/tools/arch_common.das +++ b/utils/mcp/tools/arch_common.das @@ -46,8 +46,9 @@ def annotated_symbol(lines : array; line : int) : string { return "" } -// Every .das under `path`, pruned the way LINT026's reverse walk prunes: `_`-prefixed and -// dot-prefixed names are fixtures and tooling, and `builtin.das` is not source anyone cites. +// Every .das and C/C++ source under `path`, pruned the way LINT026's reverse walk prunes: +// `_`-prefixed and dot-prefixed names are fixtures and tooling, and `builtin.das` is not source +// anyone cites. def scan_das_tree(path : string; var out : array) { fio::dir(path) $(name) { return if (arch_walk_skips(name)) @@ -56,8 +57,32 @@ def scan_das_tree(path : string; var out : array) { return if (!fst.is_valid) if (fst.is_dir) { scan_das_tree(full, out) - } elif (fst.is_reg && name |> ends_with(".das")) { + } elif (fst.is_reg && arch_cites_source(name)) { out |> push(full) } } } + +// The folders C++ sources live in - src/, include/, and every module's src/ - the homes a +// document's census reads beyond its own subtree, since a C++ pointer is not bound to it. +def cpp_roots() : array { + let root = arch_path(get_das_root()) + var out <- ["{root}/src", "{root}/include"] + fio::dir("{root}/modules") $(name) { + return if (arch_walk_skips(name)) + let src = "{root}/modules/{name}/src" + let st = stat(src) + if (st.is_valid && st.is_dir) { + out |> push(src) + } + } + return <- out +} + +// The symbol a citation on `line` annotates, for the languages the scan reads: a .das +// declaration below or above the marker; "" for a C++ source, whose pointer sits inside a +// comment the declaration grammar here does not parse. +def cited_symbol(file : string; lines : array; line : int) : string { + return "" if (is_cpp_source(file)) + return annotated_symbol(lines, line) +} diff --git a/utils/mcp/tools/arch_of.das b/utils/mcp/tools/arch_of.das index d20720368d..85bc41ec0d 100644 --- a/utils/mcp/tools/arch_of.das +++ b/utils/mcp/tools/arch_of.das @@ -8,37 +8,25 @@ require daslib/fio require strings require daslib/strings_boost -//! `arch_of` - the `[arch]` citations one file makes, each with the document section it cites. +//! `arch_of` - the `[arch]` citations one file makes, each located in the document it cites. //! -//! The read direction of the annotation: given code, show the architecture text it answers for, -//! so the section can be read without opening the document and counting headings. Resolution is -//! LINT026's, failures included - a citation this tool cannot follow is one lint will report. - -// Indent a section so it reads as quoted text rather than as part of the report. -def private write_section(var w : StringBuilderWriter; body : string) { - for (line in split(body, "\n")) { - w |> write(empty(line) ? "\n" : " {line}\n") - } -} +//! The read direction of the annotation: given code, say WHERE the architecture text it answers +//! for is - document, heading line and section end - so a read or an edit of the section follows +//! from the coordinates. One line per citation; the text itself is the document's to show. +//! Resolution is LINT026's, failures included - a citation this tool cannot follow is one lint +//! will report. A C/C++ source is read for its `// #` pointers. +// ` -> :- #`, or the reason it does not resolve. def private write_citation(var w : StringBuilderWriter; var docs : ArchDocs; file, rel : string; c : ArchCitation; symbol : string) { - w |> write("{rel}:{c.line}: arch(at=\"{c.raw}\")\n") - if (!empty(symbol)) { - w |> write(" symbol: {symbol}\n") - } + let who = empty(symbol) ? "{rel}:{c.line}" : "{rel}:{c.line} {symbol}" let reason = citation_failure_reason(docs, file, c) if (!empty(reason)) { - w |> write(" does not resolve - {reason}\n") - if (!empty(c.doc)) { - w |> write(" doc: {c.doc}\n") - } + w |> write("{who} -> does not resolve - {reason} (\"{c.raw}\")\n") return } - w |> write(" doc: {c.doc}\n") - let body = section_text(c.doc, c.anchor) - w |> write(" section:\n") - write_section(w, body) + let span = section_span(c.doc, c.anchor) + w |> write("{who} -> {make_relative_path(c.doc, get_das_root())}:{span.x}-{span.y - 1} #{c.anchor}\n") } def do_arch_of(file : string; symbol : string = "") : string { @@ -56,7 +44,7 @@ def do_arch_of(file : string; symbol : string = "") : string { var shown = 0 let body = build_string() $(var w) { for (c in cites) { - let owner = annotated_symbol(lines, c.line) + let owner = cited_symbol(arch_path(resolved), lines, c.line) continue if (!empty(symbol) && owner != symbol) shown ++ write_citation(w, docs, arch_path(resolved), rel, c, owner) diff --git a/utils/mcp/tools/arch_sites.das b/utils/mcp/tools/arch_sites.das index 11bf4d7b05..ad9b88b38b 100644 --- a/utils/mcp/tools/arch_sites.das +++ b/utils/mcp/tools/arch_sites.das @@ -48,7 +48,7 @@ def private collect_sites(folder, doc, anchor : string) : array { for (c in cites) { continue if (c.doc != doc || (!empty(anchor) && c.anchor != anchor)) out |> emplace(Site(file = rel, line = c.line, anchor = c.anchor, - symbol = annotated_symbol(lines, c.line))) + symbol = cited_symbol(arch_path(f), lines, c.line))) } } return <- out @@ -65,13 +65,14 @@ def private write_census(var w : StringBuilderWriter; doc : string; sites : arra for (a in collect_md_anchors(doc)) { continue if (declared |> key_exists(a.name)) declared |> insert(a.name) + let span = section_span(doc, a.name) // guarded: reading a mutable table by key would INSERT the anchor as a bucket of its own if (!(by_anchor |> key_exists(a.name))) { - w |> write("\{#{a.name}\} (line {a.line}) - no citations\n") + w |> write("\{#{a.name}\} (lines {span.x}-{span.y - 1}) - no citations\n") continue } let count = length(by_anchor[a.name]) - w |> write("\{#{a.name}\} (line {a.line}) - {count} citation(s)\n") + w |> write("\{#{a.name}\} (lines {span.x}-{span.y - 1}) - {count} citation(s)\n") write_sites(w, sites, by_anchor[a.name]) } var orphans : array @@ -104,14 +105,30 @@ def do_arch_sites(doc : string) : string { let st = stat(resolved) return make_tool_result("Document not found: {doc_arg}", true) if (!(st.is_valid && st.is_reg)) let folder = dir_name(resolved) - let sites <- collect_sites(folder, resolved, anchor) + var sites <- collect_sites(folder, resolved, anchor) + // a C++ pointer is not bound to the document's folder tree (a header under include/ cites + // src/'s document), so the census also reads the C++ homes outside the subtree + var searched_in <- [make_relative_path(folder, get_das_root())] + // a site is a citation, not a line: one comment can point at two anchors of this document + var seen <- {for (s in sites); "{s.file}:{s.line}#{s.anchor}" => null} + let roots <- cpp_roots() + searched_in |> reserve(1 + length(roots)) + for (croot in roots) { + searched_in |> push(make_relative_path(croot, get_das_root())) + let more <- collect_sites(croot, resolved, anchor) + // a C++ root may contain the document's own folder: the same site is not listed twice + sites |> push_from([for (s in more); s; where !(seen |> key_exists("{s.file}:{s.line}#{s.anchor}"))]) + for (s in more) { + seen |> insert("{s.file}:{s.line}#{s.anchor}", null) + } + } var files : array for (s in sites) { continue if (find_index(files, s.file) >= 0) files |> push(s.file) } let title = empty(anchor) ? resolved : "{resolved} \{#{anchor}\}" - let searched = make_relative_path(folder, get_das_root()) + let searched = join(searched_in, ", ") return make_tool_result(build_string() $(var w) { w |> write("{title}\nsearched: {searched}\n") if (empty(anchor)) { @@ -120,6 +137,9 @@ def do_arch_sites(doc : string) : string { } else { if (!anchor_declared(resolved, anchor)) { w |> write("note: this document declares no such anchor - a citation naming it does not resolve\n") + } else { + let span = section_span(resolved, anchor) + w |> write("section: lines {span.x}-{span.y - 1}\n") } w |> write("\n") let which <- [for (i in range(length(sites))); i]