diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a3e7b5660..2cdf3c56d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -123,6 +123,17 @@ jobs: make projection-wire-law-negative-control make projection-union-arm-order-negative-control + # THE TOOLCHAIN GATE's control (issue #599), on the same reasoning and in + # the same job. `make test` is the only thing that runs the gate, and no + # job on a pull request runs `make test`, so until this step existed the + # gate that stops a missing toolchain was itself unwatched on the diff + # that changes it. The control needs no toolchain of its own: it points + # each registered pin at a path that does not exist and reads the + # refusal, which is the state this runner is already in, and takes about + # a second. + - name: the toolchain gate goes red without the toolchains it pins + run: make toolchain-negative-control + # THE GENERATED TREE IS CURRENT (issue #30). The committed generated/ tree is # what this repo treats as ground truth — findings are receipted against its # line numbers — so a compiler change that moves the emitted text and leaves diff --git a/Makefile b/Makefile index eb8cf8bb8..4fa9f6020 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,32 @@ # serialize checkouts, no language toolchains, no generation. The full # nine-language conformance chain is `make test`, and it needs the sibling # runtime checkouts and toolchains documented below. +# +# THE TOOLCHAIN GATE (issue #599). `make test` REFUSES BY NAME when a pinned +# toolchain a make/.mk names does not resolve. It probes EVERY registered +# leg and names every one that does not resolve in the same run, with the path +# each pin looked in, rather than stopping at the first and making a fresh +# clone a queue, and it stops before the chain spends an hour on the legs that +# do resolve. A gate that passes while its toolchain is missing is a gate with +# no blade: that is how a red in the Dart leg once rode a green run, after a +# merge deleted a clone's dist link. +# +# To run the chain without a leg, name the skip ON PURPOSE: +# +# make test SCHEMA_SKIP_LEGS=js,dart +# +# Every named leg prints its skip in the toolchain gate, in the conformance +# matrix, in the two-language gates and in the leg loop, and is not run. +# Nothing under .github sets the variable: certification installs each +# toolchain and overrides the pins (NODE=node, DART=dart, JAVA=java, +# JAVAC=javac, ELIXIR=elixir, MIX=mix), which resolve and pass the gate. +# +# `make toolchain` runs the gate alone. `make toolchain-negative-control` +# points every registered pin at a path that does not exist, ONE PIN AT A TIME +# (a leg with two pins is proved twice, a leg with three, three times), and +# requires each to refuse by the leg's name and that pin's, then names every +# leg in SCHEMA_SKIP_LEGS and requires the same gate to go green with each +# skip printed by name. CXX ?= c++ CXXFLAGS ?= -std=c++17 -Wall -Wextra -Werror -ffp-contract=off @@ -13,6 +39,39 @@ CXXFLAGS ?= -std=c++17 -Wall -Wextra -Werror -ffp-contract=off SERIALIZE ?= ../serialize CXXFLAGS += -I$(SERIALIZE) +# THE SKIP LIST (issue #599): the legs whose toolchain is absent ON PURPOSE, +# comma separated, empty by default and empty in every .github workflow. A leg +# named here prints its skip and does not run; a leg NOT named here whose pin +# does not resolve stops `make test` by name. +SCHEMA_SKIP_LEGS ?= +skip_comma := , +skip_empty := +skip_space := $(skip_empty) $(skip_empty) +# a literal hash, so a message held in a variable can cite an issue by number: +# an unescaped one starts a make comment and truncates the value in silence +skip_hash := \# +SKIPPED_LEGS := $(subst $(skip_comma),$(skip_space),$(SCHEMA_SKIP_LEGS)) + +# $(call unless_skipped,,) is the text, or nothing when the leg is +# named in SCHEMA_SKIP_LEGS. A leg registers its conformance prerequisites +# through it, so a named skip reaches the matrix and not only the leg loop. +unless_skipped = $(if $(filter $(1),$(SKIPPED_LEGS)),,$(2)) + +# $(call toolchain_probe,,,[,]) is one +# line of a leg's toolchain- recipe. It says nothing when the pin +# resolves, and refuses by name when it does not: it names the leg, the pin, +# the path the pin looked in, and the three ways out. The default probe +# resolves the pin's own value; a leg whose pin carries an environment prefix +# passes its own probe as the fourth argument. +toolchain_probe = $(if $(4),$(4),command -v $(3)) >/dev/null 2>&1 || { \ + echo "make test REFUSES: the $(1) leg's pinned $(2) does not resolve"; \ + echo ' $(2) = $(3)'; \ + echo " test-$(1) would have been SKIPPED, and a leg that skips is a gate with no blade (issue $(skip_hash)599)"; \ + echo " make/$(1).mk names the pinned toolchain and where it comes from; install it,"; \ + echo " or override the pin ($(2)=... make test), or name the skip on purpose:"; \ + echo " make test SCHEMA_SKIP_LEGS=$(1)"; \ + exit 1; } + # cmd, internal, AND the public API packages: ir/ and compiler/ are compiled # into bin/schema like any other source, and leaving them out made an edit to # the layout model or the build version silently NOT rebuild the compiler every @@ -628,7 +687,14 @@ tables-block: build/schema_test_block build/schema_test_block_asan build/schema_ ./build/schema_test_block ./build/schema_test_block_asan ./build/schema_test_block_tsan - cd test/cs-block && dotnet run +# THE C# HALF of this two-language gate runs unless the cs leg is named in +# SCHEMA_SKIP_LEGS (issue #599). A skip that reaches the leg loop and not here +# is a skip that stops the chain anyway, on the same missing dotnet. +ifeq ($(filter cs,$(SKIPPED_LEGS)),) + cd test/cs-block && $(DOTNET) run +else + @echo "tables-block: the C# half is SKIPPED, SCHEMA_SKIP_LEGS names the cs leg" +endif # --------------------------------------------------------------------------- # THE FORGERY FUZZER (docs/SPEC-TABLES.md §19.2, §19.5). The hand-written battery in @@ -694,7 +760,12 @@ build/block-fuzz/.stamp: build/schema_test_block_fuzz tables-block-fuzz: build/schema_test_block_fuzz build/schema_test_block_fuzz_asan build/block-fuzz/.stamp build/tables-generated-cs/.stamp SEED=$(SEED) N=$(N) ./build/schema_test_block_fuzz SEED=$(SEED) N=$(N) ./build/schema_test_block_fuzz_asan - cd test/cs-block && SEED=$(SEED) N=$(N) dotnet run -- --fuzz +# the C# half, on the same rule as tables-block above (issue #599) +ifeq ($(filter cs,$(SKIPPED_LEGS)),) + cd test/cs-block && SEED=$(SEED) N=$(N) $(DOTNET) run -- --fuzz +else + @echo "tables-block-fuzz: the C# half is SKIPPED, SCHEMA_SKIP_LEGS names the cs leg" +endif # THE COOK FIXTURES the Rust fuzzer's cook half forges, written by test/cookgen # with the same chains the conformance harness uses — the fixture generator's @@ -764,15 +835,19 @@ define block_fuzz_sabotage fi @grep -q "^FAILED: an opened block" build/block-fuzz-$(1)/cpp.log || \ { echo "NEGATIVE CONTROL FAILED: the C++ leg went red, but not on the oracle"; cat build/block-fuzz-$(1)/cpp.log; exit 1; } - @if ( cd test/cs-block && SEED=$(SEED) N=$(N) dotnet run \ - -p:BlockGeneratedDir=../../build/block-fuzz-$(1)/generated/block-cs \ - -p:BlockHomeGeneratedDir=../../build/block-fuzz-$(1)/generated/blockhome-cs -- --fuzz ) \ - > build/block-fuzz-$(1)/cs.log 2>&1; then \ - echo "NEGATIVE CONTROL FAILED: the C# fuzzer stayed green with the $(1) check removed from the emitter"; \ - exit 1; \ + @if [ -n '$(filter cs,$(SKIPPED_LEGS))' ]; then \ + echo " the C# half of this control is SKIPPED, SCHEMA_SKIP_LEGS names the cs leg"; \ + else \ + if ( cd test/cs-block && SEED=$(SEED) N=$(N) $(DOTNET) run \ + -p:BlockGeneratedDir=../../build/block-fuzz-$(1)/generated/block-cs \ + -p:BlockHomeGeneratedDir=../../build/block-fuzz-$(1)/generated/blockhome-cs -- --fuzz ) \ + > build/block-fuzz-$(1)/cs.log 2>&1; then \ + echo "NEGATIVE CONTROL FAILED: the C# fuzzer stayed green with the $(1) check removed from the emitter"; \ + exit 1; \ + fi; \ + grep -q "^FAILED: an opened block" build/block-fuzz-$(1)/cs.log || \ + { echo "NEGATIVE CONTROL FAILED: the C# leg went red, but not on the oracle"; cat build/block-fuzz-$(1)/cs.log; exit 1; }; \ fi - @grep -q "^FAILED: an opened block" build/block-fuzz-$(1)/cs.log || \ - { echo "NEGATIVE CONTROL FAILED: the C# leg went red, but not on the oracle"; cat build/block-fuzz-$(1)/cs.log; exit 1; } @rm -f build/tables-generated-rust/.stamp ./build/block-fuzz-$(1)/schema generate --lang rust --out build/tables-generated-rust/blockdemo/src tables/block ./build/block-fuzz-$(1)/schema generate --lang rust --out build/tables-generated-rust/blockhome/src tables/blockhome @@ -1401,12 +1476,15 @@ tables-block-fill-refuser-negative-control: build/tables-generated/.stamp # at the wrong offset, which is exactly what §19.3 says Size alone cannot pin. .PHONY: tables-block-padding-negative-control tables-block-padding-negative-control: build/tables-generated-cs/.stamp +ifneq ($(filter cs,$(SKIPPED_LEGS)),) + @echo "tables-block-padding-negative-control: SKIPPED, it is a C# control and SCHEMA_SKIP_LEGS names the cs leg" +else @rm -rf build/block-padding-sabotage && cp -R build/tables-generated-cs/block build/block-padding-sabotage @sed -i.bak '/private byte _pad/d' build/block-padding-sabotage/PaddedBlock.cs @grep -q "private byte _pad" build/block-padding-sabotage/PaddedBlock.cs && \ { echo "NEGATIVE CONTROL: the sabotage did not apply"; exit 1; } || true @rm -f build/block-padding-sabotage/*.bak - @if ( cd test/cs-block && dotnet run -p:BlockGeneratedDir=../../build/block-padding-sabotage ) \ + @if ( cd test/cs-block && $(DOTNET) run -p:BlockGeneratedDir=../../build/block-padding-sabotage ) \ > build/block-padding-sabotage.log 2>&1; then \ echo "NEGATIVE CONTROL FAILED: the C# layout check passed with the padding fields deleted"; \ cat build/block-padding-sabotage.log; exit 1; \ @@ -1414,6 +1492,7 @@ tables-block-padding-negative-control: build/tables-generated-cs/.stamp @grep -q "schema block layout" build/block-padding-sabotage.log || \ { echo "NEGATIVE CONTROL FAILED: the failure was not the layout check"; cat build/block-padding-sabotage.log; exit 1; } @echo "block padding negative control: deleting the generated padding fields turns the layout check red" +endif # §19.5's FIRST named negative control: "perturb one row type's pitch constant # on one side only and the two-language test goes red". The constant is the C# @@ -1423,13 +1502,16 @@ tables-block-padding-negative-control: build/tables-generated-cs/.stamp # not inert. .PHONY: tables-block-pitch-negative-control tables-block-pitch-negative-control: build/tables-generated-cs/.stamp +ifneq ($(filter cs,$(SKIPPED_LEGS)),) + @echo "tables-block-pitch-negative-control: SKIPPED, it is a C# control and SCHEMA_SKIP_LEGS names the cs leg" +else @rm -rf build/block-pitch-sabotage && cp -R build/tables-generated-cs/block build/block-pitch-sabotage @sed -i.bak 's|public const long ShipsStride = 88;|public const long ShipsStride = 96; // SABOTAGED|' \ build/block-pitch-sabotage/RenderBlock.cs @grep -q "SABOTAGED" build/block-pitch-sabotage/RenderBlock.cs || \ { echo "NEGATIVE CONTROL: the sabotage did not apply"; exit 1; } @rm -f build/block-pitch-sabotage/*.bak - @if ( cd test/cs-block && dotnet run -p:BlockGeneratedDir=../../build/block-pitch-sabotage ) \ + @if ( cd test/cs-block && $(DOTNET) run -p:BlockGeneratedDir=../../build/block-pitch-sabotage ) \ > build/block-pitch-sabotage.log 2>&1; then \ echo "NEGATIVE CONTROL FAILED: the C# side accepted a pitch constant that is not its row's sizeof"; \ cat build/block-pitch-sabotage.log; exit 1; \ @@ -1437,6 +1519,7 @@ tables-block-pitch-negative-control: build/tables-generated-cs/.stamp @grep -q "ShipsStride" build/block-pitch-sabotage.log || \ { echo "NEGATIVE CONTROL FAILED: it went red, but not on the pitch constant"; cat build/block-pitch-sabotage.log; exit 1; } @echo "block pitch negative control: a pitch constant perturbed on ONE side turns the C# layout check red" +endif # §19.5's SECOND named negative control: "perturb one field's offset in the # compiler's layout model and the generated asserts go red on both backends". @@ -1467,7 +1550,10 @@ tables-block-layout-model-negative-control: bin/schema @grep -q "static_assert" build/block-model-cpp.log || \ { echo "NEGATIVE CONTROL FAILED: C++ went red, but not on a layout static_assert"; cat build/block-model-cpp.log; exit 1; } @echo "block layout-model negative control (C++): a moved offset in the compiler's model turns the static_asserts red" - @if ( cd test/cs-block && dotnet run -p:BlockGeneratedDir=../../build/block-model-sabotage/cs ) \ +ifneq ($(filter cs,$(SKIPPED_LEGS)),) + @echo "block layout-model negative control (C#): SKIPPED, SCHEMA_SKIP_LEGS names the cs leg" +else + @if ( cd test/cs-block && $(DOTNET) run -p:BlockGeneratedDir=../../build/block-model-sabotage/cs ) \ > build/block-model-cs.log 2>&1; then \ echo "NEGATIVE CONTROL FAILED: C# accepted a moved offset from the compiler's model"; \ cat build/block-model-cs.log; exit 1; \ @@ -1475,6 +1561,7 @@ tables-block-layout-model-negative-control: bin/schema @grep -q "schema block layout" build/block-model-cs.log || \ { echo "NEGATIVE CONTROL FAILED: C# went red, but not on the layout check"; cat build/block-model-cs.log; exit 1; } @echo "block layout-model negative control (C#): the same moved offset turns the once-run layout check red" +endif # THE BLOCK HOME's NEGATIVE CONTROL (docs/SPEC-TABLES.md §19.2's C# surface). The # dogfood found two defects with one root cause — a C# backend that emitted per @@ -1494,6 +1581,9 @@ tables-block-layout-model-negative-control: bin/schema # have nowhere to land. The compile must FAIL. .PHONY: tables-block-home-negative-control tables-block-home-negative-control: bin/schema +ifneq ($(filter cs,$(SKIPPED_LEGS)),) + @echo "tables-block-home-negative-control: SKIPPED, it is a C# control and SCHEMA_SKIP_LEGS names the cs leg" +else @mkdir -p build @sed -e 's|if home != "" \&\& !runtimeWritten {|if false \&\& !runtimeWritten { // SABOTAGED: no home is emitted for the unit|' \ internal/codegen/cstable/block.go > build/csblock-declaring-file.gotext @@ -1504,7 +1594,7 @@ tables-block-home-negative-control: bin/schema @go build -overlay=build/csblock-overlay.json -o build/schema-csblock-sabotaged ./cmd/schema @rm -rf build/blockhome-sabotage && mkdir -p build/blockhome-sabotage @./build/schema-csblock-sabotaged generate --lang cs --out build/blockhome-sabotage tables/blockhome - @if ( cd test/cs-block && dotnet build -v q --nologo -p:BlockHomeGeneratedDir=../../build/blockhome-sabotage ) \ + @if ( cd test/cs-block && $(DOTNET) build -v q --nologo -p:BlockHomeGeneratedDir=../../build/blockhome-sabotage ) \ > build/blockhome-sabotage.log 2>&1; then \ echo "NEGATIVE CONTROL FAILED: the unit compiled with its block runtime emitted nowhere"; \ cat build/blockhome-sabotage.log; exit 1; \ @@ -1512,6 +1602,7 @@ tables-block-home-negative-control: bin/schema @grep -qE "CS0103|CS0234|CS0246" build/blockhome-sabotage.log || \ { echo "NEGATIVE CONTROL FAILED: it went red, but not on an undefined name"; tail -20 build/blockhome-sabotage.log; exit 1; } @echo "block home negative control: a unit with no home for its block runtime does not compile" +endif # THE RUNTIME HOME IS THE PACKAGE (docs/SPEC-TABLES.md §19.2). A unit's shared C# # runtime — the table runtime and the text form's walk in Table.cs, the @@ -1577,6 +1668,9 @@ tables-runtime-home-negative-control: bin/schema tables-runtime-home # and offsets). The control reports which one fired. .PHONY: tables-block-inline-array-negative-control tables-block-inline-array-negative-control: bin/schema +ifneq ($(filter cs,$(SKIPPED_LEGS)),) + @echo "tables-block-inline-array-negative-control: SKIPPED, it is a C# control and SCHEMA_SKIP_LEGS names the cs leg" +else @mkdir -p build @sed -e 's|if projection \&\& ir.BlockOutOfLine(f) {|if ir.BlockOutOfLine(f) { // SABOTAGED: project at every depth|' \ -e 's|inline := !projection \|\| !ir.BlockOutOfLine(f)|inline := !ir.BlockOutOfLine(f)|' \ @@ -1588,7 +1682,7 @@ tables-block-inline-array-negative-control: bin/schema @go build -overlay=build/csblock-depth-overlay.json -o build/schema-depth-sabotaged ./cmd/schema @rm -rf build/blockhome-depth && mkdir -p build/blockhome-depth @./build/schema-depth-sabotaged generate --lang cs --out build/blockhome-depth tables/blockhome - @if ( cd test/cs-block && dotnet run -p:BlockHomeGeneratedDir=../../build/blockhome-depth ) \ + @if ( cd test/cs-block && $(DOTNET) run -p:BlockHomeGeneratedDir=../../build/blockhome-depth ) \ > build/blockhome-depth.log 2>&1; then \ echo "NEGATIVE CONTROL FAILED: a bounded array projected inside a nested record passed the layout gate"; \ cat build/blockhome-depth.log; exit 1; \ @@ -1600,6 +1694,7 @@ tables-block-inline-array-negative-control: bin/schema else \ echo "NEGATIVE CONTROL FAILED: it went red, but not on either half of the gate"; tail -20 build/blockhome-depth.log; exit 1; \ fi +endif # GATE 2 (docs/SPEC-TABLES.md §12.1): the MEASURED gate, two numbers, and it is not # part of `make test` on purpose — a correctness suite whose verdict depends on @@ -1617,7 +1712,7 @@ build/schema_test_block_gate2: build/tables-generated/.stamp test/tables/block_g .PHONY: tables-block-gate2 tables-block-gate2: build/schema_test_block_gate2 build/tables-generated-cs/.stamp ./build/schema_test_block_gate2 - cd test/cs-block && dotnet run -c Release -- --gate2 + cd test/cs-block && $(DOTNET) run -c Release -- --gate2 # THE SMOKE, which is what CI runs (certification legs only, never a PR). # The gate above is a MEASUREMENT and its verdict belongs on a quiet box; a @@ -1630,7 +1725,7 @@ tables-block-gate2: build/schema_test_block_gate2 build/tables-generated-cs/.sta .PHONY: tables-block-gate2-smoke tables-block-gate2-smoke: build/schema_test_block_gate2 build/tables-generated-cs/.stamp ./build/schema_test_block_gate2 --smoke - cd test/cs-block && dotnet run -c Release -- --gate2-smoke + cd test/cs-block && $(DOTNET) run -c Release -- --gate2-smoke # The NEGATIVE CONTROL for a KEYED object's duplicate counting # (docs/SPEC-TABLES.md §16.2). Last-wins inside a keyed object was already true, so # the missing count was invisible to every round-trip test — the value was @@ -2873,7 +2968,14 @@ build/schema_test_bench_table: generated/bench/tables/cpp/.stamp test/bench/tabl @mkdir -p build $(CXX) $(CXXFLAGS) -Igenerated/bench/tables/cpp test/bench/table_main.cpp -o $@ -test: build/schema_test build/schema_test_guard build/schema_test_tables build/schema_test_block build/schema_test_block_asan build/schema_test_block_fuzz build/schema_test_block_fuzz_asan build/pack-text/.stamp build/schema_test_hostile build/schema_test_hostile_asan build/hostile-values/.stamp build/schema_test_pack build/schema_test_pack_asan build/tables-pack.bin build/tables-pack-root.bin build/schema_test_tables_asan build/schema_test_random build/schema_test_ludicrous build/schema_test_bench build/schema_test_bench_table build/conformance-harness build/schema_test_wide build/schema_test_wide_table +# THE TOOLCHAIN GATE IS THE FIRST PREREQUISITE (issue #599): a pinned toolchain +# that does not resolve stops the chain by name here, before an hour of C++, +# rather than in the middle of a leg with a shell's "command not found". +test: toolchain build/schema_test build/schema_test_guard build/schema_test_tables build/schema_test_block build/schema_test_block_asan build/schema_test_block_fuzz build/schema_test_block_fuzz_asan build/pack-text/.stamp build/schema_test_hostile build/schema_test_hostile_asan build/hostile-values/.stamp build/schema_test_pack build/schema_test_pack_asan build/tables-pack.bin build/tables-pack-root.bin build/schema_test_tables_asan build/schema_test_random build/schema_test_ludicrous build/schema_test_bench build/schema_test_bench_table build/conformance-harness build/schema_test_wide build/schema_test_wide_table + # THE TOOLCHAIN GATE's own control (issue #599): the gate above is what + # stands between a missing toolchain and a green run, so it is held to the + # same rule as every other gate here. It has to go red on demand, by name. + $(MAKE) toolchain-negative-control ./build/schema_test ./build/schema_test_guard ./build/schema_test_wide @@ -3016,8 +3118,16 @@ test: build/schema_test build/schema_test_guard build/schema_test_tables build/s ./build/schema_test_bench_table # EVERY REGISTERED LEG (make/.mk, TEST_LEGS): its generated trees, its # gates and negative controls, its packet and table tests. One sub-make per - # leg, so a red names the leg. - @set -e; for leg in $(TEST_LEGS); do echo "$(MAKE) $$leg"; $(MAKE) $$leg; done + # leg, so a red names the leg. A leg named in SCHEMA_SKIP_LEGS is not run + # and says so by name (issue #599): a run that measured eight legs and a run + # that measured four must not read the same. + @set -e; for leg in $(TEST_LEGS); do \ + case " $(SKIPPED_LEGS) " in \ + *" $${leg#test-} "*) \ + echo "make test SKIPS $$leg: SCHEMA_SKIP_LEGS names the $${leg#test-} leg on purpose"; \ + continue ;; \ + esac; \ + echo "$(MAKE) $$leg"; $(MAKE) $$leg; done go test ./... @@ -4663,6 +4773,12 @@ conformance-negative-control-block-dump: build/conformance-harness build/conform # CONFORMANCE_ENV environment the harness run carries to the drivers # BENCH_TABLES_LEGS the generated unit a leg of `make bench-tables` needs # GOLDENS_LEGS update-goldens-: the leg's committed table goldens +# TOOLCHAIN_LEGS the legs with a pinned toolchain, each of which also +# defines toolchain- (one $(call toolchain_probe) +# per pin) and sets TOOLCHAIN_PINS_ to EVERY pin +# that target probes. The negative control points each +# one in turn at a path that does not exist, so a pin +# left off the list is a probe nothing watches # THE WIDE-SCALAR REFUSAL GATE (docs/SPEC-TABLES.md §3, §15): every scalar the # type wire carries rides in a table in the C++ reference and the tool, and a # port that has not landed the kinds yet must REFUSE a unit declaring them, by @@ -4707,9 +4823,14 @@ include make/checks/packet-text.mk # THE CONFORMANCE MATRIX (test/conformance/README.md): every discovered driver # over every surface it lists. The reference leg is C++ and is built here; the # rest are what the legs registered. +# +# A leg named in SCHEMA_SKIP_LEGS is not built here and its driver is not run +# (issue #599): --skip takes the same names, and the harness prints each one it +# passes over. The reference leg cannot be skipped. .PHONY: conformance conformance: build/conformance-harness build/conformance-cpp build/schema_test_cook $(CONFORMANCE_LEGS) - $(CONFORMANCE_ENV) ./build/conformance-harness run + $(CONFORMANCE_ENV) ./build/conformance-harness run \ + $(if $(SKIPPED_LEGS),--skip $(subst $(skip_space),$(skip_comma),$(strip $(SKIPPED_LEGS)))) # THE TABLES BENCH PASS (bench/tables/README.md): every leg under # bench/tables/*/leg, results under bench/tables/results/. A PUBLISHABLE @@ -4729,6 +4850,113 @@ registry: @echo "conformance: $(CONFORMANCE_LEGS)" @echo "bench-tables: $(BENCH_TABLES_LEGS)" @echo "goldens: $(GOLDENS_LEGS)" + @echo "toolchain: $(TOOLCHAIN_LEGS)" + +# --------------------------------------------------------------------------- +# THE TOOLCHAIN GATE (issue #599), the aggregate ---------------------------- +# +# Every leg that registered a pinned toolchain is probed here, before the chain +# spends its hour, and a pin that does not resolve stops `make test` by name. +# A leg named in SCHEMA_SKIP_LEGS is not probed and prints its skip instead, so +# the run says which legs it did not measure rather than passing over them. +# Nothing here lists a language: a leg registers TOOLCHAIN_LEGS, its pin name +# and its own toolchain- target in make/.mk, like every other list +# above. +REGISTERED_LEGS := $(patsubst test-%,%,$(TEST_LEGS)) +UNKNOWN_SKIPS := $(filter-out $(REGISTERED_LEGS),$(SKIPPED_LEGS)) +SKIPPED_TOOLCHAIN_LEGS := $(filter $(SKIPPED_LEGS),$(TOOLCHAIN_LEGS)) +PROBED_TOOLCHAIN_LEGS := $(filter-out $(SKIPPED_LEGS),$(TOOLCHAIN_LEGS)) + +# A NAME THAT IS NOT A LEG IS A SKIP THAT SKIPS NOTHING, and it reads exactly +# like one that works, so it is refused before any pin is probed. +.PHONY: toolchain-names +toolchain-names: +ifneq ($(UNKNOWN_SKIPS),) + @echo "make test REFUSES: SCHEMA_SKIP_LEGS names $(UNKNOWN_SKIPS), which is no registered leg" + @echo " the registered legs are: $(REGISTERED_LEGS) (make registry prints them)" + @exit 1 +endif + +# EVERY LEG NOT NAMED IN THE SKIP LIST IS PROBED, and every one that does not +# resolve is NAMED, in one run. A leg's probe as a PREREQUISITE stops the gate +# at the first one that fails, turning a fresh clone into a queue: install one +# toolchain, run again, learn the next name. The legs go through sub-makes here +# instead, each leg's own refusal passed through as it printed it, and the +# summary names them together with the skip line that runs the chain without +# them, already carrying the skips in force. +# +# The green line says how many of the registered legs were probed, and when +# that is none it says so as a skip rather than as a claim about toolchains it +# never looked at. +TOOLCHAIN_GREEN = toolchain gate: $(words $(PROBED_TOOLCHAIN_LEGS)) of $(words $(TOOLCHAIN_LEGS)) registered legs probed, $(if $(PROBED_TOOLCHAIN_LEGS),every pinned toolchain resolves,every registered leg is named in SCHEMA_SKIP_LEGS) +.PHONY: toolchain +toolchain: toolchain-names + @rm -rf build/toolchain && mkdir -p build/toolchain + @missing=""; \ + $(foreach leg,$(PROBED_TOOLCHAIN_LEGS), \ + $(MAKE) --no-print-directory toolchain-$(leg) \ + > build/toolchain/$(leg).log 2> build/toolchain/$(leg).err \ + || missing="$$missing $(leg)"; \ + cat build/toolchain/$(leg).log; \ + grep -q "REFUSES: the $(leg) leg" build/toolchain/$(leg).log \ + || cat build/toolchain/$(leg).err;) \ + $(foreach leg,$(SKIPPED_TOOLCHAIN_LEGS), \ + echo "toolchain gate: the $(leg) leg is SKIPPED on purpose, SCHEMA_SKIP_LEGS names it";) \ + if [ -n "$$missing" ]; then \ + echo "make test REFUSES: the pinned toolchain does not resolve for:$$missing"; \ + echo " every refusal above names its leg, its pin and the path that pin looked in"; \ + echo " to run the chain without those legs, name the skips on purpose:"; \ + echo " make test SCHEMA_SKIP_LEGS=$$(echo $(SKIPPED_LEGS) $$missing | tr ' ' ',')"; \ + exit 1; \ + fi; \ + echo "$(TOOLCHAIN_GREEN)" + +# ITS NEGATIVE CONTROL. A gate that has never gone red is a gate with no blade, +# and going red is this one's whole job, so the control makes it go red: EVERY +# PIN OF EVERY registered leg, one pin at a time, pointed at a path that does +# not exist, and each must refuse BY THE LEG'S OWN NAME AND THAT PIN'S. A leg +# with a second pin is exactly where the blade goes blunt unseen: a control +# that points one pin per leg leaves the rest of them watching nothing, and +# deleting their probes keeps it green. Then the POSITIVE half, with every pin +# still pointed at that absent path and SCHEMA_SKIP_LEGS naming every leg: the +# same gate goes green and prints one named skip per leg. Nothing here lists a +# language either. +# +# While one pin is absent, THE LEG'S OTHER PINS POINT AT A PATH THAT RESOLVES, +# so the refusal is attributable to the pin under test rather than to whichever +# pin the bench happens to be missing anyway. A bench with none of the pinned +# toolchains installed is the case this gate exists for, so the control has to +# hold there too, and it does not hold if an earlier probe answers first. +TOOLCHAIN_NC_ABSENT := $(CURDIR)/build/toolchain-nc/absent +TOOLCHAIN_NC_PRESENT := /bin/sh +TOOLCHAIN_ALL_PINS = $(foreach leg,$(TOOLCHAIN_LEGS),$(TOOLCHAIN_PINS_$(leg))) +.PHONY: toolchain-negative-control +toolchain-negative-control: + @rm -rf build/toolchain-nc && mkdir -p build/toolchain-nc + @$(foreach leg,$(TOOLCHAIN_LEGS),$(foreach pin,$(TOOLCHAIN_PINS_$(leg)), \ + if $(MAKE) --no-print-directory toolchain-$(leg) \ + $(foreach other,$(filter-out $(pin),$(TOOLCHAIN_PINS_$(leg))),$(other)=$(TOOLCHAIN_NC_PRESENT)) \ + $(pin)=$(TOOLCHAIN_NC_ABSENT) \ + > build/toolchain-nc/$(leg)-$(pin).log 2>&1; then \ + echo "NEGATIVE CONTROL FAILED: the $(leg) leg stayed green with $(pin) pointed at a path that does not exist"; \ + cat build/toolchain-nc/$(leg)-$(pin).log; exit 1; \ + fi; \ + grep -q "REFUSES: the $(leg) leg's pinned $(pin) does not resolve" build/toolchain-nc/$(leg)-$(pin).log || \ + { echo "NEGATIVE CONTROL FAILED: the $(leg) leg went red with $(pin) absent, but not by that pin's name"; \ + cat build/toolchain-nc/$(leg)-$(pin).log; exit 1; }; \ + sed -n '1,2p' build/toolchain-nc/$(leg)-$(pin).log;)) + @$(MAKE) --no-print-directory toolchain \ + SCHEMA_SKIP_LEGS=$(subst $(skip_space),$(skip_comma),$(strip $(TOOLCHAIN_LEGS))) \ + $(foreach pin,$(TOOLCHAIN_ALL_PINS),$(pin)=$(TOOLCHAIN_NC_ABSENT)) \ + > build/toolchain-nc/skipped.log 2>&1 || \ + { echo "POSITIVE FAILED: SCHEMA_SKIP_LEGS named every leg and the gate refused anyway"; \ + cat build/toolchain-nc/skipped.log; exit 1; } + @$(foreach leg,$(TOOLCHAIN_LEGS), \ + grep -q "the $(leg) leg is SKIPPED on purpose" build/toolchain-nc/skipped.log || \ + { echo "POSITIVE FAILED: the $(leg) leg was passed over without being named"; \ + cat build/toolchain-nc/skipped.log; exit 1; };) + @cat build/toolchain-nc/skipped.log + @echo "negative control: each of the $(words $(TOOLCHAIN_ALL_PINS)) registered pins ($(TOOLCHAIN_ALL_PINS)) pointed at a path that does not exist turns make test RED, by leg and by pin; naming the leg in SCHEMA_SKIP_LEGS turns it GREEN with the skip printed by name" # THE `was` CONTROL (docs/SPEC-TABLES.md §5). A table renamed under `was` keeps # the node type id every stored record carries, so W1's fleet reads under W2's diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 2e9cb65ae..5bd10173e 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -52,6 +52,29 @@ compiles each, and compares the emitted wire against pinned goldens. That cross-language bit-identity check is the property this project exists to provide, so a change that breaks it is wrong until proven otherwise. +**`make test` refuses a missing pinned toolchain by name, and runs without a +leg only when you name the skip on purpose.** A pin the Makefile names +(`NODE`, `DART`, `JAVA`/`JAVAC`, `ELIXIR`/`MIX`/`ELIXIRC`, `DOTNET`) that does +not resolve stops the run before it starts. Every registered leg is probed, so +one run names every leg it would have skipped and the path each pin looked in +rather than sending you back for the next name; to run the chain without those +legs, name them in `SCHEMA_SKIP_LEGS`, which prints every skip by name and +which the refusal spells out ready to paste: + +```bash +make test SCHEMA_SKIP_LEGS=dart,java +``` + +A gate that passes while its toolchain is missing is a gate with no blade, and +that is not a hypothetical here: a merge once deleted a clone's `dist` link, +`make test` passed over four legs in silence, and a red inside one of them rode +the green run (issue #599). Nothing under `.github` sets `SCHEMA_SKIP_LEGS`: +certification installs every toolchain and overrides the pins, which resolve +and pass the gate. `make toolchain` runs the gate alone, and +`make toolchain-negative-control` proves it still has its blade, on every pin +of every leg. That control runs inside `make test` and on every pull request, +in the `go-test` job of `ci.yml`. + The Makefile's `SERIALIZE*` variables override the sibling paths if you keep them elsewhere. @@ -112,7 +135,7 @@ touches and two ports landing in one week do not conflict: | the target | `compiler/target_.go` (`target_javascript.go`: a `_js` suffix is a Go build constraint) | its `init` registers the generator | | the runtime's claimed names | `internal/tablenames/.go` | its `init` defines the backend's bit and names | | the compiler's tests | `compiler/tables_test.go` | the package | -| the build | `make/.mk` | the Makefile's wildcard include; the file registers its `test-` leg, its conformance build, its bench unit and its goldens | +| the build | `make/.mk` | the Makefile's wildcard include; the file registers its `test-` leg, its conformance build, its bench unit, its goldens and its pinned toolchain | | the conformance leg | `test/conformance//driver` and `ci.json` | the harness discovers the driver; `harness matrix` builds the pull-request matrix from the rows | | the tables bench leg | `bench/tables//leg` | `bench/tables/run.sh` runs every leg | | the shape gate's exemptions | `bench//SHAPE-GATE.allow`, `bench/tables//SHAPE-GATE.allow` | the gate reads every ledger under the tree | @@ -124,6 +147,17 @@ of the tree and requires the harness, the CI matrix, the bench pass and the Makefile to find it with no shared file edited. If a port needs to edit a file that lists languages, that is a defect in the registry, not a step. +A port with a pinned toolchain registers it in the same file and the same way: +`TOOLCHAIN_LEGS += `, `TOOLCHAIN_PINS_ :=` every pin the leg +probes, and a `toolchain-` target carrying one +`$(call toolchain_probe,...)` per pin. That is what makes `make test` refuse +the leg by name instead of skipping it, and what makes +`SCHEMA_SKIP_LEGS=` a skip anyone can read in the log. The negative +control points each pin on that list at a path that does not exist IN TURN, +with the leg's other pins pointed at one that resolves, and requires the +refusal to name that pin. A pin the target probes but the list leaves out is +a probe nothing watches, and deleting it keeps the control green. + **Three shared edits are tolerated, and are the whole list.** The port's column on [PORTING.md](PORTING.md), the techniques register, is written by hand — every technique carried, cited or stated impossible — and its gate diff --git a/make/checks/packet-arm-defaults.mk b/make/checks/packet-arm-defaults.mk index 5543015fe..6ce666af5 100644 --- a/make/checks/packet-arm-defaults.mk +++ b/make/checks/packet-arm-defaults.mk @@ -106,8 +106,8 @@ packet-arm-defaults-cs-negative-control: @cp test/cs/schematest.csproj $(ARM_NC_ROOT)/cs/schema/test/cs/ @cp test/cs/src/Program.cs $(ARM_NC_ROOT)/cs/schema/test/cs/src/ @ln -s "$(abspath $(SERIALIZE_CS))" $(ARM_NC_ROOT)/cs/serialize.cs - cd $(ARM_NC_ROOT)/cs/schema/test/cs && dotnet build --nologo -v quiet -o ../../bin/cs - $(call packet_arm_expect,cs,cd $(ARM_NC_ROOT)/cs/schema/test/cs && dotnet ../../bin/cs/schematest.dll,$(ARM_NC_MANAGED_FAILURE)) + cd $(ARM_NC_ROOT)/cs/schema/test/cs && $(DOTNET) build --nologo -v quiet -o ../../bin/cs + $(call packet_arm_expect,cs,cd $(ARM_NC_ROOT)/cs/schema/test/cs && $(DOTNET) ../../bin/cs/schematest.dll,$(ARM_NC_MANAGED_FAILURE)) packet-arm-defaults-java-negative-control: $(call packet_arm_prepare,java,internal/codegen/java/functions.go,java,java) diff --git a/make/cs.mk b/make/cs.mk index ad74e2704..23e19ba16 100644 --- a/make/cs.mk +++ b/make/cs.mk @@ -6,6 +6,21 @@ # test/cs/schematest.csproj and its ludicrous twin carry the same relative path SERIALIZE_CS ?= ../serialize.cs +# The .NET SDK, pinned per project the way every other leg's toolchain is. The +# SDK VERSION lives in .github/dotnet-version, which both workflows read and +# internal/ci gates; there is no unpacked copy under dist/ because the SDK +# installs itself into the machine, so this pin names the COMMAND and the +# toolchain gate holds it to resolving. Point it at another SDK with +# DOTNET=/path/to/dotnet. +DOTNET ?= dotnet + +# THE TOOLCHAIN GATE, this leg's half (issue #599; the Makefile's header and +# docs/CONTRIBUTING.md, "Adding a language"). The C# half of the block gates in +# the Makefile reads the same pin, so a dotnet that does not resolve stops the +# chain here rather than in the middle of a two-language control. +.PHONY: toolchain-cs +toolchain-cs: + @$(call toolchain_probe,cs,DOTNET,$(DOTNET)) # Packet defaults consume the shared C++ oracle in both build modes. build/packet-defaults/cs/.stamp: bin/schema test/packet-defaults/Defaults.schema test/packet-defaults/Plain.schema make/cs.mk ./bin/schema generate --lang cs --out build/packet-defaults/cs/defaults test/packet-defaults/Defaults.schema @@ -14,8 +29,8 @@ build/packet-defaults/cs/.stamp: bin/schema test/packet-defaults/Defaults.schema .PHONY: packet-defaults-cs packet-defaults-cs-negative-control packet-defaults-cs: build/packet-defaults/cs/.stamp packet-defaults-cpp - dotnet run --project test/packet-defaults/cs/packet-defaults.csproj -- testdata/wire/packet-defaults - dotnet run --configuration Release --project test/packet-defaults/cs/packet-defaults.csproj -- testdata/wire/packet-defaults + $(DOTNET) run --project test/packet-defaults/cs/packet-defaults.csproj -- testdata/wire/packet-defaults + $(DOTNET) run --configuration Release --project test/packet-defaults/cs/packet-defaults.csproj -- testdata/wire/packet-defaults packet-defaults-cs-negative-control: packet-defaults-cs @mkdir -p build/packet-defaults/cs-negative @@ -25,9 +40,9 @@ packet-defaults-cs-negative-control: packet-defaults-cs "$(CURDIR)" "$(CURDIR)" > build/packet-defaults/cs-negative/overlay.json go build -overlay=build/packet-defaults/cs-negative/overlay.json -o build/packet-defaults/cs-negative/schema ./cmd/schema ./build/packet-defaults/cs-negative/schema generate --lang cs --out build/packet-defaults/cs-negative/generated test/packet-defaults/Defaults.schema - dotnet build test/packet-defaults/cs/packet-defaults.csproj \ + $(DOTNET) build test/packet-defaults/cs/packet-defaults.csproj \ -p:DefaultsDir="$(CURDIR)/build/packet-defaults/cs-negative/generated" -o build/packet-defaults/cs-negative/checker - @if dotnet build/packet-defaults/cs-negative/checker/packet-defaults.dll testdata/wire/packet-defaults > build/packet-defaults/cs-negative/log 2>&1; then \ + @if $(DOTNET) build/packet-defaults/cs-negative/checker/packet-defaults.dll testdata/wire/packet-defaults > build/packet-defaults/cs-negative/log 2>&1; then \ echo 'NEGATIVE CONTROL FAILED: missing constructor bytes passed in C#'; exit 1; fi @grep -Fq 'FAILED: packet-default constructor bytes' build/packet-defaults/cs-negative/log || \ { echo 'NEGATIVE CONTROL FAILED: C# failed for another reason'; cat build/packet-defaults/cs-negative/log; exit 1; } @@ -157,11 +172,11 @@ tables-cs-refuses-pointers: bin/schema # million mutants each is ~23 s — inside the 60 s this gate is allowed and ten # times the shared N. `make ... N=` still overrides it. COOK_CS_N ?= 1000000 -COOK_CS := cd test/cs-cook && dotnet run --no-build -- +COOK_CS := cd test/cs-cook && $(DOTNET) run --no-build -- .PHONY: build-cs-cook build-cs-cook: build/tables-generated-cs/.stamp - cd test/cs-cook && dotnet build -v q --nologo + cd test/cs-cook && $(DOTNET) build -v q --nologo .PHONY: tables-cook-open-cs tables-cook-open-cs: build-cs-cook build/schema_test_cook build/cook-open/.stamp build/cook-open-fixed/.stamp @@ -185,8 +200,8 @@ tables-cook-open-cs: build-cs-cook build/schema_test_cook build/cook-open/.stamp $(COOK_CS) usage Scene ../../build/cook-open/Scene.cook $(COOK_CS) forge Scene ../../build/cook-open/Scene.cook $(COOK_CS) forge Depot ../../build/cook-open/Depot.cook - cd test/cs-cook && SEED=$(SEED) N=$(if $(filter-out 100000,$(N)),$(N),$(COOK_CS_N)) dotnet run --no-build -- fuzz Scene ../../build/cook-open/Scene.cook - cd test/cs-cook && SEED=$(SEED) N=$(if $(filter-out 100000,$(N)),$(N),$(COOK_CS_N)) dotnet run --no-build -- fuzz TreeNode ../../build/cook-open/TreeNode.cook + cd test/cs-cook && SEED=$(SEED) N=$(if $(filter-out 100000,$(N)),$(N),$(COOK_CS_N)) $(DOTNET) run --no-build -- fuzz Scene ../../build/cook-open/Scene.cook + cd test/cs-cook && SEED=$(SEED) N=$(if $(filter-out 100000,$(N)),$(N),$(COOK_CS_N)) $(DOTNET) run --no-build -- fuzz TreeNode ../../build/cook-open/TreeNode.cook $(COOK_CS) accept Scene ../../build/cook-open/Scene.cook # THE BYTE-ORDER LEG's C# half, and it is HALF: a cook written --byte-order # big is refused by the MAGIC here, which is the refusal the page promises. @@ -226,7 +241,7 @@ define cook_open_cs_sabotage "$(CURDIR)" "$(CURDIR)" > build/cook-open-cs-$(1)/overlay.json @go build -overlay=build/cook-open-cs-$(1)/overlay.json -o build/cook-open-cs-$(1)/schema ./cmd/schema ./build/cook-open-cs-$(1)/schema generate --lang cs --out build/cook-open-cs-$(1)/gen tables/pointers - @if ( cd test/cs-cook && dotnet build -v q --nologo \ + @if ( cd test/cs-cook && $(DOTNET) build -v q --nologo \ -p:CookGeneratedDir=../../build/cook-open-cs-$(1)/gen \ -p:BaseOutputPath=../../build/cook-open-cs-$(1)/bin/ \ -p:BaseIntermediateOutputPath=../../build/cook-open-cs-$(1)/obj/ \ @@ -234,7 +249,7 @@ define cook_open_cs_sabotage echo "NEGATIVE CONTROL FAILED: the sabotaged emitter's output does not compile"; \ cat build/cook-open-cs-$(1)/build.log; exit 1; \ fi - @if ( cd test/cs-cook && dotnet run --no-build \ + @if ( cd test/cs-cook && $(DOTNET) run --no-build \ -p:CookGeneratedDir=../../build/cook-open-cs-$(1)/gen \ -p:BaseOutputPath=../../build/cook-open-cs-$(1)/bin/ \ -p:BaseIntermediateOutputPath=../../build/cook-open-cs-$(1)/obj/ \ @@ -268,7 +283,7 @@ tables-cook-open-cs-walk-negative-control: build/cook-open/.stamp "$(CURDIR)" "$(CURDIR)" > build/cook-open-cs-walk/overlay.json @go build -overlay=build/cook-open-cs-walk/overlay.json -o build/cook-open-cs-walk/schema ./cmd/schema ./build/cook-open-cs-walk/schema generate --lang cs --out build/cook-open-cs-walk/gen tables/pointers - @if ( cd test/cs-cook && dotnet build -v q --nologo \ + @if ( cd test/cs-cook && $(DOTNET) build -v q --nologo \ -p:CookGeneratedDir=../../build/cook-open-cs-walk/gen \ -p:BaseOutputPath=../../build/cook-open-cs-walk/bin/ \ -p:BaseIntermediateOutputPath=../../build/cook-open-cs-walk/obj/ \ @@ -276,7 +291,7 @@ tables-cook-open-cs-walk-negative-control: build/cook-open/.stamp echo "NEGATIVE CONTROL FAILED: the sabotaged emitter's output does not compile"; \ cat build/cook-open-cs-walk/build.log; exit 1; \ fi - @if ( cd test/cs-cook && ITERATIONS=10 dotnet run --no-build \ + @if ( cd test/cs-cook && ITERATIONS=10 $(DOTNET) run --no-build \ -p:CookGeneratedDir=../../build/cook-open-cs-walk/gen \ -p:BaseOutputPath=../../build/cook-open-cs-walk/bin/ \ -p:BaseIntermediateOutputPath=../../build/cook-open-cs-walk/obj/ \ @@ -301,7 +316,7 @@ generated/bench/cs/.stamp: bin/schema $(SCHEMAS_BENCH) .PHONY: build-conformance-cs build-conformance-cs: build/tables-generated-cs/.stamp - cd test/conformance/cs && dotnet build -v q --nologo + cd test/conformance/cs && $(DOTNET) build -v q --nologo # THE NEGATIVE CONTROL FOR THE C# WALK (docs/SPEC-TABLES.md §16.5), and it is a # different sabotage from the C++ one above on purpose. That one flips a byte of @@ -358,7 +373,7 @@ tables-cs-leg: # (a unit that generates but does not compile is issue #80's lesson), and the # packet tests. .PHONY: test-cs -test-cs: build/tables-generated-cs/.stamp generated/bench/tables/cs/.stamp generated/cs/.stamp generated/cs-ludicrous/.stamp generated/bench/cs/.stamp +test-cs: toolchain-cs build/tables-generated-cs/.stamp generated/bench/tables/cs/.stamp generated/cs/.stamp generated/cs-ludicrous/.stamp generated/bench/cs/.stamp $(MAKE) tables-cs-json-walk $(MAKE) tables-cs-standalone $(MAKE) tables-cs-refuses-pointers @@ -368,13 +383,15 @@ test-cs: build/tables-generated-cs/.stamp generated/bench/tables/cs/.stamp gener $(MAKE) tables-cook-open-cs-lengths-negative-control $(MAKE) tables-cook-open-cs-root-negative-control $(MAKE) tables-cook-open-cs-walk-negative-control - dotnet build bench/tables/cs -c Release --nologo -v quiet - cd bench/cs && dotnet build -c Release --nologo -v quiet - cd test/cs && dotnet run - cd test/cs-ludicrous && dotnet run + $(DOTNET) build bench/tables/cs -c Release --nologo -v quiet + cd bench/cs && $(DOTNET) build -c Release --nologo -v quiet + cd test/cs && $(DOTNET) run + cd test/cs-ludicrous && $(DOTNET) run TEST_LEGS += test-cs -CONFORMANCE_LEGS += build-conformance-cs build-cs-cook +TOOLCHAIN_LEGS += cs +TOOLCHAIN_PINS_cs := DOTNET +CONFORMANCE_LEGS += $(call unless_skipped,cs,build-conformance-cs build-cs-cook) BENCH_TABLES_LEGS += generated/bench/tables/cs/.stamp GOLDENS_LEGS += update-goldens-cs # Packet UTF-8 content validation, including a compiled mutation control. @@ -384,9 +401,9 @@ build/packet-text/cs/.stamp: bin/schema test/packet-text/Narrow.schema .PHONY: packet-utf8-cs packet-utf8-cs-negative-control packet-utf8-cs: build/packet-text/cs/.stamp build/packet-text/cpp/driver build/packet-text/harness - dotnet build test/packet-text/cs/packet-text.csproj -c Debug -o build/packet-text/cs/debug --nologo + $(DOTNET) build test/packet-text/cs/packet-text.csproj -c Debug -o build/packet-text/cs/debug --nologo ./build/packet-text/harness dotnet build/packet-text/cs/debug/packet-text.dll - dotnet build test/packet-text/cs/packet-text.csproj -c Release -o build/packet-text/cs/release --nologo + $(DOTNET) build test/packet-text/cs/packet-text.csproj -c Release -o build/packet-text/cs/release --nologo ./build/packet-text/harness dotnet build/packet-text/cs/release/packet-text.dll packet-utf8-cs-negative-control: packet-utf8-cs @@ -394,7 +411,7 @@ packet-utf8-cs-negative-control: packet-utf8-cs go run ./tools/sabotage -name packet-utf8-cs-read -out build/packet-text/cs-negative/utf8.gotext internal/codegen/csharp/utf8.go @printf '{"Replace":{"%s/internal/codegen/csharp/utf8.go":"%s/build/packet-text/cs-negative/utf8.gotext"}}\n' "$(CURDIR)" "$(CURDIR)" > build/packet-text/cs-negative/overlay.json go run -overlay=build/packet-text/cs-negative/overlay.json ./cmd/schema generate --lang cs --out build/packet-text/cs-negative test/packet-text/Narrow.schema - dotnet build test/packet-text/cs/packet-text.csproj -c Release -o build/packet-text/cs-negative/bin -p:TextDir="$(CURDIR)/build/packet-text/cs-negative" --nologo + $(DOTNET) build test/packet-text/cs/packet-text.csproj -c Release -o build/packet-text/cs-negative/bin -p:TextDir="$(CURDIR)/build/packet-text/cs-negative" --nologo @if ./build/packet-text/harness -mutations-only dotnet build/packet-text/cs-negative/bin/packet-text.dll > build/packet-text/cs-negative/log 2>&1; then echo 'NEGATIVE CONTROL FAILED: C# UTF-8 removal passed'; exit 1; fi @grep -Fq 'FAILED: packet-text verdict on ' build/packet-text/cs-negative/log || { cat build/packet-text/cs-negative/log; exit 1; } @echo 'packet UTF-8 C# negative control: removed read validation fails bit-flip agreement' @@ -409,11 +426,11 @@ build/packet-wide/cs/.stamp: bin/schema build/packet-wide/source/WideText.schema .PHONY: packet-wide-cs packet-wide-cs-negative-control packet-wide-cs: build/packet-wide/cs/.stamp build/packet-wide/cpp/driver build/packet-text/harness - dotnet build test/packet-wide/cs/packet-wide.csproj -c Debug -o build/packet-wide/cs/debug --nologo - dotnet build/packet-wide/cs/debug/packet-wide.dll --contracts + $(DOTNET) build test/packet-wide/cs/packet-wide.csproj -c Debug -o build/packet-wide/cs/debug --nologo + $(DOTNET) build/packet-wide/cs/debug/packet-wide.dll --contracts ./build/packet-text/harness -wide -corpus testdata/conformance/text/wstring.txt -oracle build/packet-wide/cpp/driver dotnet build/packet-wide/cs/debug/packet-wide.dll - dotnet build test/packet-wide/cs/packet-wide.csproj -c Release -o build/packet-wide/cs/release --nologo - dotnet build/packet-wide/cs/release/packet-wide.dll --contracts + $(DOTNET) build test/packet-wide/cs/packet-wide.csproj -c Release -o build/packet-wide/cs/release --nologo + $(DOTNET) build/packet-wide/cs/release/packet-wide.dll --contracts ./build/packet-text/harness -wide -corpus testdata/conformance/text/wstring.txt -oracle build/packet-wide/cpp/driver dotnet build/packet-wide/cs/release/packet-wide.dll packet-wide-cs-negative-control: packet-wide-cs @@ -421,7 +438,7 @@ packet-wide-cs-negative-control: packet-wide-cs go run ./tools/sabotage -name packet-wide-cs-pairing -out build/packet-wide/cs-negative/wstring.gotext internal/codegen/csharp/wstring.go @printf '{"Replace":{"%s/internal/codegen/csharp/wstring.go":"%s/build/packet-wide/cs-negative/wstring.gotext"}}\n' "$(CURDIR)" "$(CURDIR)" > build/packet-wide/cs-negative/overlay.json go run -overlay=build/packet-wide/cs-negative/overlay.json ./cmd/schema generate --lang cs --out build/packet-wide/cs-negative build/packet-wide/source/WideText.schema - dotnet build test/packet-wide/cs/packet-wide.csproj -c Release -o build/packet-wide/cs-negative/bin -p:TextDir="$(CURDIR)/build/packet-wide/cs-negative" --nologo + $(DOTNET) build test/packet-wide/cs/packet-wide.csproj -c Release -o build/packet-wide/cs-negative/bin -p:TextDir="$(CURDIR)/build/packet-wide/cs-negative" --nologo @if ./build/packet-text/harness -wide -corpus testdata/conformance/text/wstring.txt -oracle build/packet-wide/cpp/driver -mutations-only dotnet build/packet-wide/cs-negative/bin/packet-wide.dll > build/packet-wide/cs-negative/log 2>&1; then echo 'NEGATIVE CONTROL FAILED: C# wide pairing removal passed'; exit 1; fi @grep -Fq 'FAILED: packet-text verdict on ' build/packet-wide/cs-negative/log || { cat build/packet-wide/cs-negative/log; exit 1; } @echo 'packet wide C# negative control: removed pairing fails bit-flip agreement' diff --git a/make/dart.mk b/make/dart.mk index a4e3c7e3b..811a02fb1 100644 --- a/make/dart.mk +++ b/make/dart.mk @@ -12,6 +12,13 @@ # unzip into dist/ and rename dart-sdk -> dart-sdk-3.13.2 DART ?= $(CURDIR)/dist/dart-sdk-3.13.2/bin/dart +# THE TOOLCHAIN GATE, this leg's half (issue #599; the Makefile's header and +# docs/CONTRIBUTING.md, "Adding a language"). This leg is the one the issue was +# opened over: a merge deleted the clone's dist link, the leg was passed over +# in silence, and the red inside it rode a green run. +.PHONY: toolchain-dart +toolchain-dart: + @$(call toolchain_probe,dart,DART,$(DART)) build/packet-defaults/dart/.stamp: bin/schema test/packet-defaults/Defaults.schema test/packet-defaults/Plain.schema make/dart.mk ./bin/schema generate --lang dart --out build/packet-defaults/dart/defaults test/packet-defaults/Defaults.schema ./bin/schema generate --lang dart --out build/packet-defaults/dart/plain test/packet-defaults/Plain.schema @@ -384,7 +391,7 @@ tables-dart-zero-cost: build/tables-generated-dart/.stamp # then the analyzer and the formatter over every generated tree, and the # packet tests, checked and compiled. .PHONY: test-dart -test-dart: generated/dart/.stamp generated/dart-ludicrous/.stamp generated/bench/dart/.stamp generated/bench/tables/dart/.stamp +test-dart: toolchain-dart generated/dart/.stamp generated/dart-ludicrous/.stamp generated/bench/dart/.stamp generated/bench/tables/dart/.stamp $(MAKE) tables-dart-clean $(MAKE) tables-dart-zero-cost $(MAKE) tables-dart-alloc DART_ALLOC_ITERATIONS=20000 @@ -408,7 +415,9 @@ test-dart: generated/dart/.stamp generated/dart-ludicrous/.stamp generated/bench cd test/dart-ludicrous && $(DART) compile exe -o ../../build/schema_test_dart_ludicrous main.dart >/dev/null && ../../build/schema_test_dart_ludicrous TEST_LEGS += test-dart -CONFORMANCE_LEGS += build/conformance-dart +TOOLCHAIN_LEGS += dart +TOOLCHAIN_PINS_dart := DART +CONFORMANCE_LEGS += $(call unless_skipped,dart,build/conformance-dart) BENCH_TABLES_LEGS += generated/bench/tables/dart/.stamp # Packet UTF-8 content validation, including a compiled mutation control. build/packet-text/dart/.stamp: bin/schema test/packet-text/Narrow.schema diff --git a/make/elixir.mk b/make/elixir.mk index ae15d09f7..0a3c7e811 100644 --- a/make/elixir.mk +++ b/make/elixir.mk @@ -22,6 +22,16 @@ ELIXIR ?= PATH="$(BEAM_PATH):$$PATH" elixir MIX ?= PATH="$(BEAM_PATH):$$PATH" mix ELIXIRC ?= PATH="$(BEAM_PATH):$$PATH" elixirc +# THE TOOLCHAIN GATE, this leg's half (issue #599; the Makefile's header and +# docs/CONTRIBUTING.md, "Adding a language"). The three pins carry an +# environment prefix rather than a path, so each probe resolves the launcher +# UNDER that prefix: BEAM_PATH pointing into an empty dist/ and elixir on PATH +# is what CI has, and it resolves, which is the state the gate has to pass. +.PHONY: toolchain-elixir +toolchain-elixir: + @$(call toolchain_probe,elixir,ELIXIR,$(ELIXIR),PATH="$(BEAM_PATH):$$PATH" command -v $(lastword $(ELIXIR))) + @$(call toolchain_probe,elixir,ELIXIRC,$(ELIXIRC),PATH="$(BEAM_PATH):$$PATH" command -v $(lastword $(ELIXIRC))) + @$(call toolchain_probe,elixir,MIX,$(MIX),PATH="$(BEAM_PATH):$$PATH" command -v $(lastword $(MIX))) build/packet-defaults/elixir/.stamp: bin/schema test/packet-defaults/Defaults.schema test/packet-defaults/Plain.schema make/elixir.mk ./bin/schema generate --lang elixir --out build/packet-defaults/elixir/defaults test/packet-defaults/Defaults.schema ./bin/schema generate --lang elixir --out build/packet-defaults/elixir/plain test/packet-defaults/Plain.schema @@ -365,7 +375,7 @@ conformance-negative-control-elixir: # lead gate; the hour is `make tables-elixir-soak` — then the format check and # the packet tests. .PHONY: test-elixir -test-elixir: generated/bench/tables/elixir/.stamp generated/elixir/.stamp generated/elixir-ludicrous/.stamp generated/bench/elixir/.stamp +test-elixir: toolchain-elixir generated/bench/tables/elixir/.stamp generated/elixir/.stamp generated/elixir-ludicrous/.stamp generated/bench/elixir/.stamp $(MAKE) tables-elixir-walk $(MAKE) conformance-negative-control-elixir $(MAKE) tables-elixir-alloc-audit @@ -376,8 +386,10 @@ test-elixir: generated/bench/tables/elixir/.stamp generated/elixir/.stamp genera cd test/elixir && $(ELIXIR) main.exs cd test/elixir-ludicrous && $(ELIXIR) main.exs -TEST_LEGS += test-elixir -CONFORMANCE_LEGS += build-conformance-elixir +TEST_LEGS += test-elixir +TOOLCHAIN_LEGS += elixir +TOOLCHAIN_PINS_elixir := ELIXIR ELIXIRC MIX +CONFORMANCE_LEGS += $(call unless_skipped,elixir,build-conformance-elixir) BENCH_TABLES_LEGS += generated/bench/tables/elixir/.stamp # Packet UTF-8 content validation, including a compiled mutation control. build/packet-text/elixir/.stamp: bin/schema test/packet-text/Narrow.schema diff --git a/make/java.mk b/make/java.mk index 87f6d7560..273edb495 100644 --- a/make/java.mk +++ b/make/java.mk @@ -15,6 +15,14 @@ JAVA ?= $(CURDIR)/dist/jdk-21.0.12.1/Contents/Home/bin/java JAVAC ?= $(CURDIR)/dist/jdk-21.0.12.1/Contents/Home/bin/javac +# THE TOOLCHAIN GATE, this leg's half (issue #599; the Makefile's header and +# docs/CONTRIBUTING.md, "Adding a language"). BOTH pins are probed: a JDK whose +# java runs and whose javac does not is a leg that compiles nothing and says so +# forty minutes in. +.PHONY: toolchain-java +toolchain-java: + @$(call toolchain_probe,java,JAVA,$(JAVA)) + @$(call toolchain_probe,java,JAVAC,$(JAVAC)) build/packet-defaults/java/.stamp: bin/schema test/packet-defaults/Defaults.schema test/packet-defaults/Plain.schema make/java.mk ./bin/schema generate --lang java --out build/packet-defaults/java/defaults test/packet-defaults/Defaults.schema ./bin/schema generate --lang java --out build/packet-defaults/java/plain test/packet-defaults/Plain.schema @@ -531,7 +539,7 @@ conformance-negative-control-java-block: build/conformance-harness # are `make tables-java-release`, because `make test` has no budget for them # (see that target). Then the packet tests, with and without -ea. .PHONY: test-java -test-java: generated/java/.stamp generated/java-ludicrous/.stamp generated/bench/java/.stamp build/java-test/.stamp build/java-test-ludicrous/.stamp build/java-bench/.stamp +test-java: toolchain-java generated/java/.stamp generated/java-ludicrous/.stamp generated/bench/java/.stamp build/java-test/.stamp build/java-test-ludicrous/.stamp build/java-bench/.stamp $(MAKE) conformance-negative-control-java $(MAKE) tables-java-compile $(MAKE) tables-java-json-walk @@ -548,9 +556,11 @@ test-java: generated/java/.stamp generated/java-ludicrous/.stamp generated/bench cd test/java-ludicrous && $(JAVA) -ea -cp ../../build/java-test-ludicrous Main cd test/java-ludicrous && $(JAVA) -cp ../../build/java-test-ludicrous Main -TEST_LEGS += test-java -CONFORMANCE_LEGS += build-conformance-java -CONFORMANCE_ENV += JAVA=$(JAVA) +TEST_LEGS += test-java +TOOLCHAIN_LEGS += java +TOOLCHAIN_PINS_java := JAVA JAVAC +CONFORMANCE_LEGS += $(call unless_skipped,java,build-conformance-java) +CONFORMANCE_ENV += JAVA=$(JAVA) BENCH_TABLES_LEGS += generated/bench/tables/java/.stamp # Packet UTF-8 content validation, including a compiled mutation control. build/packet-text/java/.stamp: bin/schema test/packet-text/Narrow.schema diff --git a/make/js.mk b/make/js.mk index 1bab07cea..db62f0b7f 100644 --- a/make/js.mk +++ b/make/js.mk @@ -26,6 +26,13 @@ NODE ?= $(CURDIR)/dist/node-v20.20.2-darwin-arm64/bin/node # pin from the environment and falls back to PATH export NODE +# THE TOOLCHAIN GATE, this leg's half (issue #599; the Makefile's header and +# docs/CONTRIBUTING.md, "Adding a language"). `make test` runs this before the +# chain starts and refuses by name when the pin does not resolve, because a leg +# that skips in silence is a leg whose red rides a green run. +.PHONY: toolchain-js +toolchain-js: + @$(call toolchain_probe,js,NODE,$(NODE)) build/packet-defaults/js/.stamp: bin/schema test/packet-defaults/Defaults.schema test/packet-defaults/Plain.schema make/js.mk ./bin/schema generate --lang js --out build/packet-defaults/js/defaults test/packet-defaults/Defaults.schema ./bin/schema generate --lang js --out build/packet-defaults/js/plain test/packet-defaults/Plain.schema @@ -456,7 +463,7 @@ conformance-negative-control-js: # controls, the conformance negative control, the runtime-home gate, and the # packet tests in both node modes. .PHONY: test-js -test-js: generated/js/.stamp generated/js-ludicrous/.stamp generated/bench/js/.stamp generated/bench/tables/js/.stamp +test-js: toolchain-js generated/js/.stamp generated/js-ludicrous/.stamp generated/bench/js/.stamp generated/bench/tables/js/.stamp $(MAKE) tables-js-json-walk $(MAKE) tables-js-standalone $(MAKE) tables-js-refuses-pointers @@ -477,7 +484,9 @@ test-js: generated/js/.stamp generated/js-ludicrous/.stamp generated/bench/js/.s cd test/js-ludicrous && node main.mjs && NODE_ENV=production node main.mjs TEST_LEGS += test-js -CONFORMANCE_LEGS += build/tables-generated-js/.stamp +TOOLCHAIN_LEGS += js +TOOLCHAIN_PINS_js := NODE +CONFORMANCE_LEGS += $(call unless_skipped,js,build/tables-generated-js/.stamp) BENCH_TABLES_LEGS += generated/bench/tables/js/.stamp # Both JavaScript packet tiers share the UTF-8 rule and mutation corpus. build/packet-text/js/.stamp: bin/schema test/packet-text/Narrow.schema diff --git a/test/conformance/harness/absence_test.go b/test/conformance/harness/absence_test.go index 06fa4a036..846fcedcc 100644 --- a/test/conformance/harness/absence_test.go +++ b/test/conformance/harness/absence_test.go @@ -130,13 +130,23 @@ func (c fakeCorpus) substituted(t *testing.T, lang string, leg fakeLeg) string { // runHarness runs the gate over this corpus and hands back what it printed. func (c fakeCorpus) runHarness(t *testing.T, drivers string) (string, bool) { t.Helper() - var out bytes.Buffer - ok, err := run(&out, c.m, c.manifest, c.jsonDir, c.reports, drivers, - filepath.Join(t.TempDir(), "work"), "") + out, ok, err := c.runHarnessSkipping(t, drivers, "") if err != nil { - t.Fatalf("harness run: %v\n%s", err, out.String()) + t.Fatalf("harness run: %v\n%s", err, out) } - return out.String(), ok + return out, ok +} + +// runHarnessSkipping is the same run with --skip filled in, which is what the +// Makefile passes from SCHEMA_SKIP_LEGS (issue #599). It hands the error back +// rather than failing on it, because REFUSING a skip is one of the behaviours +// under test (skip_test.go). +func (c fakeCorpus) runHarnessSkipping(t *testing.T, drivers, skip string) (string, bool, error) { + t.Helper() + var out bytes.Buffer + ok, err := run(&out, c.m, c.manifest, c.jsonDir, c.reports, drivers, + filepath.Join(t.TempDir(), "work"), "", skip) + return out.String(), ok, err } func needShell(t *testing.T) { diff --git a/test/conformance/harness/main.go b/test/conformance/harness/main.go index 848d0b14f..3fd7a82ba 100644 --- a/test/conformance/harness/main.go +++ b/test/conformance/harness/main.go @@ -74,6 +74,10 @@ flags: " " lines (a substituted one) --work scratch: fixtures, driver output, the derived manifest --only run one registered language (run only) + --skip registered languages this run does not exercise, comma + separated, each printed by name; the Makefile fills it + from SCHEMA_SKIP_LEGS and the reference leg is refused + (run only) --driver the leg to fuzz (wire-fuzz only) --seed --n the random pass (wire-fuzz only) --replay --unit --root [--message] @@ -95,6 +99,7 @@ func main() { drivers := fs.String("drivers", defaultDrivers, "the driver registry") work := fs.String("work", defaultWork, "scratch directory") only := fs.String("only", "", "run one registered language") + skip := fs.String("skip", "", "registered languages this run does not exercise, comma separated") driver := fs.String("driver", "", "the leg to fuzz, as a command (wire-fuzz only)") seed := fs.Uint64("seed", 24845619678, "the random pass's seed (wire-fuzz only)") n := fs.Int("n", 100000, "the random pass's mutant count (wire-fuzz only)") @@ -137,7 +142,7 @@ func main() { fatalf("%v", err) } case "run": - ok, err := run(os.Stdout, m, *manifest, *jsonDir, *reports, *drivers, *work, *only) + ok, err := run(os.Stdout, m, *manifest, *jsonDir, *reports, *drivers, *work, *only, *skip) if err != nil { fatalf("%v", err) } diff --git a/test/conformance/harness/run.go b/test/conformance/harness/run.go index 0cfc46530..e6631ec3f 100644 --- a/test/conformance/harness/run.go +++ b/test/conformance/harness/run.go @@ -339,11 +339,33 @@ func expectations(m *Manifest, surface string, reports map[string]Counts, jsonDi return out, nil } -func run(w io.Writer, m *Manifest, manifestPath, jsonDir, reportsPath, driversPath, work, only string) (bool, error) { +// skipSet reads --skip: the legs this run does not exercise, comma separated, +// which the Makefile fills from SCHEMA_SKIP_LEGS (issue #599). A skipped leg is +// PRINTED and left out, never passed over in silence, and the reference leg +// cannot be one: the corpus would lose its own expectation while every other +// leg kept comparing against nothing. +func skipSet(list string) (map[string]bool, error) { + out := map[string]bool{} + for s := range strings.SplitSeq(list, ",") { + if s = strings.TrimSpace(s); s != "" { + out[s] = true + } + } + if out[referenceLang] { + return nil, fmt.Errorf("--skip names the reference leg %s, which every other leg compares against", referenceLang) + } + return out, nil +} + +func run(w io.Writer, m *Manifest, manifestPath, jsonDir, reportsPath, driversPath, work, only, skip string) (bool, error) { drivers, discovered, err := loadDrivers(driversPath) if err != nil { return false, err } + skipped, err := skipSet(skip) + if err != nil { + return false, err + } reports, err := readReports(reportsPath) if err != nil { return false, err @@ -382,6 +404,12 @@ func run(w io.Writer, m *Manifest, manifestPath, jsonDir, reportsPath, driversPa if only != "" && d.lang != only { continue } + if skipped[d.lang] { + // on the same rule as the matrix below: a stdout that cannot be + // written to is not a verdict this gate can improve on + _, _ = io.WriteString(w, "conformance SKIPS the "+d.lang+" leg: --skip names it (SCHEMA_SKIP_LEGS)\n") + continue + } langs = append(langs, d.lang) results[d.lang] = map[string]*result{} diff --git a/test/conformance/harness/skip_test.go b/test/conformance/harness/skip_test.go new file mode 100644 index 000000000..47c8ff762 --- /dev/null +++ b/test/conformance/harness/skip_test.go @@ -0,0 +1,159 @@ +// A NAMED SKIP REACHES THE MATRIX, and the reference leg cannot be one +// (issue #599, `skipSet` and the driver loop in run.go). +// +// `make test SCHEMA_SKIP_LEGS=` becomes `--skip ` on the harness, +// and the whole point of naming a skip on purpose is that the run says which +// legs it did not measure rather than passing over them in silence. These +// tests read the EFFECT and not only the line: the skipped leg's driver would +// turn the run red if it were reached, so a skip that failed to skip cannot +// pass here, and a skip line printed for a leg that ran anyway cannot either. +// +// The corpus, the fake drivers and the run helpers are absence_test.go's +// (`fakeCorpus`), which is the one place in this package that drives `run` +// without a language leg. +package main + +import ( + "fmt" + "path/filepath" + "strings" + "testing" +) + +// loudLeg is what the driver of a skipped leg says on stderr when it is run. +// The harness folds a failing driver's stderr into the matrix, so this string +// appearing in the output is proof the leg was reached. +const loudLeg = "the %s driver was RUN" + +// withLoudLeg plants a DISCOVERED registry: the reference leg answering this +// corpus, and one port whose driver registers the wire surface and then exits +// 1 on it, saying so. A run that reaches that leg is RED and names it; a run +// that skips it by name is green. +func (c fakeCorpus) withLoudLeg(t *testing.T, lang string) string { + t.Helper() + dir := filepath.Join(t.TempDir(), "registry") + c.driverScript(t, filepath.Join(dir, referenceLang, "driver"), fakeLeg{}) + writeExec(t, filepath.Join(dir, lang, "driver"), fmt.Sprintf(`#!/bin/sh +if [ "$2" = list ]; then + echo wire + exit 0 +fi +echo %q >&2 +exit 1 +`, fmt.Sprintf(loudLeg, lang))) + return dir +} + +// TestUnskippedLegRuns is the control for the two below: without --skip, the +// loud leg is reached and the run is red. A skip test whose leg would have +// been green either way measures nothing. +func TestUnskippedLegRuns(t *testing.T) { + needShell(t) + c := newFakeCorpus(t) + out, ok := c.runHarness(t, c.withLoudLeg(t, "zz")) + if ok { + t.Errorf("the zz driver exited 1 on the wire surface and the harness stayed green:\n%s", out) + } + mustSay(t, out, []string{fmt.Sprintf(loudLeg, "zz"), "FAIL"}, []string{footer}) +} + +// TestNamedSkipIsPrintedAndTheLegIsNotRun is the rule itself: the leg is left +// out, the run is green without it, and the line that says so names the leg +// and where the name came from. +func TestNamedSkipIsPrintedAndTheLegIsNotRun(t *testing.T) { + needShell(t) + c := newFakeCorpus(t) + out, ok, err := c.runHarnessSkipping(t, c.withLoudLeg(t, "zz"), "zz") + if err != nil { + t.Fatalf("--skip zz was refused: %v\n%s", err, out) + } + if !ok { + t.Errorf("--skip zz named the only failing leg and the harness stayed red:\n%s", out) + } + mustSay(t, out, + []string{ + "conformance SKIPS the zz leg: --skip names it (SCHEMA_SKIP_LEGS)", + footer, + }, + []string{ + fmt.Sprintf(loudLeg, "zz"), // it was named, not merely tolerated + "FAIL", + }) + // and it is gone from the matrix's columns, not printed as an empty one: + // a column of blanks reads like a leg that answered nothing. + header := matrixHeader(t, out) + if strings.Contains(header, "zz") { + t.Errorf("the skipped leg still has a matrix column:\n%s", header) + } + if !strings.Contains(header, referenceLang) { + t.Errorf("the reference leg lost its matrix column:\n%s", header) + } +} + +// TestSkipIsMatchedByName holds the list to entry-by-entry reading: surrounding +// space and empty entries are not names, and a name no leg carries skips +// nothing rather than everything. +func TestSkipIsMatchedByName(t *testing.T) { + needShell(t) + + t.Run("space and empty entries are not names", func(t *testing.T) { + c := newFakeCorpus(t) + out, ok, err := c.runHarnessSkipping(t, c.withLoudLeg(t, "zz"), " , zz , ") + if err != nil { + t.Fatalf("--skip ' , zz , ' was refused: %v\n%s", err, out) + } + if !ok { + t.Errorf("--skip named zz around blanks and the harness stayed red:\n%s", out) + } + mustSay(t, out, + []string{"conformance SKIPS the zz leg: --skip names it (SCHEMA_SKIP_LEGS)", footer}, + []string{fmt.Sprintf(loudLeg, "zz")}) + }) + + t.Run("a name no leg carries skips nothing", func(t *testing.T) { + c := newFakeCorpus(t) + out, ok, err := c.runHarnessSkipping(t, c.withLoudLeg(t, "zz"), "yy") + if err != nil { + t.Fatalf("--skip yy was refused: %v\n%s", err, out) + } + if ok { + t.Errorf("--skip named yy and the zz leg was passed over anyway:\n%s", out) + } + mustSay(t, out, []string{fmt.Sprintf(loudLeg, "zz")}, []string{"conformance SKIPS", footer}) + }) +} + +// TestSkipRefusesTheReferenceLeg is the half a skip list without a floor would +// lose: every other leg compares against the reference, so skipping it leaves +// the matrix comparing against nothing while still printing a verdict. It is +// refused before any driver runs. +func TestSkipRefusesTheReferenceLeg(t *testing.T) { + needShell(t) + c := newFakeCorpus(t) + out, ok, err := c.runHarnessSkipping(t, c.withLoudLeg(t, "zz"), "zz,"+referenceLang) + if err == nil { + t.Fatalf("--skip named the reference leg %s and the harness ran anyway (ok=%v):\n%s", + referenceLang, ok, out) + } + for _, want := range []string{"--skip names the reference leg", referenceLang, "compares against"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("the refusal does not say %q: %v", want, err) + } + } + if out != "" { + t.Errorf("the reference skip was refused and the harness printed anyway:\n%s", out) + } +} + +// matrixHeader is the matrix's column line, the one place the run says which +// legs it measured. +func matrixHeader(t *testing.T, out string) string { + t.Helper() + for line := range strings.SplitSeq(out, "\n") { + if strings.HasPrefix(line, "surface ") { + return line + } + } + t.Fatalf("the run printed no matrix header:\n%s", out) + return "" +}