Skip to content

Commit 4a8c86f

Browse files
CI: select members by package identity, not by substring (#573)
* CI selects the members whose declared graph holds a changed package identity The select step searched each member's manifest for a descriptor's short name as a substring: pkgs/o/os.lua selected 113 of 205 members, and members reaching compat.openssl through asio or curl were never selected. tests/select_members.py reads members' dependencies as mcpp does, follows descriptors' deps and the manifests inside packages' archives (cached by actions/cache, the archives being pinned), and selects by identity. Co-authored-by: speak-agent <248744407+speak-agent@users.noreply.github.com> * select: the selection's tests run on every select Co-authored-by: speak-agent <248744407+speak-agent@users.noreply.github.com> --------- Co-authored-by: speak-agent <248744407+speak-agent@users.noreply.github.com>
1 parent cdaaeb6 commit 4a8c86f

5 files changed

Lines changed: 618 additions & 9 deletions

File tree

‎.github/workflows/validate.yml‎

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -704,6 +704,16 @@ jobs:
704704
fetch-depth: 0
705705
- name: Install lua
706706
run: sudo apt-get install -y --no-install-recommends lua5.4
707+
# The dependencies stated by the manifests inside packages' archives,
708+
# read by tests/select_members.py. An archive is pinned by its sha256,
709+
# so an entry never goes stale; a new key per descriptor set keeps the
710+
# entries of the previous one and adds the archives it adds.
711+
- name: Restore the archive manifests read before
712+
uses: actions/cache@v4
713+
with:
714+
path: ${{ runner.temp }}/select-manifests.json
715+
key: select-manifests-${{ hashFiles('pkgs/**/*.lua') }}
716+
restore-keys: select-manifests-
707717
# ── Selective member testing ──────────────────────────────────────
708718
# `mcpp test --workspace` builds every member (opencv, ffmpeg, …) and
709719
# dominates CI wall-clock, while a PR almost always touches one
@@ -830,7 +840,7 @@ jobs:
830840
changed=""
831841
fi
832842
printf 'changed files vs %s:\n%s\n' "${base:-<none>}" "$changed"
833-
sel=""; pkgsel=""
843+
sel=""; pkgsel=""; descriptors=""
834844
add() { case " $sel " in *" $1 "*) ;; *) sel="$sel $1" ;; esac; }
835845
while IFS= read -r f; do
836846
[ -n "$f" ] || continue
@@ -856,15 +866,11 @@ jobs:
856866
if [ -d "tests/examples/$m" ] || [ -d "tests/mwrap/$m" ]; then add "$m"; fi ;;
857867
pkgs/*.lua|pkgs/*/*.lua)
858868
# A DELETED descriptor is in the diff but not on disk, and its
859-
# CN urls are gone with it — nothing left to reach.
869+
# CN urls are gone with it — nothing left to reach. The members
870+
# it reaches are selected after the loop, all descriptors at
871+
# once (tests/select_members.py).
860872
if [ -f "$f" ]; then pkgsel="$pkgsel $f"; fi
861-
lib=$(basename "$f" .lua); lib=${lib##*.}
862-
hit=0
863-
for mt in tests/examples/*/mcpp.toml tests/mwrap/*/mcpp.toml; do
864-
[ -f "$mt" ] || continue
865-
if grep -Fq "$lib" "$mt"; then add "$(basename "$(dirname "$mt")")"; hit=1; fi
866-
done
867-
[ "$hit" = 1 ] || echo "note: no workspace member exercises $f" ;;
873+
descriptors="$descriptors $f" ;;
868874
# tools/ holds OFFLINE descriptor-generation and publishing
869875
# helpers (tools/compat-*/, tools/gtc/, publish_mcpp_index.sh).
870876
# Nothing under it is consumed by a package build: when one of
@@ -890,6 +896,18 @@ jobs:
890896
done <<EOF
891897
$changed
892898
EOF
899+
# The members a changed descriptor reaches: those whose declared
900+
# dependency graph holds its identity, directly or through other
901+
# packages of this index (tests/select_members.py says how the
902+
# graph is read). It replaced a substring search for the short name,
903+
# under which pkgs/o/os.lua selected 113 members and two consumed it.
904+
python3 tests/test_select_members.py
905+
if [ -n "$descriptors" ]; then
906+
for m in $(python3 tests/select_members.py ${base:+--base "$base"} \
907+
--cache "$RUNNER_TEMP/select-manifests.json" $descriptors); do
908+
add "$m"
909+
done
910+
fi
893911
sel=${sel# }
894912
echo "MEMBERS=$sel" >> "$GITHUB_ENV"
895913
echo "selected members: ${sel:-<none>}"

‎CHANGELOG.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,13 @@
1616

1717
### Changed
1818

19+
- **CI 按包身份选择受影响的成员。** `validate.yml` 的 select 曾在成员清单中查找描述符短名的
20+
子串:`pkgs/o/os.lua` 选中 205 个成员中的 113 个(`boost-*`、`macos`、`cfg(os = ...)`、
21+
注释里的 `exists`),而经 asio、curl 依赖 `compat.openssl` 的成员一个也未选中。
22+
`tests/select_members.py` 以 mcpp 的规则读取成员声明的 `(namespace, name)`,沿描述符的
23+
`deps` 与包归档内清单(按 tag 读 raw.githubusercontent.com;release 资产则读归档本身,
24+
并经 `path` 依赖到达同一归档的其他清单)陈述的依赖求传递闭包,选出依赖图含被修改身份的成员。
25+
归档以 sha256 固定,已读结果由 `actions/cache` 保存。读不到的清单记为警告,由每周全量运行覆盖。
1926
- **`compat.sycl-runtime` 在 Linux 上要求 `mcpp:c++-abi=libstdc++`。** `libsycl.so` 在 libstdc++
2027
上抛出 `sycl::exception`,而一个进程中的两套 C++ 运行时之间带类型的 `catch` 不会匹配
2128
(mcpp-community/mcpp#838)。gcc 工具链,或 llvm 加 `gcc.libstdcxx`,满足该要求;其他构建

‎tests/descriptor_identities.lua‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
-- Print what each descriptor declares about itself and about its neighbours:
2+
--
3+
-- <file> TAB <namespace> TAB <name> TAB <dependency>,... TAB <archive>#<manifest> ...
4+
--
5+
-- A dependency is a key of any `deps` table inside `package`, spelled as the
6+
-- descriptor spells it (`compat.openssl`, `compat:openssl`, `zlib`); the
7+
-- reader of this output (tests/select_members.py) resolves the spelling
8+
-- against the identities printed here.
9+
--
10+
-- The last column is where the rest of the package's dependencies are stated
11+
-- when the descriptor does not state them: a package whose `mcpp` is not an
12+
-- inline table builds from the manifest inside its archive (`mcpp =
13+
-- "*/ld/mcpp.toml"`, or `mcpp.toml` at the archive's root when the field is
14+
-- absent). One entry per version: the archive's GLOBAL url, `#`, and the
15+
-- manifest's path inside it, prefixed with `?` when the field is absent --
16+
-- the archive may then hold no manifest at all (a package built by xpm, or
17+
-- one whose install() writes it). A descriptor that does not load prints no line.
18+
--
19+
-- Usage: lua5.4 tests/descriptor_identities.lua pkgs/*/*.lua
20+
21+
function import(...)
22+
return setmetatable({}, {__index = function() return function() end end})
23+
end
24+
25+
local function collect(t, out, seen)
26+
if seen[t] then return end
27+
seen[t] = true
28+
for k, v in pairs(t) do
29+
if k == "deps" and type(v) == "table" then
30+
for dk, dv in pairs(v) do
31+
if type(dk) == "string" then out[dk] = true
32+
elseif type(dv) == "string" then out[dv] = true end
33+
end
34+
end
35+
if type(v) == "table" then collect(v, out, seen) end
36+
end
37+
end
38+
39+
for i = 1, #arg do
40+
package = nil
41+
local chunk = loadfile(arg[i], "t")
42+
if chunk then
43+
-- `package = { ... }` is the first statement, so it is populated even
44+
-- if a later top-level line fails under the stubbed `import`.
45+
pcall(chunk)
46+
local p = package
47+
if type(p) == "table" and type(p.name) == "string" then
48+
local deps = {}
49+
collect(p, deps, {})
50+
local list = {}
51+
for d in pairs(deps) do list[#list + 1] = d end
52+
table.sort(list)
53+
local manifests = {}
54+
if type(p.mcpp) ~= "table" then
55+
local inside = "?mcpp.toml"
56+
if type(p.mcpp) == "string" then inside = (p.mcpp:gsub("^%*/", "")) end
57+
local seen = {}
58+
for _, versions in pairs(type(p.xpm) == "table" and p.xpm or {}) do
59+
for _, entry in pairs(type(versions) == "table" and versions or {}) do
60+
local url = type(entry) == "table" and entry.url
61+
if type(url) == "table" then url = url.GLOBAL end
62+
if type(url) == "string" and not seen[url] then
63+
seen[url] = true
64+
manifests[#manifests + 1] = url .. "#" .. inside
65+
end
66+
end
67+
end
68+
table.sort(manifests)
69+
end
70+
print(table.concat({ arg[i],
71+
type(p.namespace) == "string" and p.namespace or "",
72+
p.name, table.concat(list, ","),
73+
table.concat(manifests, " ") }, "\t"))
74+
end
75+
end
76+
end

0 commit comments

Comments
 (0)