diff --git a/.github/workflows/kernel-ci-jobs.yml b/.github/workflows/kernel-ci-jobs.yml index 0953618523e805..32b832713dc7f6 100644 --- a/.github/workflows/kernel-ci-jobs.yml +++ b/.github/workflows/kernel-ci-jobs.yml @@ -218,9 +218,9 @@ jobs: retention-days: 14 if-no-files-found: ignore + # Needs nothing from plan, so it starts with it. host-tests: name: host tests (${{ matrix.group }}) - needs: plan runs-on: ubuntu-24.04 timeout-minutes: 45 permissions: @@ -228,7 +228,7 @@ jobs: strategy: fail-fast: false matrix: - group: [tools, aurora-sep] + group: [tools, aurora-sep, aurora-sep-slow] env: GROUP: ${{ matrix.group }} steps: diff --git a/tools/aurora-ci/host-tests.json b/tools/aurora-ci/host-tests.json index 6fcf5f811cd7c9..a1b93d695dfd6e 100644 --- a/tools/aurora-ci/host-tests.json +++ b/tools/aurora-ci/host-tests.json @@ -7,7 +7,8 @@ "commits: history the suite reads; host-tests.py --fetch-commits fetches it into a shallow checkout.", "Every test-like file under the discovery roots is listed in suites or excluded (tests/test_host_tests.py checks).", "helpers: files that look like suites but are run by a listed suite.", - "serial: the suite measures time; it runs alone, after the parallel batch." + "serial: the suite measures time; it runs alone, after the parallel batch.", + "group: the CI job that runs the suite; aurora-sep-slow holds the longest aurora-sep suites so they run beside the rest." ], "apt": [ "bison", @@ -1286,7 +1287,7 @@ }, { "name": "tools/aurora-sep/tests/test_m3_air_default.py", - "group": "aurora-sep", + "group": "aurora-sep-slow", "argv": [ "python3", "-m", @@ -1447,7 +1448,7 @@ }, { "name": "tools/aurora-sep/tests/test_m3_pro_mesa.py", - "group": "aurora-sep", + "group": "aurora-sep-slow", "argv": [ "python3", "-m", @@ -1512,7 +1513,7 @@ }, { "name": "tools/aurora-sep/tests/test_m3max_kit.py", - "group": "aurora-sep", + "group": "aurora-sep-slow", "argv": [ "python3", "-m", diff --git a/tools/aurora-ci/lookup-build.py b/tools/aurora-ci/lookup-build.py index 37bf6275fc13ac..fa52af3e293414 100755 --- a/tools/aurora-ci/lookup-build.py +++ b/tools/aurora-ci/lookup-build.py @@ -68,8 +68,11 @@ def kernel_id(repo, rev): def ensure_commit(repo, sha, remote="origin"): - """Fetch a commit's trees (no file contents) if the checkout lacks it.""" - if subprocess.run(["git", "-C", str(repo), "cat-file", "-e", f"{sha}^{{commit}}"], + """Fetch a commit's trees (no file contents) if the checkout lacks it. + + The check must not fetch: a partial clone fetches a missing object on + demand with all of its history, millions of kernel commits and trees.""" + if subprocess.run(["git", "-C", str(repo), "--no-lazy-fetch", "cat-file", "-e", f"{sha}^{{commit}}"], stderr=subprocess.DEVNULL).returncode: git(repo, "fetch", "--quiet", "--no-tags", "--depth=1", "--filter=blob:none", remote, sha) diff --git a/tools/aurora-ci/tests/test_host_tests.py b/tools/aurora-ci/tests/test_host_tests.py index 44fdf4978fba94..e92b85a6835892 100644 --- a/tools/aurora-ci/tests/test_host_tests.py +++ b/tools/aurora-ci/tests/test_host_tests.py @@ -136,6 +136,16 @@ def setUp(self): super().setUp() self.manifest = host.load_manifest() + def workflow_groups(self): + """The groups the host-tests job's matrix runs.""" + workflow = (TREE / ".github/workflows/kernel-ci-jobs.yml").read_text() + job = workflow.split("\n host-tests:\n", 1)[1].split("\n self-test:\n", 1)[0] + return re.search(r"^ +group: \[(.*)\]$", job, re.M).group(1).replace(" ", "").split(",") + + def test_the_workflow_runs_every_group(self): + groups = {suite.get("group", "tools") for suite in self.manifest["suites"]} + self.assertEqual(sorted(groups), sorted(self.workflow_groups())) + def test_every_test_like_file_is_listed_or_excluded(self): listed = set(self.manifest["excluded"]) | set(self.manifest.get("helpers", {})) for suite in self.manifest["suites"]: @@ -157,7 +167,7 @@ def test_entries_are_well_formed(self): for suite in self.manifest["suites"]: with self.subTest(suite=suite["name"]): self.assertTrue(suite["argv"]) - self.assertIn(suite.get("group", "tools"), ("tools", "aurora-sep")) + self.assertIn(suite.get("group", "tools"), self.workflow_groups()) for commit in suite.get("commits", []): self.assertRegex(commit, r"^[0-9a-f]{40}$") if "known_failure" in suite: diff --git a/tools/aurora-ci/tests/test_lookup_build.py b/tools/aurora-ci/tests/test_lookup_build.py index a2b93af97e67d7..a70a3e4ef50819 100644 --- a/tools/aurora-ci/tests/test_lookup_build.py +++ b/tools/aurora-ci/tests/test_lookup_build.py @@ -1,4 +1,6 @@ # SPDX-License-Identifier: GPL-2.0 +import subprocess + from support import TempDirTestCase, commit_file, git, load lookup = load("lookup-build") @@ -72,6 +74,28 @@ def test_ci_changes_do_not_change_the_key(self): self.assertEqual(self.key(), before) +class EnsureCommitTest(TempDirTestCase): + def present(self, repo, sha): + return subprocess.run(["git", "-C", str(repo), "--no-lazy-fetch", "cat-file", "-e", f"{sha}^{{commit}}"], + stderr=subprocess.DEVNULL).returncode == 0 + + def test_fetches_only_the_missing_commit_into_a_partial_clone(self): + # The plan job's checkout: depth 1, blob:none. Asking a partial clone + # for a missing object fetches it with all of its history. + origin = self.init_repo("origin") + first = commit_file(origin, "Makefile", "1\n", "first") + base = commit_file(origin, "Makefile", "2\n", "base") + commit_file(origin, "Makefile", "3\n", "head") + git(origin, "config", "uploadpack.allowFilter", "true") + git(origin, "config", "uploadpack.allowAnySHA1InWant", "true") + clone = self.tmp / "clone" + git(self.tmp, "clone", "--quiet", "--depth=1", "--filter=blob:none", "--no-checkout", + origin.as_uri(), str(clone)) + lookup.ensure_commit(clone, base) + self.assertTrue(self.present(clone, base)) + self.assertFalse(self.present(clone, first)) + + REPO = 1000