diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml index 6d00df773b..6bafc06e34 100644 --- a/.github/workflows/integration_tests.yml +++ b/.github/workflows/integration_tests.yml @@ -14,8 +14,20 @@ on: default: true secrets: CODECOV_TOKEN: { required: false } +env: + # Number of parallel shards each compression config is split into. Every + # top-level test lands in exactly one shard, so the shards together are the + # full suite — keep this in sync with strategy.matrix.shard below. + # + # Deliberately 1: this wires the sharding up without spending anything. The + # infra-tests pool is saturated, and measured at 4 shards the extra jobs + # spent their savings queueing for runners. Raise it (and the shard list) to + # spend new capacity once #3352 lands. + SHARDS: "1" + jobs: run: + name: run (${{ matrix.name }}, shard ${{ matrix.shard }}) if: ${{ inputs.run-tests == true }} runs-on: infra-tests timeout-minutes: 30 @@ -25,6 +37,8 @@ jobs: strategy: fail-fast: false matrix: + name: [uncompressed, zstd1, lz4] + shard: [1] include: - name: uncompressed compress_enabled: "false" @@ -86,6 +100,7 @@ jobs: TESTS_ORCHESTRATOR_HOST: "localhost:5008" TESTS_ENVD_PROXY: "http://localhost:3002" TESTS_CLIENT_PROXY: "http://localhost:3002" + SHARD: ${{ matrix.shard }} run: | # Run the integration tests make test-integration @@ -123,7 +138,7 @@ jobs: if: ${{ always() && inputs.publish == true }} uses: actions/upload-artifact@v6 with: - name: Integration Tests Results (${{ matrix.name }}) + name: Integration Tests Results (${{ matrix.name }} ${{ matrix.shard }}) path: ./tests/integration/test-results.xml - name: Upload test results to Codecov @@ -140,7 +155,7 @@ jobs: if: ${{ always() && inputs.publish == true }} uses: actions/upload-artifact@v6 with: - name: Service Logs (${{ matrix.name }}) + name: Service Logs (${{ matrix.name }} ${{ matrix.shard }}) path: ~/logs/*.log integration_tests: diff --git a/tests/integration/Makefile b/tests/integration/Makefile index b1492223b3..5775c62b18 100644 --- a/tests/integration/Makefile +++ b/tests/integration/Makefile @@ -21,6 +21,15 @@ seed: go run seed.go @echo "Done" +# Tunables, overridable from the environment (CI sets SHARDS/SHARD). +# SHARDS > 1 runs only shard SHARD of the selected packages; the shards +# partition the suite, so the union of all SHARDS runs is the full suite. +TEST_PARALLEL := $(or $(strip $(TEST_PARALLEL)),4) +TEST_TIMEOUT := $(or $(strip $(TEST_TIMEOUT)),20m) +JUNIT_FILE := $(or $(strip $(JUNIT_FILE)),test-results.xml) +SHARDS := $(or $(strip $(SHARDS)),1) +SHARD := $(or $(strip $(SHARD)),1) + .PHONY: test test: test/. test/%: @@ -35,14 +44,24 @@ test/%: export TESTS_SANDBOX_USER_ID=$(TESTS_SANDBOX_USER_ID); \ go test -v ./internal/main_test.go -count=1 && \ TEST_PATH="./internal/tests/$(subst test/,,$@)"; \ + TEST_PATH="$${TEST_PATH%/.}"; \ + RUN=""; \ case "$${TEST_PATH}" in \ - *.go:*) \ - BASE=$${TEST_PATH%%:*}; \ - TEST_FN=$${TEST_PATH#*:}; \ - go tool gotestsum --rerun-fails=1 --packages="$$BASE" --format standard-verbose --junitfile=test-results.xml -- -count=1 -parallel=4 -timeout=20m -run "$${TEST_FN}" ;; \ - *.go) go tool gotestsum --rerun-fails=1 --packages="$$TEST_PATH" --format standard-verbose --junitfile=test-results.xml -- -count=1 -parallel=4 -timeout=20m ;; \ - *) go tool gotestsum --rerun-fails=1 --packages="$$TEST_PATH/..." --format standard-verbose --junitfile=test-results.xml -- -count=1 -parallel=4 -timeout=20m ;; \ - esac + *.go:*) PACKAGES="$${TEST_PATH%%:*}"; RUN="$${TEST_PATH#*:}" ;; \ + *.go) PACKAGES="$${TEST_PATH}" ;; \ + *) PACKAGES="$${TEST_PATH}/..." ;; \ + esac; \ + if [ "$(SHARDS)" -gt 1 ]; then RUN=$$(./scripts/select-tests.sh $(SHARD) $(SHARDS) "$${PACKAGES}") || exit 1; fi; \ + if [ -n "$${RUN}" ]; then set -- -run "$${RUN}"; else set --; fi; \ + go tool gotestsum --rerun-fails=1 --packages="$${PACKAGES}" --format standard-verbose --junitfile=$(JUNIT_FILE) \ + -- -count=1 -parallel=$(TEST_PARALLEL) -timeout=$(TEST_TIMEOUT) "$$@" + +# Refresh the shard-balancing weights from the JUnit files of a full CI run: +# gh run download -p 'Integration Tests Results*' -D /tmp/junit +# make -C tests/integration update-test-weights JUNIT_DIR=/tmp/junit +.PHONY: update-test-weights +update-test-weights: + go run scripts/update-test-weights.go $(JUNIT_DIR) .PHONY: connect-orchestrator connect-orchestrator: diff --git a/tests/integration/README.md b/tests/integration/README.md index 10dc8abf53..36319d254d 100644 --- a/tests/integration/README.md +++ b/tests/integration/README.md @@ -9,6 +9,23 @@ Package for defining integration tests. Currently, there is a setup for API and 3. If necessary, run `make connect-orchestrator` to create a tunnel to one orchestrator client VM in GCP (you may need to run `make setup-ssh` the first time) 4. Run `make test` in this folder or `make test-integration` from the root `infra/` folder. +Narrow the run with `make test/`, e.g. `make test/api/templates` +or `make test/api/templates/build_template_test.go:TestTemplateBuildCOPY`. + +## Sharding + +A compression config can be split across parallel CI jobs, each running one +shard: `SHARDS=2 SHARD=1 make test`. `scripts/select-tests.sh` enumerates the +top-level tests from the source tree and bin-packs each package across the +shards using the recorded per-test times in `scripts/test-weights.tsv`, so every +test runs in exactly one shard and the shards finish at roughly the same time. +Enumeration happens at run time, so a newly added test always lands in exactly +one shard; a test missing from the weights file still runs, it just gets a +median-time estimate for balancing. Refresh the weights with +`make update-test-weights JUNIT_DIR=…` when the suite's shape changes. + +CI ships `SHARDS: 1`, i.e. sharding off — see the note in the workflow. + ## Usage of clients (api, orchestrator, envd) All tests are in the folder internal/tests. You can see the usage of different clients in the tests. Here are just basics. diff --git a/tests/integration/scripts/select-tests.sh b/tests/integration/scripts/select-tests.sh new file mode 100755 index 0000000000..2ebe8d3094 --- /dev/null +++ b/tests/integration/scripts/select-tests.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash +# Print the `go test -run` regex selecting shard $1 of $2 for the given packages. +# +# select-tests.sh ... +# +# Every top-level test is enumerated from the source tree at run time, so a +# newly added test always lands in exactly one shard: the shards partition the +# suite, they never filter it. test-weights.tsv only decides *which* shard a +# test goes to; a missing entry costs balance, never coverage. +# +# Balancing is per package: each package's tests are bin-packed (longest +# processing time first) across all shards, so every shard gets a slice of +# every package and keeps the cross-package parallelism `go test` gives us. +set -euo pipefail + +if [ "$#" -lt 3 ]; then + echo "usage: $0 ..." >&2 + exit 2 +fi + +shard=$1 +shards=$2 +shift 2 + +script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +weights=${script_dir}/test-weights.tsv + +if [ "$shard" -lt 1 ] || [ "$shard" -gt "$shards" ]; then + echo "shard $shard out of range 1..$shards" >&2 + exit 2 +fi + +dirs=() +for pkg in "$@"; do + pkg=${pkg#./} + case "$pkg" in + */...) + while IFS= read -r dir; do + dirs+=("${dir#./}") + done < <(find "${pkg%/...}" -type d | sort) + ;; + *) dirs+=("$pkg") ;; + esac +done + +# Median of the known weights, used for tests that predate the last refresh of +# test-weights.tsv (or were added since). +default_weight=$(awk -F'\t' '!/^#/ && NF == 3 { print $3 }' "$weights" | sort -n | + awk '{ v[NR] = $1 } END { if (NR) print v[int((NR + 1) / 2)]; else print 30 }') + +names=$( + for dir in "${dirs[@]}"; do + compgen -G "${dir}/*_test.go" >/dev/null || continue + sed -n 's/^func \(Test[A-Za-z0-9_]*\)(.*/\1/p' "${dir}"/*_test.go | + sort -u | sed "s|^|${dir}\t|" + done +) + +if [ -z "$names" ]; then + echo "no tests found in: $*" >&2 + exit 1 +fi + + +# Attach a weight to every test, then hand the list to the packer sorted by +# package, weight descending, name — the input order LPT bin-packing needs. +printf '%s\n' "$names" | + awk -F'\t' -v weights="$weights" -v def="$default_weight" ' + BEGIN { while ((getline line < weights) > 0) { n = split(line, f, "\t"); if (line !~ /^#/ && n == 3) w[f[1] SUBSEP f[2]] = f[3] } } + { key = $1 SUBSEP $2; print $1 "\t" ((key in w) ? w[key] : def) "\t" $2 } + ' | + sort -t"$(printf '\t')" -k1,1 -k2,2nr -k3,3 | + awk -F'\t' -v shard="$shard" -v shards="$shards" ' + function reset(i) { for (i = 1; i <= shards; i++) load[i] = 0 } + BEGIN { reset(); pkg = "" } + { + if ($1 != pkg) { pkg = $1; reset() } + best = 1 + for (i = 2; i <= shards; i++) if (load[i] < load[best]) best = i + load[best] += $2 + if (best == shard) out = out (out ? "|" : "") $3 + } + END { + if (out == "") { print "shard " shard " of " shards " matched no tests" > "/dev/stderr"; exit 1 } + print "^(" out ")$" + } + ' diff --git a/tests/integration/scripts/test-weights.tsv b/tests/integration/scripts/test-weights.tsv new file mode 100644 index 0000000000..c6331e75de --- /dev/null +++ b/tests/integration/scripts/test-weights.tsv @@ -0,0 +1,230 @@ +# package test seconds +# Slowest-of-all-compression-configs wall time per top-level test, used only to +# balance CI shards (see shard-tests.sh). Regenerate with: make -C tests/integration update-test-weights +internal/tests TestBannedTeam 0.0 +internal/tests TestBlockedTeam 0.1 +internal/tests/api TestAPIKeyLastUsedUpdated 0.0 +internal/tests/api TestHealth 0.0 +internal/tests/api/metrics TestSandboxListMetrics 13.0 +internal/tests/api/metrics TestSandboxMetrics 13.2 +internal/tests/api/metrics TestTeamMetrics 3.2 +internal/tests/api/metrics TestTeamMetricsEmpty 0.1 +internal/tests/api/metrics TestTeamMetricsInvalidDate 0.0 +internal/tests/api/metrics TestTeamMetricsMaxConcurrentSandboxes 3.1 +internal/tests/api/metrics TestTeamMetricsMaxEmpty 0.0 +internal/tests/api/metrics TestTeamMetricsMaxSandboxStartRate 6.6 +internal/tests/api/metrics TestTeamMetricsWithTimeRange 6.7 +internal/tests/api/sandboxes TestCreateSandboxWithDisabledPublicTrafficAndDisabledEnvdSecure 0.0 +internal/tests/api/sandboxes TestCreateSandboxWithSecuredEnvd 1.6 +internal/tests/api/sandboxes TestEgressFirewallAllowAll 132.6 +internal/tests/api/sandboxes TestEgressFirewallAllowAllDomainsWildcard 109.1 +internal/tests/api/sandboxes TestEgressFirewallAllowAllDuplicate 2.8 +internal/tests/api/sandboxes TestEgressFirewallAllowAndBlockCombination 161.7 +internal/tests/api/sandboxes TestEgressFirewallAllowCIDRRange 72.8 +internal/tests/api/sandboxes TestEgressFirewallAllowDomainAndIP 131.6 +internal/tests/api/sandboxes TestEgressFirewallAllowDomainThroughBlockedInternet 161.6 +internal/tests/api/sandboxes TestEgressFirewallAllowOverridesBlock 133.0 +internal/tests/api/sandboxes TestEgressFirewallAllowSpecificIP 66.2 +internal/tests/api/sandboxes TestEgressFirewallAllowWildcardDomainThroughBlockedInternet 164.6 +internal/tests/api/sandboxes TestEgressFirewallBlockCIDRRange 8.9 +internal/tests/api/sandboxes TestEgressFirewallBlockSpecificIP 185.5 +internal/tests/api/sandboxes TestEgressFirewallDNSSpoofingNeutralized 1.6 +internal/tests/api/sandboxes TestEgressFirewallDomainCaseInsensitive 1.5 +internal/tests/api/sandboxes TestEgressFirewallDomainPersistsAfterResume 168.5 +internal/tests/api/sandboxes TestEgressFirewallEmptyConfig 201.2 +internal/tests/api/sandboxes TestEgressFirewallExactDomainMatchVsSubdomain 4.3 +internal/tests/api/sandboxes TestEgressFirewallHTTPDomainFiltering 131.2 +internal/tests/api/sandboxes TestEgressFirewallHTTPSByIPNoHostname 163.6 +internal/tests/api/sandboxes TestEgressFirewallMultipleAllowedIPs 200.7 +internal/tests/api/sandboxes TestEgressFirewallPersistsAfterResume 141.7 +internal/tests/api/sandboxes TestEgressFirewallPrivateIPRangesAlwaysBlocked 16.5 +internal/tests/api/sandboxes TestEgressFirewallRegularIPThenAllowAll 3.3 +internal/tests/api/sandboxes TestEgressFirewallUDPAllowedCIDR 173.9 +internal/tests/api/sandboxes TestEgressFirewallUDPAllowedIP 165.5 +internal/tests/api/sandboxes TestEgressFirewallWithInternetAccessFalse 201.3 +internal/tests/api/sandboxes TestFuseDevicePermissions 2.1 +internal/tests/api/sandboxes TestFuseNonRootAccess 0.6 +internal/tests/api/sandboxes TestGPGKeyserverWorks 159.5 +internal/tests/api/sandboxes TestInternetAccess 10.8 +internal/tests/api/sandboxes TestInternetAccessResumedSbx 30.1 +internal/tests/api/sandboxes TestNoNetworkConfig_SSHWorks 1.8 +internal/tests/api/sandboxes TestSandboxAutoPausePauseResume 23.9 +internal/tests/api/sandboxes TestSandboxAutoPauseResumePersisted 7.2 +internal/tests/api/sandboxes TestSandboxAutoPause_FilesystemOnly 13.8 +internal/tests/api/sandboxes TestSandboxConnect 33.6 +internal/tests/api/sandboxes TestSandboxConnect_CrossTeamAccess_Paused 2.5 +internal/tests/api/sandboxes TestSandboxConnect_CrossTeamAccess_Running 1.1 +internal/tests/api/sandboxes TestSandboxConnect_FilesystemOnlyResumes 7.4 +internal/tests/api/sandboxes TestSandboxCreate 1.1 +internal/tests/api/sandboxes TestSandboxCreate_FilesystemOnlyAutoPauseRejectsAutoResume 0.0 +internal/tests/api/sandboxes TestSandboxCreate_FilesystemOnlyAutoPauseRequiresAutoPause 0.0 +internal/tests/api/sandboxes TestSandboxDetailPaused 2.5 +internal/tests/api/sandboxes TestSandboxDetailPausingSandbox 3.0 +internal/tests/api/sandboxes TestSandboxDetailReturnsLifecycleAndNetworkConfig 2.3 +internal/tests/api/sandboxes TestSandboxDetailRunning 0.9 +internal/tests/api/sandboxes TestSandboxFork 16.2 +internal/tests/api/sandboxes TestSandboxFork_AlreadyPaused 5.5 +internal/tests/api/sandboxes TestSandboxFork_CrossTeamAccess 2.2 +internal/tests/api/sandboxes TestSandboxFork_InvalidCount 0.5 +internal/tests/api/sandboxes TestSandboxFork_Multiple 6.0 +internal/tests/api/sandboxes TestSandboxFork_NotFound 1.7 +internal/tests/api/sandboxes TestSandboxKill 11.4 +internal/tests/api/sandboxes TestSandboxList 0.5 +internal/tests/api/sandboxes TestSandboxListPaginationPaused 6.0 +internal/tests/api/sandboxes TestSandboxListPaginationRunning 2.8 +internal/tests/api/sandboxes TestSandboxListPaginationRunningAndPaused 4.9 +internal/tests/api/sandboxes TestSandboxListPaginationRunningLargerLimit 15.9 +internal/tests/api/sandboxes TestSandboxListPaused 5.6 +internal/tests/api/sandboxes TestSandboxListPaused_NoMetadata 4.8 +internal/tests/api/sandboxes TestSandboxListPausing 5.2 +internal/tests/api/sandboxes TestSandboxListRunning 1.6 +internal/tests/api/sandboxes TestSandboxListRunningV1 0.8 +internal/tests/api/sandboxes TestSandboxListRunning_NoMetadata 1.0 +internal/tests/api/sandboxes TestSandboxListSortedV1 1.8 +internal/tests/api/sandboxes TestSandboxListWithFilter 1.5 +internal/tests/api/sandboxes TestSandboxListWithFilterV1 0.8 +internal/tests/api/sandboxes TestSandboxNotAutoPause 1.0 +internal/tests/api/sandboxes TestSandboxPause 9.8 +internal/tests/api/sandboxes TestSandboxPauseNonFound 0.0 +internal/tests/api/sandboxes TestSandboxRapidSnapshotForkChain 39.1 +internal/tests/api/sandboxes TestSandboxRefresh 2.8 +internal/tests/api/sandboxes TestSandboxRefresh_CrossTeamAccess 1.2 +internal/tests/api/sandboxes TestSandboxRefresh_NotFound 0.0 +internal/tests/api/sandboxes TestSandboxResume 23.3 +internal/tests/api/sandboxes TestSandboxResumeUnknownSandbox 1.1 +internal/tests/api/sandboxes TestSandboxResumeWithSecuredEnvd 13.4 +internal/tests/api/sandboxes TestSandboxResume_CrossTeamAccess_Paused 2.4 +internal/tests/api/sandboxes TestSandboxResume_CrossTeamAccess_Running 1.1 +internal/tests/api/sandboxes TestSandboxResume_FilesystemOnlyReboots 17.4 +internal/tests/api/sandboxes TestSandboxSetTimeoutPausingSandbox 4.3 +internal/tests/api/sandboxes TestSandboxTimeout 10.9 +internal/tests/api/sandboxes TestSandboxTimeout_CrossTeamAccess 1.0 +internal/tests/api/sandboxes TestSandboxTimeout_NotFound 0.0 +internal/tests/api/sandboxes TestSnapshotTemplateConcurrentOperations 22.6 +internal/tests/api/sandboxes TestSnapshotTemplateCreate 70.9 +internal/tests/api/sandboxes TestSnapshotTemplateCreateSandbox 57.2 +internal/tests/api/sandboxes TestSnapshotTemplateDelete 11.8 +internal/tests/api/sandboxes TestSnapshotTemplateList 77.1 +internal/tests/api/sandboxes TestUpdateNetworkConfig 28.5 +internal/tests/api/sandboxes TestWithNetworkConfig_SSHWorks 133.6 +internal/tests/api/templates TestAssignmentOrderingAfterTagReassignment 77.1 +internal/tests/api/templates TestAssignmentOrderingLatestWins 200.4 +internal/tests/api/templates TestDeleteTemplate 128.7 +internal/tests/api/templates TestDeleteTemplateFromAnotherTeamAPIKey 46.5 +internal/tests/api/templates TestInvalidBuildStatus 0.0 +internal/tests/api/templates TestListTemplatesV2Pagination 0.1 +internal/tests/api/templates TestListTemplatesV2WithAPIKey 0.0 +internal/tests/api/templates TestListTemplatesV2WithAPIKeyAndTeamID 0.0 +internal/tests/api/templates TestListTemplatesV2WithInvalidAPIKey 0.0 +internal/tests/api/templates TestListTemplatesV2WithMismatchedTeamID 0.0 +internal/tests/api/templates TestListTemplatesWithAPIKey 0.0 +internal/tests/api/templates TestListTemplatesWithAPIKeyAndMismatchedTeamID 0.0 +internal/tests/api/templates TestListTemplatesWithAPIKeyAndTeamID 0.0 +internal/tests/api/templates TestListTemplatesWithInvalidAPIKey 0.0 +internal/tests/api/templates TestMultipleTagsOnSameTemplate 130.7 +internal/tests/api/templates TestRequestTemplateBuild 0.1 +internal/tests/api/templates TestRequestTemplateMemoryNonDivisibleBy2 0.0 +internal/tests/api/templates TestRequestTemplateOddCPU 0.0 +internal/tests/api/templates TestRequestTemplateTooHighCPU 0.0 +internal/tests/api/templates TestRequestTemplateTooHighMemory 0.0 +internal/tests/api/templates TestRequestTemplateTooLowCPU 0.0 +internal/tests/api/templates TestRequestTemplateTooLowRAM 0.0 +internal/tests/api/templates TestSandboxCreateWithAliasAndTag 168.9 +internal/tests/api/templates TestSandboxCreateWithDefaultTag 0.8 +internal/tests/api/templates TestSandboxCreateWithNonExistentTag 0.0 +internal/tests/api/templates TestSandboxCreateWithTag 116.3 +internal/tests/api/templates TestTagReassignment 165.9 +internal/tests/api/templates TestTemplateBuildCOPY 139.8 +internal/tests/api/templates TestTemplateBuildCache 174.2 +internal/tests/api/templates TestTemplateBuildDistroFamilies 340.3 +internal/tests/api/templates TestTemplateBuildENV 438.9 +internal/tests/api/templates TestTemplateBuildFromTemplate 41.9 +internal/tests/api/templates TestTemplateBuildFromTemplateBaseCommandsInheritance 51.2 +internal/tests/api/templates TestTemplateBuildFromTemplateCommandOverride 215.3 +internal/tests/api/templates TestTemplateBuildFromTemplateInheritance 199.4 +internal/tests/api/templates TestTemplateBuildFromTemplateLayered 100.2 +internal/tests/api/templates TestTemplateBuildFromTemplateStartCommand 106.9 +internal/tests/api/templates TestTemplateBuildFuseConfiguration 120.2 +internal/tests/api/templates TestTemplateBuildInstalledPackagesAvailable 106.9 +internal/tests/api/templates TestTemplateBuildRUN 63.8 +internal/tests/api/templates TestTemplateBuildStartReadyCommandExecution 78.7 +internal/tests/api/templates TestTemplateBuildUnsupportedDistro 72.9 +internal/tests/api/templates TestTemplateBuildWORKDIR 76.4 +internal/tests/api/templates TestTemplateBuildWithDifferentSourceImages 31.2 +internal/tests/api/templates TestTemplateBuildWithTagInAlias 27.6 +internal/tests/api/templates TestTemplateBuildWithTags 134.7 +internal/tests/api/templates TestTemplateBuildWithTagsAndSandboxCreation 77.0 +internal/tests/api/templates TestTemplateTagAssign 113.9 +internal/tests/api/templates TestTemplateTagAssignFromSourceTag 129.8 +internal/tests/api/templates TestTemplateTagDeleteLatestNotAllowed 126.7 +internal/tests/api/templates TestTemplateTagInvalidTagName 0.0 +internal/tests/api/templates TestTemplateTagNotFoundForNonExistentTemplate 0.0 +internal/tests/api/templates TestUpdateNonExistentTemplateWithAPIKey 0.0 +internal/tests/api/templates TestUpdateTemplateNotOwnedByTeam 97.8 +internal/tests/api/templates TestUpdateTemplateVisibilityToPrivateWithAPIKey 167.9 +internal/tests/api/templates TestUpdateTemplateVisibilityToPublicWithAPIKey 54.8 +internal/tests/api/templates TestUpdateTemplateWithAPIKey 117.0 +internal/tests/api/templates TestUpdateTemplateWithInvalidAPIKey 29.2 +internal/tests/api/volumes TestVolumeRoundTrip 5.3 +internal/tests/envd TestAccessAuthorizedPathWithResumedSandboxWithValidAccessToken 6.0 +internal/tests/envd TestAccessAuthorizedPathWithResumedSandboxWithoutAccessToken 4.2 +internal/tests/envd TestAccessToAuthorizedPathWithoutToken 0.8 +internal/tests/envd TestAccessWithNotCorrespondingSignatureAndSignatureExpiration 0.8 +internal/tests/envd TestAccessingHyperloopServerViaDomain 1.6 +internal/tests/envd TestAccessingHyperloopServerViaIP 1.9 +internal/tests/envd TestAccessingHyperloopServerViaIPWithBlockedInternet 2.1 +internal/tests/envd TestBindLocalhost 12.3 +internal/tests/envd TestCACertBundleOnTmpfs 1.4 +internal/tests/envd TestCACertFromBuildSurvivesInBakedBundle 264.2 +internal/tests/envd TestCACertInjection 1.3 +internal/tests/envd TestCACertPersistsThroughUpdateCACertificates 3.9 +internal/tests/envd TestCACertRotationOnResume 1.3 +internal/tests/envd TestCACertTrustedAfterFilesystemOnlyReboot 141.2 +internal/tests/envd TestChangeAccessAuthorizedToken 0.7 +internal/tests/envd TestCommandKillNextApp 217.7 +internal/tests/envd TestCommandKillWithAnd 6.8 +internal/tests/envd TestConcurrentFileUpload 1.8 +internal/tests/envd TestDownloadFileWhenAuthIsDisabled 2.6 +internal/tests/envd TestDownloadFileWithSigningWhenAuthIsEnabled 2.5 +internal/tests/envd TestDownloadFileWithoutSigningWhenAuthIsEnabled 2.3 +internal/tests/envd TestDownloadWithAlreadyExpiredToken 1.5 +internal/tests/envd TestDownloadWithHealthyToken 1.6 +internal/tests/envd TestFilePermissions 1.7 +internal/tests/envd TestFsFreezeThaw 5.0 +internal/tests/envd TestInitWithNilTokenOnSecuredSandboxReturnsUnauthorized 1.1 +internal/tests/envd TestInitWithWrongTokenOnSecuredSandboxReturnsUnauthorized 0.5 +internal/tests/envd TestListDir 1.1 +internal/tests/envd TestListDirEntry 5.0 +internal/tests/envd TestListDirFileEntry 3.8 +internal/tests/envd TestListDirMixedEntries 5.1 +internal/tests/envd TestRelativePath 1.7 +internal/tests/envd TestSecureSandboxFilesystemOnlyResumeAuth 14.0 +internal/tests/envd TestStat 2.9 +internal/tests/envd TestStdinCantRead 1.7 +internal/tests/envd TestWatcher 2.0 +internal/tests/envd TestWorkdirDeletion 1.0 +internal/tests/envd TestWorkdirPermissionDenied 1.8 +internal/tests/orchestrator TestHardwareEntropyDeviceAvailability 3.3 +internal/tests/orchestrator TestList 0.0 +internal/tests/orchestrator TestSandboxFilesystemPauseResumeIntegrity 34.0 +internal/tests/orchestrator TestSandboxFilesystemPauseResumeIntegrityStress 0.0 +internal/tests/orchestrator TestSandboxMemoryIntegrity 45.8 +internal/tests/orchestrator TestSandboxObjectNotFound 0.0 +internal/tests/proxies TestEnvdAccessTokenAutoResumeViaProxy 5.3 +internal/tests/proxies TestEnvdPortIsNotAffectedByTrafficAccessToken 0.4 +internal/tests/proxies TestMaskRequestHostAPIParameter 4.8 +internal/tests/proxies TestMaskRequestHostIncorrectUrl 0.0 +internal/tests/proxies TestSandboxAutoResumeUsesInitialTimeoutNotUpdatedTimeout 5.2 +internal/tests/proxies TestSandboxAutoResumeViaExec 8.0 +internal/tests/proxies TestSandboxAutoResumeViaProxy 11.1 +internal/tests/proxies TestSandboxAutoResumeWithoutExplicitTimeoutUsesMinimumTimeout 8.4 +internal/tests/proxies TestSandboxNoAutoResumeFilesystemOnly 5.9 +internal/tests/proxies TestSandboxNoAutoResumeViaProxyWithoutFlag 10.2 +internal/tests/proxies TestSandboxNoAutoResumeWithoutFlag 4.4 +internal/tests/proxies TestSandboxNotFound 0.0 +internal/tests/proxies TestSandboxProxyClosedPort 3.8 +internal/tests/proxies TestSandboxProxyWorkingPort 5.5 +internal/tests/proxies TestSandboxWithEnabledTrafficAccessToken 5.8 +internal/tests/proxies TestSandboxWithEnabledTrafficAccessTokenButInvalidHeader 0.7 +internal/tests/proxies TestSandboxWithEnabledTrafficAccessTokenButMissingHeader 0.7 +internal/tests/proxies TestSandboxWithTrafficAccessTokenAutoResumeViaProxy 7.5 diff --git a/tests/integration/scripts/update-test-weights.go b/tests/integration/scripts/update-test-weights.go new file mode 100755 index 0000000000..8861f098a3 --- /dev/null +++ b/tests/integration/scripts/update-test-weights.go @@ -0,0 +1,131 @@ +//go:build ignore + +// Command update-test-weights regenerates test-weights.tsv from the JUnit XML +// of a full integration run: +// +// go run scripts/update-test-weights.go +// +// Run it from tests/integration; it rewrites scripts/test-weights.tsv in place. +// +// The weights only balance CI shards (see shard-tests.sh), so they are +// approximate by design: per top-level test we keep the slowest observation +// across every compression config in the directory, since the slowest config +// is what sets the wall clock. +package main + +import ( + "encoding/xml" + "fmt" + "io/fs" + "log" + "os" + "path/filepath" + "sort" + "strconv" + "strings" +) + +const header = `# package test seconds +# Slowest-of-all-compression-configs wall time per top-level test, used only to +# balance CI shards (see shard-tests.sh). Regenerate with: make -C tests/integration update-test-weights +` + +type report struct { + Suites []struct { + Name string `xml:"name,attr"` + Cases []struct { + Name string `xml:"name,attr"` + Time string `xml:"time,attr"` + } `xml:"testcase"` + } `xml:"testsuite"` +} + +type key struct{ pkg, test string } + +func main() { + log.SetFlags(0) + if len(os.Args) != 2 { + log.Fatalf("usage: %s ", filepath.Base(os.Args[0])) + } + + weights, files, err := collect(os.Args[1]) + if err != nil { + log.Fatal(err) + } + if files == 0 { + log.Fatalf("no JUnit XML found under %s", os.Args[1]) + } + + const out = "scripts/test-weights.tsv" + if err := write(out, weights); err != nil { + log.Fatal(err) + } + fmt.Printf("wrote %d weights from %d file(s) to %s\n", len(weights), files, out) +} + +func collect(dir string) (map[key]float64, int, error) { + weights := map[key]float64{} + files := 0 + + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || filepath.Ext(path) != ".xml" { + return err + } + files++ + + raw, err := os.ReadFile(path) + if err != nil { + return err + } + var parsed report + if err := xml.Unmarshal(raw, &parsed); err != nil { + return fmt.Errorf("%s: %w", path, err) + } + + for _, suite := range parsed.Suites { + _, pkg, _ := strings.Cut(suite.Name, "tests/integration/") + times := make(map[string]float64, len(suite.Cases)) + for _, c := range suite.Cases { + seconds, _ := strconv.ParseFloat(c.Time, 64) + times[c.Name] = seconds + } + for name, own := range times { + if strings.Contains(name, "/") { + continue + } + // A parent with parallel subtests reports ~0s of its own, so + // take whichever of the two accountings is larger. + subtests := 0.0 + for sub, seconds := range times { + if strings.HasPrefix(sub, name+"/") { + subtests += seconds + } + } + weights[key{pkg, name}] = max(weights[key{pkg, name}], own, subtests) + } + } + return nil + }) + + return weights, files, err +} + +func write(path string, weights map[key]float64) error { + keys := make([]key, 0, len(weights)) + for k := range weights { + keys = append(keys, k) + } + sort.Slice(keys, func(i, j int) bool { + if keys[i].pkg != keys[j].pkg { + return keys[i].pkg < keys[j].pkg + } + return keys[i].test < keys[j].test + }) + + var b strings.Builder + b.WriteString(header) + for _, k := range keys { + fmt.Fprintf(&b, "%s\t%s\t%.1f\n", k.pkg, k.test, weights[k]) + } + return os.WriteFile(path, []byte(b.String()), 0o644) +}