From 4822701c3d4cfd5e38b992e454dbed3f76f14905 Mon Sep 17 00:00:00 2001 From: Mohammad Abdolirad Date: Fri, 12 Jun 2026 16:33:39 +0200 Subject: [PATCH] refactor integration tests into flake apps - add test-docker, test-containerd, and test-dind flake apps - simplify ci jobs and per-engine coverage uploads - read CONTAINERD_ADDRESS and set docker ExposedPorts --- .github/workflows/ci.yml | 162 ++++++++++++++++++--------------------- detect.go | 4 + driver_docker.go | 7 ++ flake.nix | 124 ++++++++++++++++++++++++++++-- 4 files changed, 201 insertions(+), 96 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e8bcdef..cf5acaa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -74,29 +74,6 @@ jobs: path: coverage-unit.out retention-days: 1 - integration-docker: - name: Integration Tests — Docker - runs-on: ubuntu-latest - needs: [check] - steps: - - uses: actions/checkout@v6 - - - uses: cachix/install-nix-action@v31 - with: - github_access_token: ${{ secrets.GITHUB_TOKEN }} - - - name: Run integration tests against Docker - env: - CURRUS_TEST_ENGINE: docker - run: nix run .#test-integration - - - name: Upload coverage artifact - uses: actions/upload-artifact@v7 - with: - name: coverage-integration-docker - path: coverage-integration.out - retention-days: 1 - integration-podman: name: Integration Tests — Podman runs-on: ubuntu-latest @@ -108,25 +85,21 @@ jobs: with: github_access_token: ${{ secrets.GITHUB_TOKEN }} - - name: Start Podman socket - run: | - systemctl --user enable --now podman.socket - echo "DOCKER_HOST=unix://${XDG_RUNTIME_DIR}/podman/podman.sock" >> "$GITHUB_ENV" + - name: Allow unprivileged user namespaces + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - name: Run integration tests against Podman - env: - CURRUS_TEST_ENGINE: podman - run: nix run .#test-integration + run: nix run .#test-podman - name: Upload coverage artifact uses: actions/upload-artifact@v7 with: name: coverage-integration-podman - path: coverage-integration.out + path: coverage-podman.out retention-days: 1 - integration-docker-rootless: - name: Integration Tests — Docker (rootless) + integration-docker: + name: Integration Tests — Docker runs-on: ubuntu-latest needs: [check] steps: @@ -136,44 +109,24 @@ jobs: with: github_access_token: ${{ secrets.GITHUB_TOKEN }} - - name: Start Docker rootless - env: - # The install script aborts when /var/run/docker.sock is writable; - # we stop rootful Docker below, so skip that guard explicitly. - FORCE_ROOTLESS_INSTALL: "1" + - name: Install rootless Docker prerequisites run: | - # GitHub runners ship rootful Docker; stop it so rootless can take over - sudo systemctl stop docker.service docker.socket || true - sudo systemctl disable docker.service docker.socket || true - # the stopped service leaves the socket file behind; remove it - sudo rm -f /var/run/docker.sock - - # rootless prerequisite (newuidmap/newgidmap) sudo apt-get update -q sudo apt-get install -y uidmap - - # Ubuntu 24.04 restricts unprivileged user namespaces, which RootlessKit - # needs; relax it so the rootless daemon can start (safe on ephemeral CI) sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - curl -fsSL https://get.docker.com/rootless | sh - export PATH=$HOME/bin:$PATH - export DOCKER_HOST=unix://${XDG_RUNTIME_DIR}/docker.sock - # the installer already starts docker.service; just wait for readiness - for i in $(seq 1 10); do - docker -H "$DOCKER_HOST" info >/dev/null 2>&1 && break - sleep 1 - done - echo "DOCKER_HOST=unix://${XDG_RUNTIME_DIR}/docker.sock" >> "$GITHUB_ENV" - echo "PATH=$HOME/bin:$PATH" >> "$GITHUB_ENV" - - name: Run integration tests against rootless Docker - env: - CURRUS_TEST_ENGINE: docker - run: nix run .#test-integration + run: nix run .#test-docker + + - name: Upload coverage artifact + uses: actions/upload-artifact@v7 + with: + name: coverage-integration-docker + path: coverage-docker.out + retention-days: 1 - integration-podman-rootless: - name: Integration Tests — Podman (rootless) + integration-containerd: + name: Integration Tests — containerd runs-on: ubuntu-latest needs: [check] steps: @@ -183,18 +136,18 @@ jobs: with: github_access_token: ${{ secrets.GITHUB_TOKEN }} - - name: Start Podman rootless socket - run: | - systemctl --user enable --now podman.socket - echo "DOCKER_HOST=unix://${XDG_RUNTIME_DIR}/podman/podman.sock" >> "$GITHUB_ENV" + - name: Run integration tests against containerd + run: nix run .#test-containerd - - name: Run integration tests against rootless Podman - env: - CURRUS_TEST_ENGINE: podman - run: nix run .#test-integration + - name: Upload coverage artifact + uses: actions/upload-artifact@v7 + with: + name: coverage-integration-containerd + path: coverage-containerd.out + retention-days: 1 - integration-containerd: - name: Integration Tests — containerd + integration-dind: + name: Integration Tests — DinD runs-on: ubuntu-latest needs: [check] steps: @@ -204,29 +157,24 @@ jobs: with: github_access_token: ${{ secrets.GITHUB_TOKEN }} - - name: Install containerd - run: | - sudo apt-get update -q - sudo apt-get install -y containerd - sudo systemctl start containerd + - name: Allow unprivileged user namespaces + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 - - name: Run integration tests against containerd - env: - CURRUS_TEST_ENGINE: containerd - run: nix run .#test-integration + - name: Run DinD integration tests + run: nix run .#test-dind - name: Upload coverage artifact uses: actions/upload-artifact@v7 with: - name: coverage-integration-containerd - path: coverage-integration.out + name: coverage-integration-dind + path: coverage-dind.out retention-days: 1 report: name: Coverage Report runs-on: ubuntu-latest - needs: [unit-test, integration-docker, integration-podman, integration-containerd, integration-docker-rootless, integration-podman-rootless] - if: always() && (needs.unit-test.result == 'success' || needs.integration-docker.result == 'success' || needs.integration-containerd.result == 'success') + needs: [unit-test, integration-podman, integration-docker, integration-containerd, integration-dind] + if: always() && (needs.unit-test.result == 'success' || needs.integration-podman.result == 'success' || needs.integration-docker.result == 'success' || needs.integration-containerd.result == 'success' || needs.integration-dind.result == 'success') permissions: contents: read id-token: write @@ -239,6 +187,13 @@ jobs: with: name: coverage-unit + - name: Download Podman integration coverage + if: needs.integration-podman.result == 'success' + uses: actions/download-artifact@v8 + with: + name: coverage-integration-podman + path: coverage-integration-podman + - name: Download Docker integration coverage if: needs.integration-docker.result == 'success' uses: actions/download-artifact@v8 @@ -253,6 +208,13 @@ jobs: name: coverage-integration-containerd path: coverage-integration-containerd + - name: Download DinD integration coverage + if: needs.integration-dind.result == 'success' + uses: actions/download-artifact@v8 + with: + name: coverage-integration-dind + path: coverage-integration-dind + - name: Upload unit coverage to Codecov if: needs.unit-test.result == 'success' uses: codecov/codecov-action@v6 @@ -264,13 +226,24 @@ jobs: name: currus-unit disable_search: true + - name: Upload Podman integration coverage to Codecov + if: needs.integration-podman.result == 'success' + uses: codecov/codecov-action@v6 + with: + use_oidc: true + fail_ci_if_error: true + files: ./coverage-integration-podman/coverage-podman.out + flags: integration + name: currus-integration-podman + disable_search: true + - name: Upload Docker integration coverage to Codecov if: needs.integration-docker.result == 'success' uses: codecov/codecov-action@v6 with: use_oidc: true fail_ci_if_error: true - files: ./coverage-integration-docker/coverage-integration.out + files: ./coverage-integration-docker/coverage-docker.out flags: integration name: currus-integration-docker disable_search: true @@ -281,7 +254,18 @@ jobs: with: use_oidc: true fail_ci_if_error: true - files: ./coverage-integration-containerd/coverage-integration.out + files: ./coverage-integration-containerd/coverage-containerd.out flags: integration name: currus-integration-containerd disable_search: true + + - name: Upload DinD integration coverage to Codecov + if: needs.integration-dind.result == 'success' + uses: codecov/codecov-action@v6 + with: + use_oidc: true + fail_ci_if_error: true + files: ./coverage-integration-dind/coverage-dind.out + flags: integration + name: currus-integration-dind + disable_search: true diff --git a/detect.go b/detect.go index 8cf89a1..67594ec 100644 --- a/detect.go +++ b/detect.go @@ -139,6 +139,10 @@ func openKind(ctx context.Context, kind EngineKind, cfg engineConfig) (Engine, e socket = cfg.endpoint.Host ns = cfg.endpoint.Namespace } + // CONTAINERD_ADDRESS is the standard env var used by ctr and other containerd tools. + if socket == "" { + socket = os.Getenv("CONTAINERD_ADDRESS") + } return newContainerdEngine(containerdConfig{ Socket: socket, diff --git a/driver_docker.go b/driver_docker.go index 257871c..14d4a3a 100644 --- a/driver_docker.go +++ b/driver_docker.go @@ -253,6 +253,13 @@ func (e *dockerEngine) CreateContainer(ctx context.Context, spec ContainerSpec) if err != nil { return "", err } + if len(portBindings) > 0 { + exposed := make(network.PortSet, len(portBindings)) + for port := range portBindings { + exposed[port] = struct{}{} + } + cfg.ExposedPorts = exposed + } hc := &container.HostConfig{ Mounts: dockerConvertMounts(spec.Mounts), diff --git a/flake.nix b/flake.nix index 78581d4..5e31ee8 100644 --- a/flake.nix +++ b/flake.nix @@ -162,13 +162,6 @@ coverProfile = "coverage-unit.out"; }; - test-integration = mkTaggedRaceTest { - name = "test-integration"; - description = "Run integration tests with race detector; write coverage-integration.out (build tag integration). Set CURRUS_TEST_ENGINE=docker|podman|containerd."; - tags = "integration"; - coverProfile = "coverage-integration.out"; - }; - test-podman = mkApp { name = "test-podman"; description = "Start an ephemeral rootless Podman socket and run integration tests against it"; @@ -204,6 +197,123 @@ -timeout 10m "''${testpkgs[@]}" ''; }; + + test-docker = mkApp { + name = "test-docker"; + description = "Start an ephemeral rootless Docker daemon and run integration tests against it"; + script = '' + set -euo pipefail + dockerd="${pkgs.docker}/bin/dockerd-rootless" + go="${pkgs.go}/bin/go" + + run_dir="$(mktemp -d)" + sock="$run_dir/docker.sock" + trap 'kill "$svc_pid" 2>/dev/null; wait "$svc_pid" 2>/dev/null; rm -rf "$run_dir" 2>/dev/null || true' EXIT + + export XDG_RUNTIME_DIR="$run_dir" + "$dockerd" \ + --host "unix://$sock" \ + --data-root "$run_dir/data" \ + 2>"$run_dir/dockerd.log" & + svc_pid=$! + + for i in $(seq 1 60); do + [ -S "$sock" ] && break + sleep 0.5 + done + if [ ! -S "$sock" ]; then + echo "dockerd-rootless did not start; log:" >&2 + cat "$run_dir/dockerd.log" >&2 + exit 1 + fi + + export DOCKER_HOST="unix://$sock" + export CURRUS_TEST_ENGINE=docker + export GOTOOLCHAIN=local + + mapfile -t testpkgs < <("$go" list -tags=integration ./... | grep -vE '/examples(/|$)' || true) + "$go" test -tags=integration -race -shuffle=on -covermode=atomic \ + -coverpkg=./... -coverprofile=coverage-docker.out \ + -timeout 10m "''${testpkgs[@]}" + ''; + }; + + test-containerd = mkApp { + name = "test-containerd"; + description = "Start an ephemeral containerd daemon (requires sudo) and run integration tests against it"; + script = '' + set -euo pipefail + containerd_bin="${pkgs.containerd}/bin/containerd" + go="${pkgs.go}/bin/go" + export PATH="${pkgs.runc}/bin:$PATH" + + state_dir="$(mktemp -d)" + sock="$state_dir/containerd.sock" + trap 'sudo kill "$svc_pid" 2>/dev/null; wait "$svc_pid" 2>/dev/null; sudo rm -rf "$state_dir" 2>/dev/null || true' EXIT + + sudo env PATH="$PATH" "$containerd_bin" \ + --address "$sock" \ + --state "$state_dir/state" \ + --root "$state_dir/root" \ + 2>"$state_dir/containerd.log" & + svc_pid=$! + + for i in $(seq 1 60); do + [ -S "$sock" ] && break + sleep 0.5 + done + if [ ! -S "$sock" ]; then + echo "containerd did not start; log:" >&2 + cat "$state_dir/containerd.log" >&2 + exit 1 + fi + + export CONTAINERD_ADDRESS="$sock" + export CURRUS_TEST_ENGINE=containerd + export GOTOOLCHAIN=local + + mapfile -t testpkgs < <("$go" list -tags=integration ./... | grep -vE '/examples(/|$)' || true) + sudo -E "$go" test -tags=integration -race -shuffle=on -covermode=atomic \ + -coverpkg=./... -coverprofile=coverage-containerd.out \ + -timeout 10m "''${testpkgs[@]}" + ''; + }; + + test-dind = mkApp { + name = "test-dind"; + description = "Start an ephemeral Podman socket and run TestConformanceDinD against it"; + script = '' + set -euo pipefail + podman="${pkgs.podman}/bin/podman" + go="${pkgs.go}/bin/go" + + sock_dir="$(mktemp -d)" + sock="$sock_dir/podman.sock" + trap 'kill "$svc_pid" 2>/dev/null; rm -rf "$sock_dir"' EXIT + + "$podman" system service --time=0 "unix://$sock" & + svc_pid=$! + + for i in $(seq 1 30); do + [ -S "$sock" ] && break + sleep 0.2 + done + if [ ! -S "$sock" ]; then + echo "podman socket did not appear at $sock" >&2 + exit 1 + fi + + export DOCKER_HOST="unix://$sock" + export CURRUS_TEST_ENGINE=docker + export GOTOOLCHAIN=local + + mapfile -t testpkgs < <("$go" list -tags=integration ./... | grep -vE '/examples(/|$)' || true) + "$go" test -tags=integration -race -shuffle=on -covermode=atomic \ + -coverpkg=./... -coverprofile=coverage-dind.out \ + -run TestConformanceDinD \ + -timeout 10m "''${testpkgs[@]}" + ''; + }; }; } );