diff --git a/.github/workflows/sync-kubeadm.yml b/.github/workflows/sync-kubeadm.yml index a73a886..07ef4fe 100644 --- a/.github/workflows/sync-kubeadm.yml +++ b/.github/workflows/sync-kubeadm.yml @@ -16,4 +16,4 @@ jobs: - uses: actions/checkout@v3 - name: sync kubeadm command - run: go run cmd/builder/main.go sync-kubeadm --kubernetes-version v1.31.8 --configFile builder.yaml --github-owner ${{ secrets.OWNER }} --github-repo ${{ secrets.REPO }} --github-token ${{ secrets.TOKEN }} + run: go run cmd/builder/main.go sync-kubeadm --kubernetes-version v1.36.2 --arch amd64 --configFile builder.yaml --github-owner ${{ secrets.OWNER }} --github-repo ${{ secrets.REPO }} --github-token ${{ secrets.TOKEN }} diff --git a/README.md b/README.md index a5d5aa1..4a3c1d6 100644 --- a/README.md +++ b/README.md @@ -287,11 +287,14 @@ export GITHUB_TOKEN=ghp_xxx 将 builder 产物加载为常驻服务:镜像走本地 registry(短名),软件包走 HTTP yum/dnf/apt 源。不依赖 `createrepo` / `apt-ftparchive`。 +`--bundle` / `--dir` 同时支持 **builder 离线包**(含 `manifest.yaml`)与 **单镜像 docker save 的 `.tar.gz`**(含 `manifest.json`,如 `docker save img:tag | gzip > img.tar.gz`),可混放;`--dir` 热加载对新放入的单镜像包同样生效。repo/tag 优先取 docker-save 内 `RepoTags`。 + ```bash -# packages + images 两个 tar 一起加载 +# packages + images + 单镜像 save 混放 ./builder serve \ --bundle ./dist/pixiu-packages-centos-8-amd64-v1.27.3.tar.gz \ --bundle ./dist/pixiu-images-centos-8-amd64-v1.27.3.tar.gz \ + --bundle ./extra/pause.tar.gz \ --advertise-host 192.168.1.10 # 已解压目录 diff --git a/cmd/builder/main.go b/cmd/builder/main.go index e44cbaa..dbcf8da 100644 --- a/cmd/builder/main.go +++ b/cmd/builder/main.go @@ -1201,8 +1201,8 @@ func newServeCmd() *cobra.Command { return err }, } - cmd.Flags().StringArrayVar(&serveBundles, "bundle", nil, "离线包目录或 tar.gz(可重复,例如 packages + images)") - cmd.Flags().StringVar(&serveDir, "dir", "", "离线包目录:不存在则自动创建;加载其下所有 *.tar.gz 并轮询热加载新包(3s)") + cmd.Flags().StringArrayVar(&serveBundles, "bundle", nil, "离线包目录或 tar.gz(可重复;支持 builder 产物与单镜像 docker save 的 .tar.gz,可混放)") + cmd.Flags().StringVar(&serveDir, "dir", "", "离线包目录:不存在则自动创建;加载其下所有 *.tar.gz(builder 包与单镜像 save 可混放)并轮询热加载(3s)") cmd.Flags().StringVar(&serveDataDir, "data-dir", "./serve-data", "工作目录(解压、repodata、registry blob)") cmd.Flags().StringVar(&serveRegistryAddr, "registry-addr", "0.0.0.0:5000", "OCI registry 监听地址") cmd.Flags().StringVar(&serveRegistryPort, "registry-port", "", "registry 端口(覆盖 --registry-addr 端口,地址段不变)") diff --git a/internal/serve/bundle.go b/internal/serve/bundle.go index c8f3349..cc48e32 100644 --- a/internal/serve/bundle.go +++ b/internal/serve/bundle.go @@ -1,6 +1,8 @@ package serve import ( + "archive/tar" + "compress/gzip" "fmt" "io" "os" @@ -27,6 +29,8 @@ func loadBundles(inputs []string, destRoot string) ([]string, error) { return roots, nil } +// resolveBundle 加载离线包:支持 builder 产物(含 manifest.yaml)与单镜像 +// docker save 的 .tar.gz(含 manifest.json)。二者可经 --bundle / --dir 混放。 func resolveBundle(path, extractDir string) (string, error) { info, err := os.Stat(path) if err != nil { @@ -42,17 +46,121 @@ func resolveBundle(path, extractDir string) (string, error) { if !strings.HasSuffix(path, ".tar.gz") { return "", fmt.Errorf("必须是目录或 .tar.gz") } + kind, err := peekTarGzKind(path) + if err != nil { + return "", fmt.Errorf("读取 tar.gz 失败: %w", err) + } if err := os.MkdirAll(extractDir, 0o755); err != nil { return "", err } + // 单镜像 docker-save:只 gunzip 成 .tar,避免整包 Untar 占磁盘。 + if kind == "docker-save" { + return materializeDockerSaveBundle(path, extractDir) + } if err := builder.UntarGz(path, extractDir); err != nil { return "", fmt.Errorf("解压失败: %w", err) } - root := findManifestDir(extractDir) - if root == "" { - return "", fmt.Errorf("tar.gz 中未找到 manifest.yaml") + if root := findManifestDir(extractDir); root != "" { + return root, nil + } + if isDockerSaveDir(extractDir) { + return materializeDockerSaveBundle(path, extractDir) + } + return "", fmt.Errorf("tar.gz 既不是 builder 离线包(缺 manifest.yaml),也不是 docker save 镜像(缺 manifest.json)") +} + +// peekTarGzKind 扫描 tar.gz 成员名,区分 builder 离线包与 docker save。 +// 优先 manifest.yaml(bundle);否则有 manifest.json 则为 docker-save。 +func peekTarGzKind(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + gz, err := gzip.NewReader(f) + if err != nil { + return "", err + } + defer gz.Close() + tr := tar.NewReader(gz) + hasYAML, hasJSON := false, false + for { + hdr, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return "", err + } + base := filepath.Base(hdr.Name) + switch base { + case manifest.ManifestFileName: + hasYAML = true + case "manifest.json": + hasJSON = true + } + if hasYAML { + return "bundle", nil + } + } + if hasJSON { + return "docker-save", nil + } + return "", nil +} + +// isDockerSaveDir 判断目录是否为 docker save 解压结果(根目录含 manifest.json)。 +func isDockerSaveDir(dir string) bool { + st, err := os.Stat(filepath.Join(dir, "manifest.json")) + return err == nil && !st.IsDir() +} + +// materializeDockerSaveBundle 将单镜像 docker-save .tar.gz 落成伪 bundle: +// images/addons/.tar(未压缩,供 ImageFromPath / RepoTags 读取)。 +func materializeDockerSaveBundle(tarGzPath, extractDir string) (string, error) { + name := tarGzBaseName(tarGzPath) + imgDir := filepath.Join(extractDir, "images", "addons") + if err := os.MkdirAll(imgDir, 0o755); err != nil { + return "", err + } + dest := filepath.Join(imgDir, name+".tar") + if err := gunzipFile(tarGzPath, dest); err != nil { + return "", fmt.Errorf("还原 docker-save tar 失败: %w", err) + } + return extractDir, nil +} + +// tarGzBaseName 返回 foo.tar.gz → foo。 +func tarGzBaseName(path string) string { + base := filepath.Base(path) + if len(base) >= 7 && strings.EqualFold(base[len(base)-7:], ".tar.gz") { + base = base[:len(base)-7] + } + if strings.TrimSpace(base) == "" || base == "." { + return "image" } - return root, nil + return base +} + +// gunzipFile 将 .gz 文件解压为普通文件(用于 docker-save .tar.gz → .tar)。 +func gunzipFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + gz, err := gzip.NewReader(in) + if err != nil { + return err + } + defer gz.Close() + out, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + if err != nil { + return err + } + defer out.Close() + _, err = io.Copy(out, gz) + return err } func findManifestDir(root string) string { diff --git a/internal/serve/serve.go b/internal/serve/serve.go index 48b9fd7..13b9f96 100644 --- a/internal/serve/serve.go +++ b/internal/serve/serve.go @@ -18,9 +18,10 @@ import ( // Options 控制 serve 行为。 type Options struct { - // Bundles 离线包路径(目录或 .tar.gz),可多个(packages + images)。 + // Bundles 离线包路径(目录或 .tar.gz),可多个。 + // 支持 builder 产物(含 manifest.yaml)与单镜像 docker save 的 .tar.gz,可混放。 Bundles []string - // Dir 离线包目录:启动时加载其下所有 *.tar.gz,并轮询热加载新出现的包。 + // Dir 离线包目录:启动时加载其下所有 *.tar.gz(builder 包与单镜像 save 可混放),并轮询热加载。 Dir string // DataDir 工作目录(解压、repodata、registry blob)。 DataDir string diff --git a/internal/serve/serve_test.go b/internal/serve/serve_test.go index a387f4e..f982e03 100644 --- a/internal/serve/serve_test.go +++ b/internal/serve/serve_test.go @@ -6,6 +6,7 @@ import ( "compress/gzip" "context" "fmt" + "io" "net" "net/http" "os" @@ -1017,3 +1018,184 @@ func TestPrintReadyConfigDemo(t *testing.T) { } } } + +func TestTarGzBaseName(t *testing.T) { + cases := []struct{ in, want string }{ + {"/tmp/pause.tar.gz", "pause"}, + {"pause.tar.gz", "pause"}, + {"PAUSE.TAR.GZ", "PAUSE"}, + {".tar.gz", "image"}, + } + for _, c := range cases { + if got := tarGzBaseName(c.in); got != c.want { + t.Errorf("tarGzBaseName(%q)=%q want %q", c.in, got, c.want) + } + } +} + +func containerRegistryTestTar(t *testing.T) string { + t.Helper() + out, err := exec.Command("go", "list", "-m", "-f", "{{.Dir}}", "github.com/google/go-containerregistry").Output() + if err != nil { + t.Skipf("go list containerregistry: %v", err) + } + tarPath := filepath.Join(strings.TrimSpace(string(out)), "pkg/v1/tarball/testdata/test_image_1.tar") + if _, err := os.Stat(tarPath); err != nil { + t.Skip(err) + } + return tarPath +} + +func TestPeekAndResolveDockerSaveTarGz(t *testing.T) { + tarPath := containerRegistryTestTar(t) + dir := t.TempDir() + gzPath := filepath.Join(dir, "tarball-image.tar.gz") + if err := gzipFile(tarPath, gzPath); err != nil { + t.Fatal(err) + } + + kind, err := peekTarGzKind(gzPath) + if err != nil { + t.Fatal(err) + } + if kind != "docker-save" { + t.Fatalf("peekTarGzKind=%q want docker-save", kind) + } + + root, err := resolveBundle(gzPath, filepath.Join(dir, "extract")) + if err != nil { + t.Fatal(err) + } + tars, err := collectImageTars([]string{root}) + if err != nil { + t.Fatal(err) + } + if len(tars) != 1 { + t.Fatalf("collectImageTars=%v", tars) + } + if tars[0].Name != "tarball-image" { + t.Errorf("name=%q want tarball-image", tars[0].Name) + } + if _, err := os.Stat(tars[0].Path); err != nil { + t.Fatal(err) + } +} + +func TestPeekTarGzKindBuilderBundle(t *testing.T) { + dir := t.TempDir() + src := filepath.Join(dir, "src") + if err := os.MkdirAll(filepath.Join(src, "packages"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(src, "manifest.yaml"), []byte("schema_version: 1\n"), 0o644); err != nil { + t.Fatal(err) + } + gz := filepath.Join(dir, "bundle.tar.gz") + if err := tarGzDir(src, gz); err != nil { + t.Fatal(err) + } + kind, err := peekTarGzKind(gz) + if err != nil { + t.Fatal(err) + } + if kind != "bundle" { + t.Fatalf("kind=%q want bundle", kind) + } +} + +func TestResolveBundleMixedBuilderAndDockerSave(t *testing.T) { + tarPath := containerRegistryTestTar(t) + dir := t.TempDir() + + bSrc := filepath.Join(dir, "bsrc") + if err := os.MkdirAll(filepath.Join(bSrc, "packages"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(bSrc, "manifest.yaml"), []byte("schema_version: 1\n"), 0o644); err != nil { + t.Fatal(err) + } + builderGz := filepath.Join(dir, "builder.tar.gz") + if err := tarGzDir(bSrc, builderGz); err != nil { + t.Fatal(err) + } + + imgGz := filepath.Join(dir, "myimg.tar.gz") + if err := gzipFile(tarPath, imgGz); err != nil { + t.Fatal(err) + } + + roots, err := loadBundles([]string{builderGz, imgGz}, filepath.Join(dir, "out")) + if err != nil { + t.Fatal(err) + } + if len(roots) != 2 { + t.Fatalf("roots=%d", len(roots)) + } + tars, err := collectImageTars(roots) + if err != nil { + t.Fatal(err) + } + if len(tars) != 1 { + t.Fatalf("期望仅 docker-save 贡献 1 个镜像 tar,got %v", tars) + } +} + +func TestImportImagesFromDockerSaveTarGz(t *testing.T) { + tarPath := containerRegistryTestTar(t) + dir := t.TempDir() + gzPath := filepath.Join(dir, "test-image.tar.gz") + if err := gzipFile(tarPath, gzPath); err != nil { + t.Fatal(err) + } + root, err := resolveBundle(gzPath, filepath.Join(dir, "extract")) + if err != nil { + t.Fatal(err) + } + + blobDir := t.TempDir() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + srv := &http.Server{Handler: newRegistryHandler(blobDir)} + go srv.Serve(ln) + defer srv.Close() + + port := ln.Addr().(*net.TCPAddr).Port + host := fmt.Sprintf("127.0.0.1:%d", port) + ctx := context.Background() + if err := waitHTTP(ctx, "http://"+host+"/v2/"); err != nil { + t.Fatal(err) + } + refs, err := importImages(ctx, []string{root}, host, host, "pixiu") + if err != nil { + t.Fatal(err) + } + if len(refs) != 1 { + t.Fatalf("refs=%v", refs) + } + if !strings.Contains(refs[0], "/pixiu/tarball:") { + t.Fatalf("got %q, want .../pixiu/tarball:...", refs[0]) + } +} + +// gzipFile 将普通文件 gzip 压缩为 dst。 +func gzipFile(src, dst string) error { + in, err := os.Open(src) + if err != nil { + return err + } + defer in.Close() + out, err := os.Create(dst) + if err != nil { + return err + } + defer out.Close() + gw := gzip.NewWriter(out) + if _, err := io.Copy(gw, in); err != nil { + gw.Close() + return err + } + return gw.Close() +}