diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1a26edc..a3419f5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -160,6 +160,8 @@ jobs: docker: runs-on: ubuntu-latest + outputs: + digest: ${{ steps.build.outputs.digest }} steps: - uses: actions/checkout@v4 @@ -190,6 +192,7 @@ jobs: type=raw,value=latest - uses: docker/build-push-action@v6 + id: build with: context: . platforms: linux/amd64,linux/arm64 @@ -202,3 +205,77 @@ jobs: BUILD_TIME=${{ steps.tag.outputs.built }} cache-from: type=gha cache-to: type=gha,mode=max + + # Arca hosts its own image on an arca instance. The index is copied rather than + # rebuilt, so both registries serve the same digests and pulling by digest from + # either gets identical bytes. + # + # Skipped until MIRROR_IMAGE is set as a repository variable, which is what keeps a + # fork from failing on a registry it has no credentials for. + mirror: + needs: docker + runs-on: ubuntu-latest + if: ${{ vars.MIRROR_IMAGE != '' }} + steps: + - uses: docker/setup-buildx-action@v3 + + - name: Resolve the tag and the mirror registry + id: mirror + env: + MIRROR_IMAGE: ${{ vars.MIRROR_IMAGE }} + run: | + set -euo pipefail + echo "tag=${{ inputs.tag || github.ref_name }}" >> "$GITHUB_OUTPUT" + # The registry to authenticate against is the host part of the image, so the + # two can never drift apart. + echo "registry=${MIRROR_IMAGE%%/*}" >> "$GITHUB_OUTPUT" + + - uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - uses: docker/login-action@v3 + with: + registry: ${{ steps.mirror.outputs.registry }} + username: ${{ secrets.MIRROR_USERNAME }} + password: ${{ secrets.MIRROR_TOKEN }} + + - uses: docker/metadata-action@v5 + id: meta + with: + images: ${{ vars.MIRROR_IMAGE }} + tags: | + type=semver,pattern={{version}},value=${{ steps.mirror.outputs.tag }} + type=semver,pattern={{major}}.{{minor}},value=${{ steps.mirror.outputs.tag }} + type=semver,pattern={{major}},value=${{ steps.mirror.outputs.tag }} + type=raw,value=latest + + - name: Copy the index across + env: + SOURCE: ${{ env.IMAGE }}@${{ needs.docker.outputs.digest }} + TAGS: ${{ steps.meta.outputs.tags }} + run: | + set -euo pipefail + + # imagetools takes one source and any number of destination tags, so the + # whole multi-platform index is copied in a single call. + tags=() + while IFS= read -r tag; do + [ -n "$tag" ] && tags+=(--tag "$tag") + done <<< "$TAGS" + + echo "mirroring $SOURCE to ${#tags[@]} tags" + docker buildx imagetools create "${tags[@]}" "$SOURCE" + + - name: Confirm it is pullable + env: + MIRROR_IMAGE: ${{ vars.MIRROR_IMAGE }} + DIGEST: ${{ needs.docker.outputs.digest }} + run: | + set -euo pipefail + + # Inspecting by digest proves the copy landed as the same bytes, which a tag + # lookup on its own would not. + docker buildx imagetools inspect "${MIRROR_IMAGE}@${DIGEST}" diff --git a/README.md b/README.md index 1b6d1d6..0ec847d 100644 --- a/README.md +++ b/README.md @@ -3,15 +3,17 @@

# Arca [![Release](https://github.com/pixelib/arca/actions/workflows/release.yml/badge.svg)](https://github.com/pixelib/arca/actions/workflows/release.yml) [![CI](https://github.com/pixelib/arca/actions/workflows/ci.yml/badge.svg)](https://github.com/pixelib/arca/actions/workflows/ci.yml) -A Maven, npm and Eclipse p2 repository server that runs as a single Go binary. +A Maven, npm, Docker and Eclipse p2 repository server that runs as a single Go binary. ![Arca](docs/assets/img/repositories.jpg) Artifacts live on disk, metadata lives in SQLite, and the React UI is embedded in the binary. There is no separate database to run and no config file to write before the first start. -- Maven 2 and npm repositories, hosted or proxying a remote +- Maven 2, npm and Docker repositories, hosted or proxying a remote - Eclipse p2 update sites, hosted or proxied and cached, with composite groups for PDE and Tycho +- Container images pushed and pulled with any Docker client, with layers, platforms and build history + in the UI - Release, prerelease and mixed version policies - Public repositories readable without an account, private ones granted per user - Generated `maven-metadata.xml` and npm packuments, built from the database @@ -38,8 +40,8 @@ Open and complete the setup wizard. There is a ## Docs -**[pixelib.github.io/arca](https://pixelib.github.io/arca)** covers using it from Maven -and npm, deploying it, and running an installation. +**[pixelib.github.io/arca](https://pixelib.github.io/arca)** covers using it from Maven, +npm and Docker, deploying it, and running an installation. A live instance is at [repo.pixelib.dev](https://repo.pixelib.dev). @@ -65,6 +67,7 @@ make test # go tests plus a frontend typecheck | `internal/maven` | Maven coordinates, version ordering, metadata rendering | | `internal/npm` | npm routes, semver, packuments, publish parsing | | `internal/p2` | p2 paths, OSGi versions, artifact document rewriting | +| `internal/docker` | image names, tags, digests, the storage layout and manifest parsing | | `internal/proxy` | upstream HTTP client and fetch deduplication | | `internal/blob` | artifact storage on disk | | `internal/frontend` | React app plus the `go:embed` handler | diff --git a/docs/_config.yml b/docs/_config.yml index b71f4d3..4c4244d 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -1,5 +1,5 @@ title: Arca -description: A Maven and npm repository server that runs as a single Go binary. +description: A Maven, npm, Docker and Eclipse p2 repository server that runs as a single Go binary. url: https://pixelib.github.io baseurl: /arca diff --git a/docs/deploying.md b/docs/deploying.md index b5b7915..2f53677 100644 --- a/docs/deploying.md +++ b/docs/deploying.md @@ -1,7 +1,7 @@ --- title: Deploying layout: default -nav_order: 7 +nav_order: 8 --- # Deploying diff --git a/docs/docker.md b/docs/docker.md new file mode 100644 index 0000000..4a60e46 --- /dev/null +++ b/docs/docker.md @@ -0,0 +1,138 @@ +--- +title: Using it from Docker +layout: default +nav_order: 6 +--- + +# Using it from Docker + +Create a repository with a format of `docker`. The format is fixed once the repository exists, +because nothing migrates between formats. + +Unlike the other formats there is no `/repository/` URL to point a client at. A Docker client builds +its own URLs from the image reference, so the registry is the bare host and the repository name is +the first segment of the image: + +``` +docker pull repo.example.com/docker-hosted/team/api:1.4.0 +``` + +That reaches the `team/api` image of the `docker-hosted` repository. One host serves every +repository this way, with no extra port and no hostname of its own. + +## Signing in + +``` +docker login repo.example.com --username you@example.com +``` + +Use an API token from your account page as the password. Your account password works too, but a +token can be revoked on its own. + +A public repository needs no credentials to pull. Pushing always does. + +## Pushing + +``` +docker tag your-image:1.4.0 repo.example.com/docker-hosted/team/api:1.4.0 +docker push repo.example.com/docker-hosted/team/api:1.4.0 +``` + +Multi-platform images work as they are: `docker buildx build --push` sends an index and one manifest +per platform, and the repository page shows the platform matrix. + +## Pulling + +``` +docker pull repo.example.com/docker-hosted/team/api:1.4.0 +docker pull repo.example.com/docker-hosted/team/api@sha256:6c6e1260a377... +``` + +A tag can be moved to another image. Pulling by digest is the only reference that always resolves to +the same bytes, which is what to pin in a deployment. + +In a Dockerfile or a compose file: + +``` +FROM repo.example.com/docker-hosted/team/api:1.4.0 +``` + +```yaml +services: + app: + image: repo.example.com/docker-hosted/team/api:1.4.0 +``` + +## Dropping the repository prefix + +Set a default docker repository under **Manage, Settings** and a bare reference resolves against it: + +``` +docker pull repo.example.com/team/api:1.4.0 +``` + +The leading segment still wins when it names a docker repository, so both forms keep working. This +is what makes a single-repository install read the way Docker Hub does. + +## Tags and policy + +A tag is a version, so a repository's release or prerelease policy applies to it. Docker tags carry +arbitrary suffixes, though: `1.25-alpine` names a variant and `0.1.9-swaggerui-staging` names +whatever its author meant. Only a known marker (`rc`, `alpha`, `beta`, `dev`, `pre`, `snapshot`, +`nightly`, `edge`) counts as a prerelease, and everything else is a release. A **mixed** policy is +the honest default for that reason. + +Moving a tag is ordinary Docker practice, so a docker repository allows it by default. Turn off +**Allow tags to be moved** to make every tag here permanent, and a push over an existing one answers +409. + +## Proxying another registry + +Create a repository with a type of **proxy** and a remote URL: + +| Registry | Remote URL | +| --- | --- | +| Docker Hub | `https://registry-1.docker.io` | +| GitHub Container Registry | `https://ghcr.io` | +| Quay | `https://quay.io` | +| Google Artifact Registry | `https://-docker.pkg.dev` | + +Pull through it and each image is fetched once, then served from here. Docker Hub's official images +live under an implicit `library/` scope, which is applied for you: `docker pull +repo.example.com/hub/nginx` resolves `library/nginx` upstream. + +Registries that require credentials use a token service rather than accepting them directly. Set the +remote username and password and the exchange is handled for you. Credentials are only ever sent to +an HTTPS token service. + +A proxy caches a tag for its metadata TTL, because upstream can move it, and caches everything +content-addressed for ever, because a digest names one document for all time. Set a cache retention +in days to evict what nothing has pulled recently. + +## Reclaiming storage + +Deleting a tag does not free its layers. That is deliberate: layers are shared, so any other tag may +still need them. The artifact page says how much of a tag is unique to it for exactly this reason. + +**Manage, Maintenance** has a **Reclaim docker storage** action that removes layers and untagged +manifests no tag can reach any more. Check first and it reports what would go without touching +anything. It also runs on its own every few hours. + +Proxy repositories are left alone by it: their content is refetchable and ages out by last access +instead, through the cache retention setting. + +## Large layers + +The artifact upload limit does not apply to container layers, which are routinely larger than any +sane ceiling for a jar. Set `--max-blob` or `MAX_BLOB_BYTES` to cap them; the default is no limit. + +## What is not supported + +- **Schema 1 manifests.** Deprecated for years and rejected on push. Anything built this decade + sends schema 2 or OCI. +- **Manifest conversion.** A manifest is served as the type it was pushed with, because re-encoding + it would change the digest it is addressed by. Every current client accepts both encodings. +- **The referrers API.** Signatures and SBOMs attached with `cosign attach` or `docker buildx + --attest` are stored and served, since they are ordinary manifests, but they are not discoverable + through `/v2//referrers/`. Clients fall back to the tag scheme, which works. +- **Docker groups.** One URL over several repositories is not implemented yet. diff --git a/docs/index.md b/docs/index.md index 0b72e51..7752b3c 100644 --- a/docs/index.md +++ b/docs/index.md @@ -6,24 +6,28 @@ nav_order: 1 # Arca -An artifact repository server that runs as a single Go binary. It speaks Maven 2, npm and Eclipse p2, -stores artifacts on disk, keeps metadata in SQLite, and serves an embedded React UI from the same -process. There is no separate database to run, no application server, and no config file to write +An artifact repository server that runs as a single Go binary. It speaks Maven 2, npm, Docker and +Eclipse p2, stores artifacts on disk, keeps metadata in SQLite, and serves an embedded React UI from +the same process. There is no separate database to run, no application server, and no config file to write before the first start. ![The repository index]({{ site.baseurl }}/assets/img/repositories.jpg) ## What it does -- **Three formats.** Maven 2, npm and Eclipse p2, chosen per repository and fixed once it exists. +- **Four formats.** Maven 2, npm, Docker and Eclipse p2, chosen per repository and fixed once it + exists. - **Hosted, proxy and group repositories.** Publish to a hosted repository, mirror Maven Central, - registry.npmjs.org and download.eclipse.org through a proxy that caches what it serves, or put a - p2 group over several update sites so a target platform needs one location instead of a dozen. + registry.npmjs.org, Docker Hub and download.eclipse.org through a proxy that caches what it serves, + or put a p2 group over several update sites so a target platform needs one location instead of a + dozen. - **Version policies.** A repository takes releases only, prereleases only, or both. - **Per user permissions.** Public repositories are readable without an account. Private ones are granted per user, per repository. - **Generated metadata.** `maven-metadata.xml` and npm packuments are built from the database, so they cannot drift from what is actually stored. +- **Container images as first-class artifacts.** Push and pull with any Docker client, then read the + layers, platforms, build history and how much of a tag is unique to it rather than shared. - **Insights.** Traffic, storage and per version artifact management, with the numbers coming from real requests rather than estimates. - **Your own front page.** A markdown welcome text, an accent colour and a logo of your own, so the @@ -37,6 +41,7 @@ before the first start. | Resolve and publish jars | [Using it from Maven]({{ site.baseurl }}/maven) | | Resolve and publish packages | [Using it from npm]({{ site.baseurl }}/npm) | | Resolve Eclipse bundles and features | [Using it from Eclipse]({{ site.baseurl }}/p2) | +| Push and pull container images | [Using it from Docker]({{ site.baseurl }}/docker) | | Move off Sonatype Nexus | [Migrating from Nexus]({{ site.baseurl }}/migrating) | | Put it on a server | [Deploying]({{ site.baseurl }}/deploying) | | Run it day to day | [Managing an installation]({{ site.baseurl }}/managing) | @@ -52,11 +57,12 @@ Search spans every repository you can read, in any format, matching names and na ## How it fits together -One process serves three things on the same port: +One process serves four things on the same port: | Path | Serves | | --- | --- | -| `/repository//**` | the Maven or npm endpoint, depending on the repository format | +| `/repository//**` | the Maven, npm or p2 endpoint, depending on the repository format | +| `/v2/**` | the Docker registry API, which clients build their own URLs for and so cannot be given a prefix | | `/api/**` | the JSON API the UI runs on | | everything else | the embedded React bundle | diff --git a/docs/managing.md b/docs/managing.md index da48731..1951191 100644 --- a/docs/managing.md +++ b/docs/managing.md @@ -1,7 +1,7 @@ --- title: Managing an installation layout: default -nav_order: 8 +nav_order: 9 --- # Managing an installation diff --git a/docs/migrating.md b/docs/migrating.md index 94baf78..8289e44 100644 --- a/docs/migrating.md +++ b/docs/migrating.md @@ -1,7 +1,7 @@ --- title: Migrating from Nexus layout: default -nav_order: 6 +nav_order: 7 --- ![Arca]({{ site.baseurl }}/assets/img/nexusimport.png) @@ -33,8 +33,9 @@ either side. Arca lists every repository it found and what it intends to do with | `maven-central` | maven2 proxy | Recreated pointing at the same upstream, cache not copied | | `maven-releases` | maven2 hosted | Assets are copied across | | `p2-releases` | p2 hosted | Assets are copied across | +| `docker-hosted` | docker hosted | Images are copied across, with their tags | | `maven-public` | skipped | groups are not migrated; recreate it as a p2 group, or point clients at the members directly (maven-central, maven-releases) | -| `docker-hosted` | skipped | arca has no docker format | +| `gems` | skipped | arca has no rubygems format | The password is used for that run and never stored. @@ -53,6 +54,8 @@ Only one migration runs at a time. | `maven2` hosted | a hosted `maven2` repository, every asset copied | | `maven2` proxy | a proxy pointing at the same upstream, cache **not** copied | | `npm` hosted or proxy | the same, as `npm` | +| `docker` hosted | a hosted `docker` repository, every image and tag copied | +| `docker` proxy | a proxy pointing at the same registry, cache **not** copied | | `raw` | a `p2` repository, every asset copied | | group | skipped | | any other format | skipped | @@ -68,6 +71,38 @@ become** if one of yours holds something else. A Maven repository's version policy carries across, so a `SNAPSHOT` repository in Nexus arrives as a snapshot repository here. +## Docker repositories + +A docker repository takes an extra step, because Nexus exposes it through two listings that do not +overlap. Its asset list holds the blobs and the manifests addressed by digest, and names no tag at +all; the tags are only in its component list. A copy that read one and not the other would transfer +every byte of every image and leave none of them pullable, so both are walked. + +Nexus files blobs on one shared path regardless of which image used them, and addresses everything +under a `v2/` prefix, so paths are translated on the way in: + +| Nexus | Becomes | +| --- | --- | +| `v2/-/blobs/sha256:...` | the shared blob store | +| `v2//blobs/sha256:...` | the same store, since a digest names the same bytes | +| `v2//manifests/sha256:...` | the manifest, addressed by digest | +| `v2//manifests/` | the tag | + +Anything else under `v2/` has no place here and is counted as such in the progress line, rather than +stored somewhere nothing would read it back from. `v2//tags/list` is the usual one. + +Once a repository's content is all in place, every manifest is read and indexed. That has to wait +until the end: a manifest's platform and labels come from a config blob, and the copy order gives no +guarantee of having fetched it yet. If a run is interrupted, **Rebuild metadata** under +**Manage, Maintenance** finishes the job without recopying anything. + +Docker repositories arrive with a **mixed** policy. Docker tags carry arbitrary suffixes, and a real +one like `0.1.9-swaggerui-staging` is neither a release nor a prerelease by any rule worth writing. + +Nexus serves docker on its own port through a connector. Arca does not need one: the repository name +is the first segment of the image, so every repository is reachable on the same port the UI is. The +connector settings are not read, and on some Nexus versions the REST API withholds them anyway. + Names are lowercased, because arca repository names are. A Nexus name that is still not valid after that is skipped and left for you to create by hand. @@ -78,6 +113,10 @@ deliberately rather than inheriting, and arca only has groups for p2. The previe of every group it skipped, so you can either point clients at those directly or, for p2, rebuild it as a [group repository]({{ site.baseurl }}/p2#one-url-over-several-sites) afterwards. +Some Nexus versions withhold a group's members from the REST API. The preview says so rather than +reporting an empty list, since a group with no members and a group whose members could not be read +are very different things. + ## Interruptions Start it again. Repositories that already exist are reported as such, and assets already present are diff --git a/internal/blob/store.go b/internal/blob/store.go index a2955f5..5ef285a 100644 --- a/internal/blob/store.go +++ b/internal/blob/store.go @@ -16,8 +16,11 @@ import ( ) var ( - ErrInvalidKey = errors.New("blob: key escapes the storage root") - ErrTooLarge = errors.New("blob: payload exceeds the configured limit") + ErrInvalidKey = errors.New("blob: key escapes the storage root") + ErrTooLarge = errors.New("blob: payload exceeds the configured limit") + ErrNoSuchUpload = errors.New("blob: there is no upload session with that id") + ErrDigestMismatch = errors.New("blob: the upload does not match the digest it was committed with") + ErrInvalidUploadID = errors.New("blob: the upload id is not one this store issues") ) const temporaryDirectory = ".uploads" @@ -29,6 +32,33 @@ type Digests struct { SHA512 string } +// digesters computes every checksum an asset row carries in one pass. Upload +// sessions hash a file that is already on disk, so this is shared rather than +// inlined into Put. +type digesters struct { + md5 hash.Hash + sha1 hash.Hash + sha256 hash.Hash + sha512 hash.Hash +} + +func newDigesters() digesters { + return digesters{md5: md5.New(), sha1: sha1.New(), sha256: sha256.New(), sha512: sha512.New()} +} + +func (d digesters) writer() io.Writer { + return io.MultiWriter(d.md5, d.sha1, d.sha256, d.sha512) +} + +func (d digesters) digests() Digests { + return Digests{ + MD5: hex.EncodeToString(d.md5.Sum(nil)), + SHA1: hex.EncodeToString(d.sha1.Sum(nil)), + SHA256: hex.EncodeToString(d.sha256.Sum(nil)), + SHA512: hex.EncodeToString(d.sha512.Sum(nil)), + } +} + type Store struct { root string } @@ -71,24 +101,14 @@ func (s *Store) Put(key string, reader io.Reader, limit int64) (int64, Digests, os.Remove(temporary.Name()) }() - digesters := map[string]hash.Hash{ - "md5": md5.New(), - "sha1": sha1.New(), - "sha256": sha256.New(), - "sha512": sha512.New(), - } - - writers := []io.Writer{temporary} - for _, digester := range digesters { - writers = append(writers, digester) - } + hashes := newDigesters() source := reader if limit > 0 { source = io.LimitReader(reader, limit+1) } - size, err := io.Copy(io.MultiWriter(writers...), source) + size, err := io.Copy(io.MultiWriter(temporary, hashes.writer()), source) if err != nil { return 0, Digests{}, fmt.Errorf("buffer upload: %w", err) } @@ -106,13 +126,7 @@ func (s *Store) Put(key string, reader io.Reader, limit int64) (int64, Digests, return 0, Digests{}, fmt.Errorf("store upload: %w", err) } - digests := Digests{ - MD5: hex.EncodeToString(digesters["md5"].Sum(nil)), - SHA1: hex.EncodeToString(digesters["sha1"].Sum(nil)), - SHA256: hex.EncodeToString(digesters["sha256"].Sum(nil)), - SHA512: hex.EncodeToString(digesters["sha512"].Sum(nil)), - } - return size, digests, nil + return size, hashes.digests(), nil } func (s *Store) Open(key string) (*os.File, os.FileInfo, error) { @@ -134,6 +148,29 @@ func (s *Store) Open(key string) (*os.File, os.FileInfo, error) { return file, info, nil } +// SizeOf totals the named files, so a sweep can report how much it reclaimed. A key +// that has already gone contributes nothing rather than failing the measurement. +func (s *Store) SizeOf(keys []string) (int64, error) { + var total int64 + + for _, key := range keys { + target, err := s.resolve(key) + if err != nil { + return total, err + } + + info, err := os.Stat(target) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return total, err + } + total += info.Size() + } + return total, nil +} + func (s *Store) Delete(keys ...string) error { for _, key := range keys { target, err := s.resolve(key) diff --git a/internal/blob/upload.go b/internal/blob/upload.go new file mode 100644 index 0000000..3399f3b --- /dev/null +++ b/internal/blob/upload.go @@ -0,0 +1,191 @@ +package blob + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" +) + +const uploadPrefix = "session-" + +// Upload sessions back the registry API's chunked blob push, where one blob +// arrives across a POST, any number of PATCHes and a PUT. Put cannot serve that +// shape because it writes a whole reader in one call. +// +// A session is a file in the same temporary directory Put uses, so an abandoned +// one costs nothing but the space until it is swept. + +func isValidUploadID(id string) bool { + if id == "" || len(id) > 128 { + return false + } + for index := 0; index < len(id); index++ { + character := id[index] + switch { + case character >= 'a' && character <= 'z', + character >= 'A' && character <= 'Z', + character >= '0' && character <= '9', + character == '-', character == '_': + default: + return false + } + } + return true +} + +func (s *Store) uploadPath(id string) (string, error) { + if !isValidUploadID(id) { + return "", ErrInvalidUploadID + } + return filepath.Join(s.root, temporaryDirectory, uploadPrefix+id), nil +} + +// BeginUpload opens an empty session. It fails if one already exists under the +// same id rather than truncating, because a repeated id means the caller lost +// track of a session that may still be receiving chunks. +func (s *Store) BeginUpload(id string) error { + path, err := s.uploadPath(id) + if err != nil { + return err + } + + file, err := os.OpenFile(path, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o644) + if err != nil { + return fmt.Errorf("open upload session: %w", err) + } + return file.Close() +} + +// AppendUpload adds a chunk and reports the total size of the session. A chunk +// that would push the session past limit is rejected without being written, so +// a client cannot fill the disk one PATCH at a time. +func (s *Store) AppendUpload(id string, reader io.Reader, limit int64) (int64, error) { + path, err := s.uploadPath(id) + if err != nil { + return 0, err + } + + file, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0o644) + if errors.Is(err, os.ErrNotExist) { + return 0, ErrNoSuchUpload + } + if err != nil { + return 0, fmt.Errorf("open upload session: %w", err) + } + defer file.Close() + + info, err := file.Stat() + if err != nil { + return 0, fmt.Errorf("measure upload session: %w", err) + } + started := info.Size() + + source := reader + if limit > 0 { + remaining := limit - started + if remaining < 0 { + remaining = 0 + } + source = io.LimitReader(reader, remaining+1) + } + + written, err := io.Copy(file, source) + if err != nil { + return started + written, fmt.Errorf("write upload chunk: %w", err) + } + if limit > 0 && started+written > limit { + // The overshoot is truncated away so the session stays consistent with + // the size the caller is told about. + if truncateErr := file.Truncate(started); truncateErr != nil { + return started + written, fmt.Errorf("truncate oversized upload: %w", truncateErr) + } + return started, ErrTooLarge + } + + return started + written, nil +} + +func (s *Store) UploadSize(id string) (int64, error) { + path, err := s.uploadPath(id) + if err != nil { + return 0, err + } + + info, err := os.Stat(path) + if errors.Is(err, os.ErrNotExist) { + return 0, ErrNoSuchUpload + } + if err != nil { + return 0, fmt.Errorf("measure upload session: %w", err) + } + return info.Size(), nil +} + +// CompleteUpload verifies the session against the digest the client committed it +// with and only then moves it into place. Hashing here costs one extra pass over +// a file that has already crossed the network, and it is the only point at which +// the bytes on disk can be checked against what the client said they were. +// +// A mismatch leaves the session alone so the caller can decide whether to let +// the client retry or abort it. +func (s *Store) CompleteUpload(id, key, expectedSHA256 string) (int64, Digests, error) { + source, err := s.uploadPath(id) + if err != nil { + return 0, Digests{}, err + } + + target, err := s.resolve(key) + if err != nil { + return 0, Digests{}, err + } + + size, digests, err := digestFile(source) + if errors.Is(err, os.ErrNotExist) { + return 0, Digests{}, ErrNoSuchUpload + } + if err != nil { + return 0, Digests{}, err + } + + if !strings.EqualFold(digests.SHA256, expectedSHA256) { + return size, digests, ErrDigestMismatch + } + + if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil { + return size, digests, fmt.Errorf("create storage directory: %w", err) + } + if err := os.Rename(source, target); err != nil { + return size, digests, fmt.Errorf("store upload: %w", err) + } + return size, digests, nil +} + +func (s *Store) AbortUpload(id string) error { + path, err := s.uploadPath(id) + if err != nil { + return err + } + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return nil +} + +func digestFile(path string) (int64, Digests, error) { + file, err := os.Open(path) + if err != nil { + return 0, Digests{}, err + } + defer file.Close() + + digesters := newDigesters() + + size, err := io.Copy(digesters.writer(), file) + if err != nil { + return 0, Digests{}, fmt.Errorf("hash upload session: %w", err) + } + return size, digesters.digests(), nil +} diff --git a/internal/blob/upload_test.go b/internal/blob/upload_test.go new file mode 100644 index 0000000..7159c24 --- /dev/null +++ b/internal/blob/upload_test.go @@ -0,0 +1,268 @@ +package blob + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "errors" + "os" + "path/filepath" + "strings" + "testing" +) + +func newUploadStore(t *testing.T) *Store { + t.Helper() + + store, err := New(t.TempDir()) + if err != nil { + t.Fatalf("New: %v", err) + } + return store +} + +func sha256Of(payload []byte) string { + sum := sha256.Sum256(payload) + return hex.EncodeToString(sum[:]) +} + +func TestUploadSessionAcrossChunks(t *testing.T) { + store := newUploadStore(t) + payload := []byte("the whole blob, delivered in pieces") + + if err := store.BeginUpload("session-one"); err != nil { + t.Fatalf("BeginUpload: %v", err) + } + + size, err := store.UploadSize("session-one") + if err != nil || size != 0 { + t.Fatalf("a fresh session reported size %d, %v, want 0", size, err) + } + + chunks := [][]byte{payload[:10], payload[10:20], payload[20:]} + expected := 0 + for _, chunk := range chunks { + expected += len(chunk) + + size, err := store.AppendUpload("session-one", bytes.NewReader(chunk), 0) + if err != nil { + t.Fatalf("AppendUpload: %v", err) + } + if size != int64(expected) { + t.Fatalf("after appending %d bytes the session is %d, want %d", len(chunk), size, expected) + } + } + + size, digests, err := store.CompleteUpload("session-one", "repo/blob", sha256Of(payload)) + if err != nil { + t.Fatalf("CompleteUpload: %v", err) + } + if size != int64(len(payload)) { + t.Fatalf("size = %d, want %d", size, len(payload)) + } + if digests.SHA256 != sha256Of(payload) { + t.Fatalf("SHA256 = %q, want %q", digests.SHA256, sha256Of(payload)) + } + + file, _, err := store.Open("repo/blob") + if err != nil { + t.Fatalf("the committed blob is not readable: %v", err) + } + file.Close() +} + +// A mismatch must leave the session in place, so the caller can decide whether the +// client retries or the session is abandoned. +func TestCompleteUploadRejectsAMismatch(t *testing.T) { + store := newUploadStore(t) + + if err := store.BeginUpload("mismatch"); err != nil { + t.Fatalf("BeginUpload: %v", err) + } + if _, err := store.AppendUpload("mismatch", strings.NewReader("actual content"), 0); err != nil { + t.Fatalf("AppendUpload: %v", err) + } + + _, digests, err := store.CompleteUpload("mismatch", "repo/blob", sha256Of([]byte("different content"))) + if !errors.Is(err, ErrDigestMismatch) { + t.Fatalf("CompleteUpload returned %v, want ErrDigestMismatch", err) + } + if digests.SHA256 != sha256Of([]byte("actual content")) { + t.Fatalf("the reported digest is not the one on disk: %q", digests.SHA256) + } + + if _, err := store.UploadSize("mismatch"); err != nil { + t.Fatalf("the session was discarded on mismatch: %v", err) + } + if _, _, err := store.Open("repo/blob"); err == nil { + t.Fatal("a mismatched upload was moved into place") + } +} + +func TestAppendUploadHonoursTheLimit(t *testing.T) { + store := newUploadStore(t) + + if err := store.BeginUpload("bounded"); err != nil { + t.Fatalf("BeginUpload: %v", err) + } + if _, err := store.AppendUpload("bounded", strings.NewReader("12345"), 10); err != nil { + t.Fatalf("the first chunk was refused: %v", err) + } + + size, err := store.AppendUpload("bounded", strings.NewReader("678901234"), 10) + if !errors.Is(err, ErrTooLarge) { + t.Fatalf("AppendUpload returned %v, want ErrTooLarge", err) + } + // The overshoot is truncated, so the session stays consistent with the size + // the caller was told about. + if size != 5 { + t.Fatalf("size = %d, want 5", size) + } + + stored, err := store.UploadSize("bounded") + if err != nil { + t.Fatalf("UploadSize: %v", err) + } + if stored != 5 { + t.Fatalf("the session is %d bytes on disk, want 5", stored) + } +} + +func TestUploadSessionErrors(t *testing.T) { + store := newUploadStore(t) + + t.Run("appending to an unknown session", func(t *testing.T) { + if _, err := store.AppendUpload("absent", strings.NewReader("x"), 0); !errors.Is(err, ErrNoSuchUpload) { + t.Fatalf("err = %v, want ErrNoSuchUpload", err) + } + }) + + t.Run("measuring an unknown session", func(t *testing.T) { + if _, err := store.UploadSize("absent"); !errors.Is(err, ErrNoSuchUpload) { + t.Fatalf("err = %v, want ErrNoSuchUpload", err) + } + }) + + t.Run("completing an unknown session", func(t *testing.T) { + if _, _, err := store.CompleteUpload("absent", "repo/blob", sha256Of(nil)); !errors.Is(err, ErrNoSuchUpload) { + t.Fatalf("err = %v, want ErrNoSuchUpload", err) + } + }) + + t.Run("reopening a live session", func(t *testing.T) { + if err := store.BeginUpload("taken"); err != nil { + t.Fatalf("BeginUpload: %v", err) + } + if err := store.BeginUpload("taken"); err == nil { + t.Fatal("BeginUpload truncated a session that already existed") + } + }) + + t.Run("aborting an unknown session is not an error", func(t *testing.T) { + if err := store.AbortUpload("absent"); err != nil { + t.Fatalf("AbortUpload: %v", err) + } + }) +} + +// An upload id becomes a filename, so one carrying separators would let a caller +// write outside the temporary directory. +func TestUploadIDsCannotEscape(t *testing.T) { + store := newUploadStore(t) + + cases := []string{ + "", + "../escape", + "a/b", + "a\\b", + "with space", + "with.dot", + strings.Repeat("a", 129), + } + + for _, id := range cases { + t.Run(id, func(t *testing.T) { + if err := store.BeginUpload(id); !errors.Is(err, ErrInvalidUploadID) { + t.Fatalf("BeginUpload(%q) = %v, want ErrInvalidUploadID", id, err) + } + if _, err := store.UploadSize(id); !errors.Is(err, ErrInvalidUploadID) { + t.Fatalf("UploadSize(%q) = %v, want ErrInvalidUploadID", id, err) + } + if err := store.AbortUpload(id); !errors.Is(err, ErrInvalidUploadID) { + t.Fatalf("AbortUpload(%q) = %v, want ErrInvalidUploadID", id, err) + } + }) + } +} + +func TestAbortUploadRemovesTheFile(t *testing.T) { + store := newUploadStore(t) + + if err := store.BeginUpload("doomed"); err != nil { + t.Fatalf("BeginUpload: %v", err) + } + if _, err := store.AppendUpload("doomed", strings.NewReader("content"), 0); err != nil { + t.Fatalf("AppendUpload: %v", err) + } + if err := store.AbortUpload("doomed"); err != nil { + t.Fatalf("AbortUpload: %v", err) + } + + matches, err := filepath.Glob(filepath.Join(store.Root(), temporaryDirectory, uploadPrefix+"*")) + if err != nil { + t.Fatalf("Glob: %v", err) + } + if len(matches) != 0 { + t.Fatalf("the aborted session is still on disk: %v", matches) + } +} + +// Concurrent sessions are the normal case: buildx pushes several layers at once, +// and nothing may serialise them. +func TestConcurrentUploadSessions(t *testing.T) { + store := newUploadStore(t) + + payloads := map[string][]byte{ + "first": []byte("the first layer"), + "second": []byte("the second layer, which is longer"), + "third": []byte("third"), + } + + for id := range payloads { + if err := store.BeginUpload(id); err != nil { + t.Fatalf("BeginUpload(%q): %v", id, err) + } + } + // Interleaved deliberately, so a shared handle or a shared offset would show + // up as cross-contamination. + for id, payload := range payloads { + if _, err := store.AppendUpload(id, bytes.NewReader(payload[:3]), 0); err != nil { + t.Fatalf("AppendUpload(%q): %v", id, err) + } + } + for id, payload := range payloads { + if _, err := store.AppendUpload(id, bytes.NewReader(payload[3:]), 0); err != nil { + t.Fatalf("AppendUpload(%q): %v", id, err) + } + } + + for id, payload := range payloads { + t.Run(id, func(t *testing.T) { + _, digests, err := store.CompleteUpload(id, "repo/"+id, sha256Of(payload)) + if err != nil { + t.Fatalf("CompleteUpload: %v", err) + } + if digests.SHA256 != sha256Of(payload) { + t.Fatalf("session %q committed the wrong bytes", id) + } + + content, err := os.ReadFile(filepath.Join(store.Root(), "repo", id)) + if err != nil { + t.Fatalf("reading the committed blob: %v", err) + } + if !bytes.Equal(content, payload) { + t.Fatalf("session %q holds %q, want %q", id, content, payload) + } + }) + } +} diff --git a/internal/config/config.go b/internal/config/config.go index d8c11da..3b4d4a3 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -13,13 +13,18 @@ const ( ) type Config struct { - DevMode bool - Port int - DatabasePath string - StorageDir string + DevMode bool + Port int + DatabasePath string + StorageDir string + MaxUploadBytes int64 - ProxyTimeout time.Duration - ViteURL string + // MaxBlobBytes limits a container layer, which routinely runs past whatever + // is a sane ceiling for a jar or a tarball. Zero means unlimited. + MaxBlobBytes int64 + + ProxyTimeout time.Duration + ViteURL string } func Parse(arguments []string) (Config, error) { @@ -34,6 +39,7 @@ func Parse(arguments []string) (Config, error) { flags.StringVar(&config.DatabasePath, "db", envString("DATABASE_PATH", "./data/arca.db"), "path to the SQLite database file") flags.StringVar(&config.StorageDir, "storage", envString("STORAGE_DIR", "./data/storage"), "directory holding uploaded artifacts") flags.Int64Var(&config.MaxUploadBytes, "max-upload", envInt64("MAX_UPLOAD_BYTES", DefaultMaxUploadBytes), "largest artifact accepted, in bytes") + flags.Int64Var(&config.MaxBlobBytes, "max-blob", envInt64("MAX_BLOB_BYTES", 0), "largest container image layer accepted, in bytes, or 0 for no limit") flags.StringVar(&config.ViteURL, "vite-url", envString("VITE_URL", "http://localhost:5173"), "vite dev server to proxy in dev mode") if err := flags.Parse(arguments); err != nil { @@ -42,6 +48,9 @@ func Parse(arguments []string) (Config, error) { if config.MaxUploadBytes <= 0 { config.MaxUploadBytes = DefaultMaxUploadBytes } + if config.MaxBlobBytes < 0 { + config.MaxBlobBytes = 0 + } config.ProxyTimeout = time.Duration(proxyTimeoutSeconds) * time.Second if config.ProxyTimeout <= 0 { diff --git a/internal/docker/config.go b/internal/docker/config.go new file mode 100644 index 0000000..2754a72 --- /dev/null +++ b/internal/docker/config.go @@ -0,0 +1,126 @@ +package docker + +import ( + "encoding/json" + "fmt" + "sort" + "time" +) + +// Config is the subset of an image config blob worth showing. Times are +// milliseconds since the epoch, as every other timestamp in this server is. +type Config struct { + Architecture string + OS string + Variant string + OSVersion string + Created int64 + User string + WorkingDir string + Entrypoint []string + Cmd []string + Env []string + Labels map[string]string + ExposedPorts []string + DiffIDs []string + History []HistoryEntry +} + +// HistoryEntry is one build step. An empty layer marks an instruction that +// changed metadata without producing a filesystem layer, which is what makes the +// history longer than the layer list. +type HistoryEntry struct { + Created int64 + CreatedBy string + Comment string + EmptyLayer bool +} + +type configDocument struct { + Architecture string `json:"architecture"` + OS string `json:"os"` + Variant string `json:"variant"` + OSVersion string `json:"os.version"` + Created string `json:"created"` + Config struct { + User string `json:"User"` + WorkingDir string `json:"WorkingDir"` + Entrypoint []string `json:"Entrypoint"` + Cmd []string `json:"Cmd"` + Env []string `json:"Env"` + Labels map[string]string `json:"Labels"` + ExposedPorts map[string]any `json:"ExposedPorts"` + } `json:"config"` + RootFS struct { + DiffIDs []string `json:"diff_ids"` + } `json:"rootfs"` + History []struct { + Created string `json:"created"` + CreatedBy string `json:"created_by"` + Comment string `json:"comment"` + EmptyLayer bool `json:"empty_layer"` + } `json:"history"` +} + +// millis reads an RFC 3339 timestamp. An absent or unreadable one reports zero +// rather than failing the parse: a config that cannot say when it was built is +// still worth everything else it carries. +func millis(raw string) int64 { + if raw == "" { + return 0 + } + parsed, err := time.Parse(time.RFC3339, raw) + if err != nil { + return 0 + } + return parsed.UnixMilli() +} + +func ParseConfig(document []byte) (Config, error) { + var parsed configDocument + if err := json.Unmarshal(document, &parsed); err != nil { + return Config{}, fmt.Errorf("docker: reading an image config: %w", err) + } + + config := Config{ + Architecture: parsed.Architecture, + OS: parsed.OS, + Variant: parsed.Variant, + OSVersion: parsed.OSVersion, + Created: millis(parsed.Created), + User: parsed.Config.User, + WorkingDir: parsed.Config.WorkingDir, + Entrypoint: parsed.Config.Entrypoint, + Cmd: parsed.Config.Cmd, + Env: parsed.Config.Env, + Labels: parsed.Config.Labels, + DiffIDs: parsed.RootFS.DiffIDs, + } + + // The ports arrive as a JSON object, so they are sorted rather than left in + // map order, which would reshuffle the UI on every read. + for port := range parsed.Config.ExposedPorts { + config.ExposedPorts = append(config.ExposedPorts, port) + } + sort.Strings(config.ExposedPorts) + + for _, entry := range parsed.History { + config.History = append(config.History, HistoryEntry{ + Created: millis(entry.Created), + CreatedBy: entry.CreatedBy, + Comment: entry.Comment, + EmptyLayer: entry.EmptyLayer, + }) + } + + return config, nil +} + +func (c Config) Platform() Platform { + return Platform{ + Architecture: c.Architecture, + OS: c.OS, + Variant: c.Variant, + OSVersion: c.OSVersion, + } +} diff --git a/internal/docker/config_test.go b/internal/docker/config_test.go new file mode 100644 index 0000000..5017939 --- /dev/null +++ b/internal/docker/config_test.go @@ -0,0 +1,138 @@ +package docker + +import "testing" + +const fullConfig = `{ + "architecture": "amd64", + "os": "linux", + "created": "2026-07-30T10:11:12Z", + "config": { + "User": "app", + "WorkingDir": "/srv", + "Entrypoint": ["/bin/arca"], + "Cmd": ["--port", "8080"], + "Env": ["PATH=/usr/bin", "PORT=8080"], + "Labels": {"org.opencontainers.image.source": "https://github.com/pixelib/arca"}, + "ExposedPorts": {"8080/tcp": {}, "443/tcp": {}} + }, + "rootfs": { + "type": "layers", + "diff_ids": ["sha256:1111111111111111111111111111111111111111111111111111111111111111"] + }, + "history": [ + {"created": "2026-07-30T10:11:10Z", "created_by": "COPY . /srv"}, + {"created": "2026-07-30T10:11:11Z", "created_by": "ENV PORT=8080", "empty_layer": true} + ] +}` + +func TestParseConfig(t *testing.T) { + config, err := ParseConfig([]byte(fullConfig)) + if err != nil { + t.Fatalf("ParseConfig returned %v", err) + } + + if config.Architecture != "amd64" || config.OS != "linux" { + t.Fatalf("platform = %q, want linux/amd64", config.Platform().String()) + } + if config.Created != 1785406272000 { + t.Fatalf("Created = %d, want 1785406272000", config.Created) + } + if config.User != "app" || config.WorkingDir != "/srv" { + t.Fatalf("User = %q, WorkingDir = %q", config.User, config.WorkingDir) + } + if len(config.Entrypoint) != 1 || config.Entrypoint[0] != "/bin/arca" { + t.Fatalf("Entrypoint = %v", config.Entrypoint) + } + if len(config.Env) != 2 { + t.Fatalf("Env = %v, want 2 entries", config.Env) + } + if config.Labels["org.opencontainers.image.source"] == "" { + t.Fatalf("Labels = %v, want the source label", config.Labels) + } + if len(config.DiffIDs) != 1 { + t.Fatalf("DiffIDs = %v, want 1 entry", config.DiffIDs) + } +} + +// Ports arrive as a JSON object, so they are sorted rather than left in map +// order, which would reshuffle the UI between reads of the same config. +func TestParseConfigSortsExposedPorts(t *testing.T) { + config, err := ParseConfig([]byte(fullConfig)) + if err != nil { + t.Fatalf("ParseConfig returned %v", err) + } + + want := []string{"443/tcp", "8080/tcp"} + if len(config.ExposedPorts) != len(want) { + t.Fatalf("ExposedPorts = %v, want %v", config.ExposedPorts, want) + } + for index := range want { + if config.ExposedPorts[index] != want[index] { + t.Fatalf("ExposedPorts = %v, want %v", config.ExposedPorts, want) + } + } +} + +// The history is longer than the layer list, because a metadata-only +// instruction produces an entry with no filesystem layer behind it. +func TestParseConfigHistory(t *testing.T) { + config, err := ParseConfig([]byte(fullConfig)) + if err != nil { + t.Fatalf("ParseConfig returned %v", err) + } + + if len(config.History) != 2 { + t.Fatalf("History = %v, want 2 entries", config.History) + } + if config.History[0].EmptyLayer { + t.Fatal("the first history entry reported an empty layer") + } + if !config.History[1].EmptyLayer { + t.Fatal("the second history entry did not report an empty layer") + } + if config.History[0].Created == 0 { + t.Fatal("the first history entry has no timestamp") + } +} + +// A config missing every optional field still has to parse. Partially populated +// configs are normal, and common in a repository that was migrated rather than +// pushed. +func TestParseConfigPartial(t *testing.T) { + cases := []struct { + name string + document string + }{ + {"empty object", `{}`}, + {"no config block", `{"architecture":"amd64","os":"linux"}`}, + {"no timestamp", `{"architecture":"amd64","config":{"Cmd":["sh"]}}`}, + {"unparseable timestamp", `{"created":"yesterday"}`}, + {"null labels", `{"config":{"Labels":null}}`}, + {"no history", `{"architecture":"amd64","rootfs":{"diff_ids":[]}}`}, + } + + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + config, err := ParseConfig([]byte(testCase.document)) + if err != nil { + t.Fatalf("ParseConfig returned %v", err) + } + // A config that cannot say when it was built is still worth + // everything else it carries, so an unreadable time reports zero + // rather than failing the parse. + if config.Created != 0 { + t.Fatalf("Created = %d, want 0", config.Created) + } + }) + } +} + +func TestParseConfigRejectsGarbage(t *testing.T) { + for _, document := range []string{"", "not json", "[]"} { + t.Run(document, func(t *testing.T) { + if _, err := ParseConfig([]byte(document)); err == nil { + t.Fatal("ParseConfig accepted a document that is not a config") + } + }) + } +} diff --git a/internal/docker/content_type.go b/internal/docker/content_type.go new file mode 100644 index 0000000..85db4cd --- /dev/null +++ b/internal/docker/content_type.go @@ -0,0 +1,11 @@ +package docker + +// ContentTypeFor is only a fallback. A manifest's real media type comes from the +// Content-Type it was pushed with, which the upload path records, because the +// filename cannot tell an OCI manifest from a Docker one or from an index. +func ContentTypeFor(filename string) string { + if filename == ManifestFilename { + return MediaTypeOCIManifest + } + return "application/octet-stream" +} diff --git a/internal/docker/errors.go b/internal/docker/errors.go new file mode 100644 index 0000000..e538231 --- /dev/null +++ b/internal/docker/errors.go @@ -0,0 +1,31 @@ +package docker + +// The registry API's error codes. A client reads these rather than the status +// line to tell a missing blob from a rejected manifest, so they are part of the +// protocol rather than decoration. +const ( + ErrorBlobUnknown = "BLOB_UNKNOWN" + ErrorBlobUploadInvalid = "BLOB_UPLOAD_INVALID" + ErrorBlobUploadUnknown = "BLOB_UPLOAD_UNKNOWN" + ErrorDigestInvalid = "DIGEST_INVALID" + ErrorManifestBlobUnknown = "MANIFEST_BLOB_UNKNOWN" + ErrorManifestInvalid = "MANIFEST_INVALID" + ErrorManifestUnknown = "MANIFEST_UNKNOWN" + ErrorNameInvalid = "NAME_INVALID" + ErrorNameUnknown = "NAME_UNKNOWN" + ErrorSizeInvalid = "SIZE_INVALID" + ErrorTagInvalid = "TAG_INVALID" + ErrorUnauthorized = "UNAUTHORIZED" + ErrorDenied = "DENIED" + ErrorUnsupported = "UNSUPPORTED" + ErrorTooManyRequests = "TOOMANYREQUESTS" +) + +const ( + // APIVersionHeader is what a client checks to confirm it is talking to a v2 + // registry rather than to something that merely answered the ping. + APIVersionHeader = "Docker-Distribution-API-Version" + APIVersion = "registry/2.0" + + ContentDigestHeader = "Docker-Content-Digest" +) diff --git a/internal/docker/layout.go b/internal/docker/layout.go new file mode 100644 index 0000000..250bcdc --- /dev/null +++ b/internal/docker/layout.go @@ -0,0 +1,56 @@ +package docker + +import ( + "strings" + + "arca/internal/format" +) + +type Layout struct{} + +// Coordinates names a tag. Only a tag manifest carries coordinates: a blob is +// shared by every image that references it and a digest-addressed manifest may +// have no tag at all, so neither belongs to one version. +func (Layout) Coordinates(path string) (format.Coordinates, bool) { + image, tag, ok := ParseTagPath(path) + if !ok { + return format.Coordinates{}, false + } + + namespace, name, ok := SplitImage(image) + if !ok { + return format.Coordinates{}, false + } + + return format.Coordinates{ + Namespace: namespace, + Name: name, + Version: tag, + BaseVersion: Core(tag), + IsSnapshot: Prerelease(tag), + }, true +} + +func (Layout) ComponentAt(prefix string) (format.Coordinates, bool) { + namespace, name, ok := SplitImage(strings.Trim(prefix, "/")) + // ManifestFilename is itself a legal name component, so a prefix ending in it + // is a stored file rather than an image directory. + if !ok || name == ManifestFilename { + return format.Coordinates{}, false + } + return format.Coordinates{Namespace: namespace, Name: name}, true +} + +func (Layout) ContentType(filename string) string { return ContentTypeFor(filename) } + +// Mutable is the mirror image of what the other formats report. Everything +// content-addressed is immutable by construction, and a tag is the one thing a +// registry lets move, so a proxy rechecks tags and caches blobs forever. +func (Layout) Mutable(path string) bool { + _, _, ok := ParseTagPath(path) + return ok +} + +func (Layout) Compare(a, b string) int { return Compare(a, b) } + +func (Layout) Prerelease(version string) bool { return Prerelease(version) } diff --git a/internal/docker/layout_test.go b/internal/docker/layout_test.go new file mode 100644 index 0000000..414721a --- /dev/null +++ b/internal/docker/layout_test.go @@ -0,0 +1,161 @@ +package docker + +import ( + "strings" + "testing" + + "arca/internal/format" +) + +func TestLayoutCoordinates(t *testing.T) { + digest := SHA256(strings.Repeat("ab", 32)) + + cases := []struct { + name string + path string + want format.Coordinates + ok bool + }{ + { + name: "namespaced tag", + path: "erc/research-grant/0.1.8-unittest-staging/manifest.json", + want: format.Coordinates{ + Namespace: "erc", + Name: "research-grant", + Version: "0.1.8-unittest-staging", + BaseVersion: "0.1.8", + }, + ok: true, + }, + { + name: "unscoped tag", + path: "nginx/latest/manifest.json", + want: format.Coordinates{Namespace: "", Name: "nginx", Version: "latest", BaseVersion: "latest"}, + ok: true, + }, + { + name: "prerelease tag", + path: "nginx/1.26-rc1/manifest.json", + want: format.Coordinates{ + Namespace: "", + Name: "nginx", + Version: "1.26-rc1", + BaseVersion: "1.26", + IsSnapshot: true, + }, + ok: true, + }, + { + name: "deep namespace", + path: "a/b/c/1.0/manifest.json", + want: format.Coordinates{Namespace: "a/b", Name: "c", Version: "1.0", BaseVersion: "1.0"}, + ok: true, + }, + + // Nothing content-addressed belongs to a version: a blob is shared by + // every image referencing it and a digest manifest may have no tag. + {name: "blob", path: BlobPath(digest)}, + {name: "digest manifest", path: ManifestPath("nginx", digest)}, + {name: "bare filename", path: "manifest.json"}, + {name: "no tag directory", path: "nginx/manifest.json"}, + {name: "wrong filename", path: "nginx/latest/config.json"}, + } + + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + coordinates, ok := Layout{}.Coordinates(testCase.path) + if ok != testCase.ok { + t.Fatalf("Coordinates(%q) ok = %v, want %v", testCase.path, ok, testCase.ok) + } + if !ok { + return + } + if coordinates != testCase.want { + t.Fatalf("Coordinates(%q) = %+v, want %+v", testCase.path, coordinates, testCase.want) + } + }) + } +} + +func TestLayoutComponentAt(t *testing.T) { + cases := []struct { + prefix string + namespace string + name string + ok bool + }{ + {"nginx/", "", "nginx", true}, + {"erc/research-grant/", "erc", "research-grant", true}, + {"erc/", "", "erc", true}, + + {"", "", "", false}, + {"_blobs/", "", "", false}, + {"nginx/latest/manifest.json", "", "", false}, + } + + for _, testCase := range cases { + t.Run(testCase.prefix, func(t *testing.T) { + coordinates, ok := Layout{}.ComponentAt(testCase.prefix) + if ok != testCase.ok { + t.Fatalf("ComponentAt(%q) ok = %v, want %v", testCase.prefix, ok, testCase.ok) + } + if !ok { + return + } + if coordinates.Namespace != testCase.namespace || coordinates.Name != testCase.name { + t.Fatalf("ComponentAt(%q) = (%q, %q), want (%q, %q)", + testCase.prefix, coordinates.Namespace, coordinates.Name, testCase.namespace, testCase.name) + } + }) + } +} + +// Mutable is the mirror image of what the other formats report, so it is worth +// pinning: a proxy has to recheck tags and may cache content forever. +func TestLayoutMutable(t *testing.T) { + layout := Layout{} + digest := SHA256(strings.Repeat("ab", 32)) + + cases := []struct { + path string + want bool + }{ + {"nginx/latest/manifest.json", true}, + {"erc/research-grant/0.1.8/manifest.json", true}, + + {BlobPath(digest), false}, + {ManifestPath("nginx", digest), false}, + {"", false}, + } + + for _, testCase := range cases { + t.Run(testCase.path, func(t *testing.T) { + if got := layout.Mutable(testCase.path); got != testCase.want { + t.Fatalf("Mutable(%q) = %v, want %v", testCase.path, got, testCase.want) + } + }) + } +} + +func TestLayoutContentType(t *testing.T) { + layout := Layout{} + + cases := []struct { + filename string + want string + }{ + {ManifestFilename, MediaTypeOCIManifest}, + {strings.Repeat("ab", 32), "application/octet-stream"}, + } + + for _, testCase := range cases { + t.Run(testCase.filename, func(t *testing.T) { + if got := layout.ContentType(testCase.filename); got != testCase.want { + t.Fatalf("ContentType(%q) = %q, want %q", testCase.filename, got, testCase.want) + } + }) + } +} + +// Layout has to satisfy the seam every other format is served through. +var _ format.Layout = Layout{} diff --git a/internal/docker/manifest.go b/internal/docker/manifest.go new file mode 100644 index 0000000..388eb37 --- /dev/null +++ b/internal/docker/manifest.go @@ -0,0 +1,244 @@ +package docker + +import ( + "encoding/json" + "errors" + "fmt" + "strings" +) + +const ( + MediaTypeOCIManifest = "application/vnd.oci.image.manifest.v1+json" + MediaTypeOCIIndex = "application/vnd.oci.image.index.v1+json" + MediaTypeOCIConfig = "application/vnd.oci.image.config.v1+json" + + MediaTypeDockerManifest = "application/vnd.docker.distribution.manifest.v2+json" + MediaTypeDockerList = "application/vnd.docker.distribution.manifest.list.v2+json" + MediaTypeDockerManifestV1 = "application/vnd.docker.distribution.manifest.v1+json" + MediaTypeDockerConfig = "application/vnd.docker.container.image.v1+json" +) + +const ( + ReferenceConfig = "config" + ReferenceLayer = "layer" + ReferenceManifest = "manifest" +) + +var ( + ErrSchemaVersion1 = errors.New("docker: schema 1 manifests are not supported") + ErrUnknownShape = errors.New("docker: the document is neither an image manifest nor an index") +) + +type Platform struct { + Architecture string `json:"architecture"` + OS string `json:"os"` + Variant string `json:"variant,omitempty"` + OSVersion string `json:"os.version,omitempty"` +} + +func (p Platform) String() string { + if p.OS == "" && p.Architecture == "" { + return "" + } + + label := p.OS + "/" + p.Architecture + if p.Variant != "" { + label += "/" + p.Variant + } + return label +} + +type Descriptor struct { + MediaType string `json:"mediaType"` + Digest string `json:"digest"` + Size int64 `json:"size"` + // URLs names where a nondistributable layer can be fetched from. It is the + // only place a manifest points outside the registry holding it. + URLs []string `json:"urls,omitempty"` + Platform *Platform `json:"platform,omitempty"` + ArtifactType string `json:"artifactType,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` +} + +// Manifest covers both shapes the registry API serves under one endpoint: an +// image manifest carrying a config and layers, and an index carrying child +// manifests per platform. +type Manifest struct { + SchemaVersion int `json:"schemaVersion"` + MediaType string `json:"mediaType"` + ArtifactType string `json:"artifactType,omitempty"` + Config Descriptor `json:"config"` + Layers []Descriptor `json:"layers"` + Manifests []Descriptor `json:"manifests"` + Subject *Descriptor `json:"subject,omitempty"` + Annotations map[string]string `json:"annotations,omitempty"` +} + +func (m Manifest) IsIndex() bool { return len(m.Manifests) > 0 } + +// Reference is one edge from a manifest to something it needs, flattened so the +// indexer can write the whole set without caring which shape it came from. +type Reference struct { + Digest string + Kind string + MediaType string + Size int64 + Position int64 + Platform string + // URLs is set only for a nondistributable layer, which is the one kind of + // reference whose bytes never live here. + URLs []string + // Annotations are the descriptor's own, not the child document's. buildkit marks + // an attestation child here, which is the only way to tell one from a platform. + Annotations map[string]string +} + +const ( + // AnnotationReferenceType is what buildkit sets on the attestation child of a + // multi-platform index. Such a child declares a platform of unknown/unknown, so + // without this it would read as a broken architecture rather than as provenance. + AnnotationReferenceType = "vnd.docker.reference.type" +) + +func (m Manifest) References() []Reference { + references := make([]Reference, 0, len(m.Layers)+len(m.Manifests)+1) + + if m.Config.Digest != "" { + references = append(references, Reference{ + Digest: m.Config.Digest, + Kind: ReferenceConfig, + MediaType: m.Config.MediaType, + Size: m.Config.Size, + }) + } + for index, layer := range m.Layers { + references = append(references, Reference{ + Digest: layer.Digest, + Kind: ReferenceLayer, + MediaType: layer.MediaType, + Size: layer.Size, + Position: int64(index), + URLs: layer.URLs, + }) + } + for index, child := range m.Manifests { + reference := Reference{ + Digest: child.Digest, + Kind: ReferenceManifest, + MediaType: child.MediaType, + Size: child.Size, + Position: int64(index), + Annotations: child.Annotations, + } + if child.Platform != nil { + reference.Platform = child.Platform.String() + } + references = append(references, reference) + } + + return references +} + +// DeclaredSize is what the document itself accounts for: the config plus the +// layers of an image, or the child manifest documents of an index. An index's +// real total needs its children resolved, which only the indexer can do. +func (m Manifest) DeclaredSize() int64 { + total := m.Config.Size + for _, layer := range m.Layers { + total += layer.Size + } + for _, child := range m.Manifests { + total += child.Size + } + return total +} + +// ParseManifest reads a manifest and settles its media type. The field is +// mandatory for Docker schema 2 but was optional in OCI before 1.1, so a +// document without one is classified by its shape instead. +func ParseManifest(document []byte) (Manifest, error) { + var manifest Manifest + if err := json.Unmarshal(document, &manifest); err != nil { + return Manifest{}, fmt.Errorf("docker: reading a manifest: %w", err) + } + + // Schema 1 describes its layers under entirely different keys, so it parses + // into an empty manifest rather than failing outright. + if manifest.SchemaVersion == 1 { + return Manifest{}, ErrSchemaVersion1 + } + + isIndex := len(manifest.Manifests) > 0 + isImage := manifest.Config.Digest != "" + if isIndex == isImage { + return Manifest{}, ErrUnknownShape + } + + if manifest.MediaType == "" { + if isIndex { + manifest.MediaType = MediaTypeOCIIndex + } else { + manifest.MediaType = MediaTypeOCIManifest + } + } + + for _, reference := range manifest.References() { + if !IsValidDigest(reference.Digest) { + return Manifest{}, fmt.Errorf("docker: %q is not a digest this server stores", reference.Digest) + } + } + if manifest.Subject != nil && !IsValidDigest(manifest.Subject.Digest) { + return Manifest{}, fmt.Errorf("docker: the subject digest %q is not one this server stores", manifest.Subject.Digest) + } + + return manifest, nil +} + +// Accepts reports whether a client's Accept header allows a stored manifest to be +// served as the type it was pushed with. A registry cannot convert between the +// Docker and OCI encodings, since either would change the digest the manifest is +// addressed by, so the only honest answers are the stored type or a 404. +// +// It errs towards serving. A header that names no manifest type at all, or one +// that wildcards, expresses no opinion worth refusing over, and only a client that +// listed manifest types and excluded this one is turned away. +func Accepts(accept, mediaType string) bool { + if strings.TrimSpace(accept) == "" { + return true + } + + opinionated := false + for _, entry := range strings.Split(accept, ",") { + candidate := strings.TrimSpace(strings.SplitN(entry, ";", 2)[0]) + + switch candidate { + case "", "*/*", "application/*": + return true + case mediaType: + return true + } + opinionated = opinionated || IsManifestMediaType(candidate) + } + return !opinionated +} + +// IsNondistributable reports whether a layer names content a client fetches from +// its vendor rather than from a registry, which Windows base images rely on. +// Nothing ever pushes one, so a manifest referencing one must not be rejected for +// the blob being absent. +func IsNondistributable(mediaType string) bool { + return strings.Contains(mediaType, "foreign") || strings.Contains(mediaType, "nondistributable") +} + +// IsManifestMediaType reports whether a content type names a manifest rather +// than a blob, which is what tells a client's Accept header and an upload's +// Content-Type apart from layer bytes. +func IsManifestMediaType(mediaType string) bool { + switch strings.TrimSpace(strings.SplitN(mediaType, ";", 2)[0]) { + case MediaTypeOCIManifest, MediaTypeOCIIndex, + MediaTypeDockerManifest, MediaTypeDockerList, MediaTypeDockerManifestV1: + return true + default: + return false + } +} diff --git a/internal/docker/manifest_test.go b/internal/docker/manifest_test.go new file mode 100644 index 0000000..d94e2b1 --- /dev/null +++ b/internal/docker/manifest_test.go @@ -0,0 +1,362 @@ +package docker + +import ( + "errors" + "strings" + "testing" +) + +func digestOf(seed string) string { + return "sha256:" + strings.Repeat(seed, 64/len(seed)) +} + +const imageManifest = `{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "config": { + "mediaType": "application/vnd.oci.image.config.v1+json", + "digest": "sha256:1111111111111111111111111111111111111111111111111111111111111111", + "size": 2048 + }, + "layers": [ + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": "sha256:2222222222222222222222222222222222222222222222222222222222222222", + "size": 100 + }, + { + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": "sha256:3333333333333333333333333333333333333333333333333333333333333333", + "size": 200 + } + ] +}` + +const indexManifest = `{ + "schemaVersion": 2, + "mediaType": "application/vnd.oci.image.index.v1+json", + "manifests": [ + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:4444444444444444444444444444444444444444444444444444444444444444", + "size": 500, + "platform": {"os": "linux", "architecture": "amd64"} + }, + { + "mediaType": "application/vnd.oci.image.manifest.v1+json", + "digest": "sha256:5555555555555555555555555555555555555555555555555555555555555555", + "size": 600, + "platform": {"os": "linux", "architecture": "arm64", "variant": "v8"} + } + ] +}` + +func TestParseImageManifest(t *testing.T) { + manifest, err := ParseManifest([]byte(imageManifest)) + if err != nil { + t.Fatalf("ParseManifest returned %v", err) + } + if manifest.IsIndex() { + t.Fatal("IsIndex() = true, want false") + } + if manifest.MediaType != MediaTypeOCIManifest { + t.Fatalf("MediaType = %q, want %q", manifest.MediaType, MediaTypeOCIManifest) + } + if got := manifest.DeclaredSize(); got != 2348 { + t.Fatalf("DeclaredSize() = %d, want 2348", got) + } + + references := manifest.References() + if len(references) != 3 { + t.Fatalf("References() returned %d entries, want 3", len(references)) + } + if references[0].Kind != ReferenceConfig { + t.Fatalf("first reference kind = %q, want %q", references[0].Kind, ReferenceConfig) + } + for index, reference := range references[1:] { + if reference.Kind != ReferenceLayer { + t.Fatalf("reference %d kind = %q, want %q", index+1, reference.Kind, ReferenceLayer) + } + if reference.Position != int64(index) { + t.Fatalf("layer %d position = %d, want %d", index, reference.Position, index) + } + } +} + +func TestParseIndexManifest(t *testing.T) { + manifest, err := ParseManifest([]byte(indexManifest)) + if err != nil { + t.Fatalf("ParseManifest returned %v", err) + } + if !manifest.IsIndex() { + t.Fatal("IsIndex() = false, want true") + } + if got := manifest.DeclaredSize(); got != 1100 { + t.Fatalf("DeclaredSize() = %d, want 1100", got) + } + + references := manifest.References() + if len(references) != 2 { + t.Fatalf("References() returned %d entries, want 2", len(references)) + } + + platforms := []string{"linux/amd64", "linux/arm64/v8"} + for index, reference := range references { + if reference.Kind != ReferenceManifest { + t.Fatalf("reference %d kind = %q, want %q", index, reference.Kind, ReferenceManifest) + } + if reference.Platform != platforms[index] { + t.Fatalf("reference %d platform = %q, want %q", index, reference.Platform, platforms[index]) + } + } +} + +// A media type is mandatory in Docker schema 2 but was optional in OCI before +// 1.1, so a document without one has to be classified by its shape. +func TestParseManifestInfersMediaType(t *testing.T) { + cases := []struct { + name string + document string + want string + }{ + { + name: "image", + document: `{"schemaVersion":2,"config":{"digest":"` + digestOf("1") + `","size":1},"layers":[]}`, + want: MediaTypeOCIManifest, + }, + { + name: "index", + document: `{"schemaVersion":2,"manifests":[{"digest":"` + digestOf("4") + `","size":1}]}`, + want: MediaTypeOCIIndex, + }, + } + + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + manifest, err := ParseManifest([]byte(testCase.document)) + if err != nil { + t.Fatalf("ParseManifest returned %v", err) + } + if manifest.MediaType != testCase.want { + t.Fatalf("MediaType = %q, want %q", manifest.MediaType, testCase.want) + } + }) + } +} + +func TestParseManifestRejections(t *testing.T) { + cases := []struct { + name string + document string + want error + }{ + { + name: "schema 1", + document: `{"schemaVersion":1,"name":"nginx","tag":"latest","fsLayers":[]}`, + want: ErrSchemaVersion1, + }, + { + name: "neither shape", + document: `{"schemaVersion":2}`, + want: ErrUnknownShape, + }, + { + name: "both shapes", + document: `{"schemaVersion":2,"config":{"digest":"` + digestOf("1") + `","size":1},"manifests":[{"digest":"` + digestOf("4") + `","size":1}]}`, + want: ErrUnknownShape, + }, + } + + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + if _, err := ParseManifest([]byte(testCase.document)); !errors.Is(err, testCase.want) { + t.Fatalf("ParseManifest returned %v, want %v", err, testCase.want) + } + }) + } +} + +// A descriptor digest that cannot be stored has to be caught at parse time, +// because every one of them becomes a storage path later. +func TestParseManifestRejectsBadDigests(t *testing.T) { + cases := []struct { + name string + document string + }{ + {"config", `{"schemaVersion":2,"config":{"digest":"sha256:short","size":1},"layers":[]}`}, + {"layer", `{"schemaVersion":2,"config":{"digest":"` + digestOf("1") + `","size":1},"layers":[{"digest":"nonsense","size":1}]}`}, + {"index child", `{"schemaVersion":2,"manifests":[{"digest":"md5:abc","size":1}]}`}, + {"subject", `{"schemaVersion":2,"config":{"digest":"` + digestOf("1") + `","size":1},"layers":[],"subject":{"digest":"bad","size":1}}`}, + {"traversal", `{"schemaVersion":2,"config":{"digest":"sha256:../../etc/passwd","size":1},"layers":[]}`}, + } + + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + if _, err := ParseManifest([]byte(testCase.document)); err == nil { + t.Fatal("ParseManifest accepted a digest it cannot store") + } + }) + } +} + +func TestParseManifestRejectsGarbage(t *testing.T) { + for _, document := range []string{"", "not json", "[]", "null"} { + t.Run(document, func(t *testing.T) { + if _, err := ParseManifest([]byte(document)); err == nil { + t.Fatal("ParseManifest accepted a document that is not a manifest") + } + }) + } +} + +func TestIsManifestMediaType(t *testing.T) { + cases := []struct { + mediaType string + want bool + }{ + {MediaTypeOCIManifest, true}, + {MediaTypeOCIIndex, true}, + {MediaTypeDockerManifest, true}, + {MediaTypeDockerList, true}, + {MediaTypeDockerManifestV1, true}, + {MediaTypeOCIManifest + "; charset=utf-8", true}, + + {MediaTypeOCIConfig, false}, + {"application/octet-stream", false}, + {"application/json", false}, + {"", false}, + } + + for _, testCase := range cases { + t.Run(testCase.mediaType, func(t *testing.T) { + if got := IsManifestMediaType(testCase.mediaType); got != testCase.want { + t.Fatalf("IsManifestMediaType(%q) = %v, want %v", testCase.mediaType, got, testCase.want) + } + }) + } +} + +func TestAccepts(t *testing.T) { + const dockerList = "application/vnd.docker.distribution.manifest.list.v2+json" + + cases := []struct { + name string + accept string + mediaType string + want bool + }{ + {name: "no header at all", accept: "", mediaType: MediaTypeOCIManifest, want: true}, + {name: "blank header", accept: " ", mediaType: MediaTypeOCIManifest, want: true}, + {name: "an exact match", accept: MediaTypeOCIManifest, mediaType: MediaTypeOCIManifest, want: true}, + {name: "a wildcard", accept: "*/*", mediaType: MediaTypeOCIManifest, want: true}, + {name: "a type wildcard", accept: "application/*", mediaType: MediaTypeOCIManifest, want: true}, + + { + name: "a modern client listing both encodings", + accept: MediaTypeDockerManifest + ", " + dockerList + ", " + MediaTypeOCIManifest + ", " + MediaTypeOCIIndex, + mediaType: MediaTypeOCIManifest, + want: true, + }, + { + name: "quality values are ignored", + accept: MediaTypeOCIManifest + ";q=0.9", + mediaType: MediaTypeOCIManifest, + want: true, + }, + + // A header that expressed no opinion about manifests is not grounds to + // refuse, which is what keeps an odd client working. + {name: "an unrelated type only", accept: "application/json", mediaType: MediaTypeOCIManifest, want: true}, + {name: "an unrelated type and a wildcard", accept: "text/html, */*", mediaType: MediaTypeOCIIndex, want: true}, + + // Only a client that listed manifest types and left this one out is refused. + { + name: "an old client that knows only the docker encoding", + accept: MediaTypeDockerManifest + ", " + dockerList, + mediaType: MediaTypeOCIManifest, + want: false, + }, + { + name: "an image manifest offered where an index is stored", + accept: MediaTypeOCIManifest, + mediaType: MediaTypeOCIIndex, + want: false, + }, + } + + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + if got := Accepts(testCase.accept, testCase.mediaType); got != testCase.want { + t.Fatalf("Accepts(%q, %q) = %v, want %v", testCase.accept, testCase.mediaType, got, testCase.want) + } + }) + } +} + +func TestIsNondistributable(t *testing.T) { + cases := []struct { + mediaType string + want bool + }{ + {"application/vnd.docker.image.rootfs.foreign.diff.tar.gzip", true}, + {"application/vnd.oci.image.layer.nondistributable.v1.tar+gzip", true}, + + {"application/vnd.oci.image.layer.v1.tar+gzip", false}, + {"application/vnd.docker.image.rootfs.diff.tar.gzip", false}, + {"", false}, + } + + for _, testCase := range cases { + t.Run(testCase.mediaType, func(t *testing.T) { + if got := IsNondistributable(testCase.mediaType); got != testCase.want { + t.Fatalf("IsNondistributable(%q) = %v, want %v", testCase.mediaType, got, testCase.want) + } + }) + } +} + +// A foreign layer's sources are the one thing in a manifest that points outside +// the registry, so they have to survive parsing to be recorded. +func TestParseManifestKeepsForeignLayerURLs(t *testing.T) { + document := `{ + "schemaVersion": 2, + "config": {"digest": "` + digestOf("1") + `", "size": 1}, + "layers": [{ + "mediaType": "application/vnd.oci.image.layer.nondistributable.v1.tar+gzip", + "digest": "` + digestOf("2") + `", + "size": 100, + "urls": ["https://vendor.example.com/layer.tar.gz"] + }] + }` + + manifest, err := ParseManifest([]byte(document)) + if err != nil { + t.Fatalf("ParseManifest returned %v", err) + } + + references := manifest.References() + layer := references[len(references)-1] + + if len(layer.URLs) != 1 || layer.URLs[0] != "https://vendor.example.com/layer.tar.gz" { + t.Fatalf("URLs = %v, want the vendor source", layer.URLs) + } +} + +func TestPlatformString(t *testing.T) { + cases := []struct { + platform Platform + want string + }{ + {Platform{OS: "linux", Architecture: "amd64"}, "linux/amd64"}, + {Platform{OS: "linux", Architecture: "arm64", Variant: "v8"}, "linux/arm64/v8"}, + {Platform{}, ""}, + } + + for _, testCase := range cases { + t.Run(testCase.want, func(t *testing.T) { + if got := testCase.platform.String(); got != testCase.want { + t.Fatalf("String() = %q, want %q", got, testCase.want) + } + }) + } +} diff --git a/internal/docker/path.go b/internal/docker/path.go new file mode 100644 index 0000000..50c900f --- /dev/null +++ b/internal/docker/path.go @@ -0,0 +1,116 @@ +package docker + +import "strings" + +const ( + // BlobsDirectory and ManifestsDirectory both begin with an underscore, which + // no image name component may, so neither can ever collide with a real + // image path. Blobs sit at the repository root because one blob is shared by + // every image that references it. + BlobsDirectory = "_blobs" + ManifestsDirectory = "_manifests" + ManifestFilename = "manifest.json" +) + +func BlobPath(digest Digest) string { + return BlobsDirectory + "/" + digest.Algorithm + "/" + digest.Hex +} + +func ManifestPath(image string, digest Digest) string { + return image + "/" + ManifestsDirectory + "/" + digest.Algorithm + "/" + digest.Hex +} + +func TagDirectory(image, tag string) string { return image + "/" + tag } + +// TagPath holds the manifest bytes a tag resolves to. Storing the document +// rather than a pointer to it means the asset row's own SHA256 is the manifest +// digest, so a tag pull needs one lookup and no indirection. +func TagPath(image, tag string) string { + return TagDirectory(image, tag) + "/" + ManifestFilename +} + +// ParseTagPath reads the image and tag back out of a stored tag manifest path. +// The name and tag grammars do the guarding: an image component cannot begin +// with an underscore and a tag cannot be ManifestsDirectory, so no path under +// the blob or digest stores can be mistaken for a tag. +func ParseTagPath(path string) (image string, tag string, ok bool) { + rest, found := strings.CutSuffix(path, "/"+ManifestFilename) + if !found { + return "", "", false + } + + cut := strings.LastIndexByte(rest, '/') + if cut <= 0 { + return "", "", false + } + + image, tag = rest[:cut], rest[cut+1:] + if !IsValidName(image) || !IsValidTag(tag) { + return "", "", false + } + return image, tag, true +} + +// IsBlobPath and IsDigestManifestPath recognise the two content-addressed stores. +// The sweep needs them because neither has coordinates and so neither is reachable +// through ParseTagPath, which is how everything else is identified. +func IsBlobPath(path string) bool { + rest, found := strings.CutPrefix(path, BlobsDirectory+"/") + if !found { + return false + } + return isDigestSuffix(rest) +} + +func IsDigestManifestPath(path string) bool { + marker := "/" + ManifestsDirectory + "/" + + cut := strings.LastIndex(path, marker) + if cut <= 0 { + return false + } + if !IsValidName(path[:cut]) { + return false + } + return isDigestSuffix(path[cut+len(marker):]) +} + +// ImageOfManifestPath reads the image out of a digest-addressed manifest path. The +// document itself never names the image it belongs to, so the path is the only source. +func ImageOfManifestPath(path string) string { + cut := strings.LastIndex(path, "/"+ManifestsDirectory+"/") + if cut <= 0 { + return "" + } + return path[:cut] +} + +// isDigestSuffix matches the "/" tail both stores end with. +func isDigestSuffix(rest string) bool { + algorithm, encoded, found := strings.Cut(rest, "/") + if !found { + return false + } + return IsValidDigest(algorithm + ":" + encoded) +} + +// SplitImage maps an image name onto arca coordinates: the leading path is the +// namespace and the last component is the name, so "erc/research-grant" reads +// the way "@scope/package" does for npm and "group:artifact" does for Maven. A +// single-component image has no namespace. +func SplitImage(image string) (namespace string, name string, ok bool) { + if !IsValidName(image) { + return "", "", false + } + if cut := strings.LastIndexByte(image, '/'); cut >= 0 { + return image[:cut], image[cut+1:], true + } + return "", image, true +} + +func ImageName(namespace, name string) string { + if namespace == "" { + return name + } + return namespace + "/" + name +} diff --git a/internal/docker/path_test.go b/internal/docker/path_test.go new file mode 100644 index 0000000..467a219 --- /dev/null +++ b/internal/docker/path_test.go @@ -0,0 +1,191 @@ +package docker + +import ( + "strings" + "testing" + + "arca/internal/format" +) + +func TestStoragePaths(t *testing.T) { + digest := SHA256(strings.Repeat("ab", 32)) + + cases := []struct { + name string + got string + want string + }{ + {"blob", BlobPath(digest), "_blobs/sha256/" + digest.Hex}, + {"manifest", ManifestPath("erc/research-grant", digest), "erc/research-grant/_manifests/sha256/" + digest.Hex}, + {"tag", TagPath("erc/research-grant", "0.1.8"), "erc/research-grant/0.1.8/manifest.json"}, + {"tag directory", TagDirectory("nginx", "latest"), "nginx/latest"}, + {"unscoped tag", TagPath("nginx", "latest"), "nginx/latest/manifest.json"}, + } + + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + if testCase.got != testCase.want { + t.Fatalf("got %q, want %q", testCase.got, testCase.want) + } + }) + } +} + +// Every path this package writes has to survive the server's own path guard, or +// it would be stored under a key the blob store refuses to resolve. +func TestStoragePathsAreSafe(t *testing.T) { + digest := SHA256(strings.Repeat("ab", 32)) + + paths := []string{ + BlobPath(digest), + ManifestPath("erc/research-grant", digest), + TagPath("erc/research-grant", "0.1.8-unittest-staging"), + TagPath("nginx", "_internal"), + } + + for _, path := range paths { + t.Run(path, func(t *testing.T) { + if !format.IsSafe(path) { + t.Fatalf("format.IsSafe(%q) = false, want true", path) + } + }) + } +} + +func TestParseTagPath(t *testing.T) { + digest := SHA256(strings.Repeat("ab", 32)) + + cases := []struct { + path string + image string + tag string + ok bool + }{ + {"nginx/latest/manifest.json", "nginx", "latest", true}, + {"erc/research-grant/0.1.8-unittest-staging/manifest.json", "erc/research-grant", "0.1.8-unittest-staging", true}, + {"a/b/c/1.0/manifest.json", "a/b/c", "1.0", true}, + + {"", "", "", false}, + {"nginx/manifest.json", "", "", false}, + {"manifest.json", "", "", false}, + {"nginx/latest/other.json", "", "", false}, + {"nginx/latest/manifest.json/extra", "", "", false}, + + // The reserved stores must not read back as a tag. + {BlobPath(digest), "", "", false}, + {ManifestPath("nginx", digest), "", "", false}, + {"_blobs/sha256/manifest.json", "", "", false}, + {"nginx/_manifests/manifest.json", "", "", false}, + } + + for _, testCase := range cases { + t.Run(testCase.path, func(t *testing.T) { + image, tag, ok := ParseTagPath(testCase.path) + if ok != testCase.ok { + t.Fatalf("ParseTagPath(%q) ok = %v, want %v", testCase.path, ok, testCase.ok) + } + if !ok { + return + } + if image != testCase.image || tag != testCase.tag { + t.Fatalf("ParseTagPath(%q) = (%q, %q), want (%q, %q)", testCase.path, image, tag, testCase.image, testCase.tag) + } + }) + } +} + +// A round trip is what keeps Coordinates and the write path from drifting apart. +func TestTagPathRoundTrip(t *testing.T) { + cases := [][2]string{ + {"nginx", "latest"}, + {"erc/research-grant", "0.1.8-unittest-staging"}, + {"a/b/c", "1.25-alpine"}, + {"library/postgres", "16"}, + } + + for _, testCase := range cases { + image, tag := testCase[0], testCase[1] + t.Run(image+":"+tag, func(t *testing.T) { + parsedImage, parsedTag, ok := ParseTagPath(TagPath(image, tag)) + if !ok { + t.Fatalf("ParseTagPath(TagPath(%q, %q)) reported not ok", image, tag) + } + if parsedImage != image || parsedTag != tag { + t.Fatalf("round trip gave (%q, %q), want (%q, %q)", parsedImage, parsedTag, image, tag) + } + }) + } +} + +func TestSplitImage(t *testing.T) { + cases := []struct { + image string + namespace string + name string + ok bool + }{ + {"nginx", "", "nginx", true}, + {"erc/research-grant", "erc", "research-grant", true}, + {"a/b/c", "a/b", "c", true}, + + {"", "", "", false}, + {"Nginx", "", "", false}, + {"a//b", "", "", false}, + {"_blobs", "", "", false}, + } + + for _, testCase := range cases { + t.Run(testCase.image, func(t *testing.T) { + namespace, name, ok := SplitImage(testCase.image) + if ok != testCase.ok { + t.Fatalf("SplitImage(%q) ok = %v, want %v", testCase.image, ok, testCase.ok) + } + if !ok { + return + } + if namespace != testCase.namespace || name != testCase.name { + t.Fatalf("SplitImage(%q) = (%q, %q), want (%q, %q)", testCase.image, namespace, name, testCase.namespace, testCase.name) + } + if rebuilt := ImageName(namespace, name); rebuilt != testCase.image { + t.Fatalf("ImageName(%q, %q) = %q, want %q", namespace, name, rebuilt, testCase.image) + } + }) + } +} + +// The sweep identifies the content-addressed stores by path, because neither has +// coordinates and so neither is reachable through ParseTagPath. +func TestRecognisingTheContentStores(t *testing.T) { + digest := SHA256(strings.Repeat("ab", 32)) + + cases := []struct { + name string + path string + blob bool + manifest bool + }{ + {name: "a blob", path: BlobPath(digest), blob: true}, + {name: "a digest manifest", path: ManifestPath("nginx", digest), manifest: true}, + {name: "a namespaced digest manifest", path: ManifestPath("erc/research-grant", digest), manifest: true}, + + {name: "a tag manifest", path: TagPath("nginx", "latest")}, + {name: "nothing", path: ""}, + {name: "the blob directory itself", path: BlobsDirectory}, + {name: "a blob without an algorithm", path: BlobsDirectory + "/" + digest.Hex}, + {name: "a blob with a short digest", path: BlobsDirectory + "/sha256/abc"}, + {name: "a manifest store with no image", path: ManifestsDirectory + "/sha256/" + digest.Hex}, + {name: "a manifest with an invalid image", path: "NGINX/" + ManifestsDirectory + "/sha256/" + digest.Hex}, + {name: "an unsupported algorithm", path: BlobsDirectory + "/md5/" + strings.Repeat("ab", 16)}, + } + + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + if got := IsBlobPath(testCase.path); got != testCase.blob { + t.Fatalf("IsBlobPath(%q) = %v, want %v", testCase.path, got, testCase.blob) + } + if got := IsDigestManifestPath(testCase.path); got != testCase.manifest { + t.Fatalf("IsDigestManifestPath(%q) = %v, want %v", testCase.path, got, testCase.manifest) + } + }) + } +} diff --git a/internal/docker/reference.go b/internal/docker/reference.go new file mode 100644 index 0000000..c7b447d --- /dev/null +++ b/internal/docker/reference.go @@ -0,0 +1,171 @@ +// Package docker addresses OCI and Docker registry content: image names, tags, +// digests, the storage layout they map onto and the manifest documents that tie +// them together. Nothing here speaks HTTP. +package docker + +import "strings" + +const ( + MaxNameLength = 255 + MaxTagLength = 128 + + digestSHA256 = "sha256" + digestSHA512 = "sha512" +) + +func isLowerAlphanumeric(character byte) bool { + return (character >= 'a' && character <= 'z') || (character >= '0' && character <= '9') +} + +func isDigit(character byte) bool { return character >= '0' && character <= '9' } + +func isSeparator(character byte) bool { + return character == '.' || character == '_' || character == '-' +} + +func isHex(character byte) bool { + return isDigit(character) || (character >= 'a' && character <= 'f') +} + +// isValidSeparatorRun holds the odd corner of the distribution spec's name +// grammar: a run between two alphanumerics may be one dot, one or two +// underscores, or any number of dashes, and it may not mix the three. +func isValidSeparatorRun(run string) bool { + for index := 1; index < len(run); index++ { + if run[index] != run[0] { + return false + } + } + + switch run[0] { + case '.': + return len(run) == 1 + case '_': + return len(run) <= 2 + case '-': + return true + default: + return false + } +} + +func isValidPathComponent(component string) bool { + index := 0 + for { + start := index + for index < len(component) && isLowerAlphanumeric(component[index]) { + index++ + } + if index == start { + return false + } + if index == len(component) { + return true + } + + start = index + for index < len(component) && isSeparator(component[index]) { + index++ + } + // Neither an unseparated illegal byte nor a trailing separator can be + // followed by the alphanumeric run the grammar requires. + if index == start || index == len(component) || !isValidSeparatorRun(component[start:index]) { + return false + } + } +} + +// IsValidName reports whether an image name is one a registry client can +// address. Every component is lowercase, which is why pushing an image built +// with a capital letter in its name fails before it reaches any registry. +func IsValidName(name string) bool { + if name == "" || len(name) > MaxNameLength { + return false + } + for _, component := range strings.Split(name, "/") { + if !isValidPathComponent(component) { + return false + } + } + return true +} + +// IsValidTag also rejects ManifestsDirectory. A tag is stored as a path segment +// beside that directory, and unlike an image component a tag may begin with an +// underscore, so it is the one name that could collide with the digest store. +func IsValidTag(tag string) bool { + if tag == "" || len(tag) > MaxTagLength || tag == ManifestsDirectory { + return false + } + + first := tag[0] + if !isLowerAlphanumeric(first) && !(first >= 'A' && first <= 'Z') && first != '_' { + return false + } + + for index := 1; index < len(tag); index++ { + character := tag[index] + if isLowerAlphanumeric(character) || (character >= 'A' && character <= 'Z') || isSeparator(character) { + continue + } + return false + } + return true +} + +// Digest is a content address. Only the algorithms this server can store are +// accepted, so an exotic one is rejected on the way in rather than producing a +// storage path nothing will ever read back. +type Digest struct { + Algorithm string + Hex string +} + +func hexLengthFor(algorithm string) int { + switch algorithm { + case digestSHA256: + return 64 + case digestSHA512: + return 128 + default: + return 0 + } +} + +func ParseDigest(raw string) (Digest, bool) { + algorithm, encoded, found := strings.Cut(raw, ":") + if !found { + return Digest{}, false + } + + expected := hexLengthFor(algorithm) + if expected == 0 || len(encoded) != expected { + return Digest{}, false + } + for index := 0; index < len(encoded); index++ { + if !isHex(encoded[index]) { + return Digest{}, false + } + } + return Digest{Algorithm: algorithm, Hex: encoded}, true +} + +func IsValidDigest(raw string) bool { + _, ok := ParseDigest(raw) + return ok +} + +func (d Digest) String() string { return d.Algorithm + ":" + d.Hex } + +// Short is the abbreviation used in the UI and in log lines. Twelve characters +// is what the Docker CLI shows and it is long enough to stay unambiguous. +func (d Digest) Short() string { + if len(d.Hex) <= 12 { + return d.Hex + } + return d.Hex[:12] +} + +// SHA256 wraps a hex digest this server computed itself. Asset rows already +// store one per blob, and for a manifest that digest is its content address. +func SHA256(hex string) Digest { return Digest{Algorithm: digestSHA256, Hex: hex} } diff --git a/internal/docker/reference_test.go b/internal/docker/reference_test.go new file mode 100644 index 0000000..0404060 --- /dev/null +++ b/internal/docker/reference_test.go @@ -0,0 +1,156 @@ +package docker + +import ( + "strings" + "testing" +) + +func TestIsValidName(t *testing.T) { + cases := []struct { + name string + valid bool + }{ + {"nginx", true}, + {"erc/research-grant", true}, + {"a/b/c/d", true}, + {"my.image", true}, + {"my_image", true}, + {"my__image", true}, + {"my---image", true}, + {"a1/b2", true}, + + {"", false}, + {"Nginx", false}, + {"my___image", false}, + {"my..image", false}, + {"my._image", false}, + {"-nginx", false}, + {"nginx-", false}, + {".nginx", false}, + {"_nginx", false}, + {"nginx/", false}, + {"/nginx", false}, + {"a//b", false}, + {"my image", false}, + {"my:image", false}, + {"my@image", false}, + {strings.Repeat("a", MaxNameLength+1), false}, + } + + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + if got := IsValidName(testCase.name); got != testCase.valid { + t.Fatalf("IsValidName(%q) = %v, want %v", testCase.name, got, testCase.valid) + } + }) + } +} + +// The reserved directories must never validate as image names, because the +// storage layout relies on that alone to keep them apart from real images. +func TestReservedDirectoriesAreNotNames(t *testing.T) { + for _, reserved := range []string{BlobsDirectory, ManifestsDirectory} { + t.Run(reserved, func(t *testing.T) { + if IsValidName(reserved) { + t.Fatalf("IsValidName(%q) = true, want false", reserved) + } + }) + } +} + +func TestIsValidTag(t *testing.T) { + cases := []struct { + tag string + valid bool + }{ + {"latest", true}, + {"1.0.0", true}, + {"0.1.8-unittest-staging", true}, + {"1.25-alpine", true}, + {"RELEASE", true}, + {"_internal", true}, + {"v1.2.3_rc.1", true}, + + {"", false}, + {".hidden", false}, + {"-leading", false}, + {"with/slash", false}, + {"with:colon", false}, + {"with space", false}, + {strings.Repeat("a", MaxTagLength+1), false}, + + // A tag may begin with an underscore, so this is the one name that could + // collide with the digest store beside it. + {ManifestsDirectory, false}, + } + + for _, testCase := range cases { + t.Run(testCase.tag, func(t *testing.T) { + if got := IsValidTag(testCase.tag); got != testCase.valid { + t.Fatalf("IsValidTag(%q) = %v, want %v", testCase.tag, got, testCase.valid) + } + }) + } +} + +func TestParseDigest(t *testing.T) { + sha256Hex := strings.Repeat("ab", 32) + sha512Hex := strings.Repeat("cd", 64) + + cases := []struct { + raw string + algorithm string + hex string + ok bool + }{ + {"sha256:" + sha256Hex, "sha256", sha256Hex, true}, + {"sha512:" + sha512Hex, "sha512", sha512Hex, true}, + + {"", "", "", false}, + {sha256Hex, "", "", false}, + {"sha256:", "", "", false}, + {"sha256:" + sha256Hex[:63], "", "", false}, + {"sha256:" + sha256Hex + "a", "", "", false}, + {"sha256:" + strings.Repeat("AB", 32), "", "", false}, + {"sha256:" + strings.Repeat("zz", 32), "", "", false}, + {"md5:" + strings.Repeat("ab", 16), "", "", false}, + {"sha256:" + strings.Repeat("ab", 31) + "../", "", "", false}, + } + + for _, testCase := range cases { + t.Run(testCase.raw, func(t *testing.T) { + digest, ok := ParseDigest(testCase.raw) + if ok != testCase.ok { + t.Fatalf("ParseDigest(%q) ok = %v, want %v", testCase.raw, ok, testCase.ok) + } + if !ok { + return + } + if digest.Algorithm != testCase.algorithm || digest.Hex != testCase.hex { + t.Fatalf("ParseDigest(%q) = %+v, want %s:%s", testCase.raw, digest, testCase.algorithm, testCase.hex) + } + if digest.String() != testCase.raw { + t.Fatalf("String() = %q, want %q", digest.String(), testCase.raw) + } + }) + } +} + +func TestDigestShort(t *testing.T) { + cases := []struct { + hex string + want string + }{ + {strings.Repeat("a", 64), strings.Repeat("a", 12)}, + {"abc", "abc"}, + {strings.Repeat("b", 12), strings.Repeat("b", 12)}, + } + + for _, testCase := range cases { + t.Run(testCase.hex, func(t *testing.T) { + if got := SHA256(testCase.hex).Short(); got != testCase.want { + t.Fatalf("Short() = %q, want %q", got, testCase.want) + } + }) + } +} diff --git a/internal/docker/route.go b/internal/docker/route.go new file mode 100644 index 0000000..959bfc1 --- /dev/null +++ b/internal/docker/route.go @@ -0,0 +1,146 @@ +package docker + +import ( + "net/http" + "strings" +) + +type RouteKind string + +const ( + RouteUnknown RouteKind = "unknown" + RouteBase RouteKind = "base" + RouteCatalog RouteKind = "catalog" + RouteTags RouteKind = "tags" + RouteManifest RouteKind = "manifest" + RouteBlob RouteKind = "blob" + RouteUploadStart RouteKind = "upload-start" + RouteUpload RouteKind = "upload" + RouteReferrers RouteKind = "referrers" +) + +const ( + segmentBlobs = "blobs" + segmentManifests = "manifests" + segmentUploads = "uploads" + segmentTags = "tags" + segmentReferrers = "referrers" + segmentCatalog = "_catalog" + segmentList = "list" +) + +type Route struct { + Kind RouteKind + // Name is the full image name from the URL, repository prefix included. The + // server splits the prefix off, because which segment names a repository is + // its business rather than the protocol's. + Name string + // Reference is the tag or digest a manifest request addresses, verbatim. + Reference string + Tag string + Digest Digest + ByDigest bool + Upload string +} + +// Resolve maps a registry request onto the endpoint it addresses. Names may +// contain slashes and a name component may legally be "manifests" or "blobs", so +// every shape is matched from the end of the path rather than the start. +func Resolve(method, path string) Route { + trimmed := strings.TrimPrefix(path, "/") + + switch trimmed { + case "": + return Route{Kind: RouteBase} + case segmentCatalog: + return Route{Kind: RouteCatalog} + } + + segments := strings.Split(trimmed, "/") + + if len(segments) >= 4 { + if route, ok := resolveUpload(segments); ok { + return route + } + } + if len(segments) >= 3 { + return resolveVerb(method, segments) + } + return Route{Kind: RouteUnknown} +} + +// resolveUpload handles the two upload shapes, which are three segments deep +// rather than two: "/blobs/uploads/" opens a session and +// "/blobs/uploads/" addresses one. +func resolveUpload(segments []string) (Route, bool) { + tail := segments[len(segments)-3:] + if tail[0] != segmentBlobs || tail[1] != segmentUploads { + return Route{}, false + } + + name := strings.Join(segments[:len(segments)-3], "/") + if !IsValidName(name) { + return Route{Kind: RouteUnknown}, true + } + + if tail[2] == "" { + return Route{Kind: RouteUploadStart, Name: name}, true + } + return Route{Kind: RouteUpload, Name: name, Upload: tail[2]}, true +} + +func resolveVerb(method string, segments []string) Route { + verb := segments[len(segments)-2] + target := segments[len(segments)-1] + + name := strings.Join(segments[:len(segments)-2], "/") + if !IsValidName(name) { + return Route{Kind: RouteUnknown} + } + route := Route{Name: name} + + switch verb { + case segmentManifests: + route.Kind, route.Reference = RouteManifest, target + if digest, ok := ParseDigest(target); ok { + route.Digest, route.ByDigest = digest, true + return route + } + if !IsValidTag(target) { + return Route{Kind: RouteUnknown} + } + route.Tag = target + return route + + case segmentBlobs: + // A POST that leaves the trailing slash off the uploads endpoint is + // common enough to accept, since nothing else can be meant by it. + if target == segmentUploads && method == http.MethodPost { + route.Kind = RouteUploadStart + return route + } + digest, ok := ParseDigest(target) + if !ok { + return Route{Kind: RouteUnknown} + } + route.Kind, route.Digest, route.ByDigest = RouteBlob, digest, true + return route + + case segmentTags: + if target != segmentList { + return Route{Kind: RouteUnknown} + } + route.Kind = RouteTags + return route + + case segmentReferrers: + digest, ok := ParseDigest(target) + if !ok { + return Route{Kind: RouteUnknown} + } + route.Kind, route.Digest, route.ByDigest = RouteReferrers, digest, true + return route + } + + return Route{Kind: RouteUnknown} +} diff --git a/internal/docker/route_test.go b/internal/docker/route_test.go new file mode 100644 index 0000000..fbd4398 --- /dev/null +++ b/internal/docker/route_test.go @@ -0,0 +1,162 @@ +package docker + +import ( + "net/http" + "strings" + "testing" +) + +func TestResolve(t *testing.T) { + digest := "sha256:" + strings.Repeat("ab", 32) + + cases := []struct { + name string + method string + path string + kind RouteKind + imageName string + tag string + upload string + }{ + {name: "version check", method: http.MethodGet, path: "/", kind: RouteBase}, + {name: "version check without a slash", method: http.MethodGet, path: "", kind: RouteBase}, + {name: "catalog", method: http.MethodGet, path: "/_catalog", kind: RouteCatalog}, + + { + name: "manifest by tag", method: http.MethodGet, path: "/images/team/api/manifests/1.4.0", + kind: RouteManifest, imageName: "images/team/api", tag: "1.4.0", + }, + { + name: "manifest by digest", method: http.MethodGet, path: "/nginx/manifests/" + digest, + kind: RouteManifest, imageName: "nginx", + }, + { + name: "blob", method: http.MethodGet, path: "/nginx/blobs/" + digest, + kind: RouteBlob, imageName: "nginx", + }, + { + name: "tag list", method: http.MethodGet, path: "/images/team/api/tags/list", + kind: RouteTags, imageName: "images/team/api", + }, + { + name: "referrers", method: http.MethodGet, path: "/nginx/referrers/" + digest, + kind: RouteReferrers, imageName: "nginx", + }, + { + name: "start an upload", method: http.MethodPost, path: "/images/app/blobs/uploads/", + kind: RouteUploadStart, imageName: "images/app", + }, + { + name: "start an upload without the trailing slash", method: http.MethodPost, path: "/images/app/blobs/uploads", + kind: RouteUploadStart, imageName: "images/app", + }, + { + name: "an upload session", method: http.MethodPatch, path: "/images/app/blobs/uploads/abc123", + kind: RouteUpload, imageName: "images/app", upload: "abc123", + }, + + {name: "an unknown verb", method: http.MethodGet, path: "/nginx/layers/1", kind: RouteUnknown}, + {name: "a blob addressed by tag", method: http.MethodGet, path: "/nginx/blobs/latest", kind: RouteUnknown}, + {name: "a bad tag", method: http.MethodGet, path: "/nginx/manifests/.hidden", kind: RouteUnknown}, + {name: "an invalid name", method: http.MethodGet, path: "/NGINX/manifests/1.0", kind: RouteUnknown}, + {name: "a nameless manifest", method: http.MethodGet, path: "/manifests/1.0", kind: RouteUnknown}, + {name: "an unlisted tag endpoint", method: http.MethodGet, path: "/nginx/tags/all", kind: RouteUnknown}, + {name: "a single segment", method: http.MethodGet, path: "/nginx", kind: RouteUnknown}, + } + + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + route := Resolve(testCase.method, testCase.path) + + if route.Kind != testCase.kind { + t.Fatalf("Resolve(%q, %q).Kind = %q, want %q", testCase.method, testCase.path, route.Kind, testCase.kind) + } + if testCase.kind == RouteUnknown { + return + } + if route.Name != testCase.imageName { + t.Fatalf("Name = %q, want %q", route.Name, testCase.imageName) + } + if route.Tag != testCase.tag { + t.Fatalf("Tag = %q, want %q", route.Tag, testCase.tag) + } + if route.Upload != testCase.upload { + t.Fatalf("Upload = %q, want %q", route.Upload, testCase.upload) + } + }) + } +} + +// A name component may legally be "manifests" or "blobs", so a resolver that +// searched forwards for the verb would cut the name in the wrong place. +func TestResolveMatchesFromTheEnd(t *testing.T) { + digest := "sha256:" + strings.Repeat("cd", 32) + + cases := []struct { + name string + path string + kind RouteKind + imageName string + reference string + }{ + { + name: "a name containing manifests", path: "/repo/manifests/app/manifests/1.0", + kind: RouteManifest, imageName: "repo/manifests/app", reference: "1.0", + }, + { + name: "a name containing blobs", path: "/repo/blobs/app/manifests/1.0", + kind: RouteManifest, imageName: "repo/blobs/app", reference: "1.0", + }, + { + name: "a name containing tags", path: "/repo/tags/app/blobs/" + digest, + kind: RouteBlob, imageName: "repo/tags/app", reference: digest, + }, + { + name: "a name ending in uploads", path: "/repo/uploads/blobs/uploads/session1", + kind: RouteUpload, imageName: "repo/uploads", + }, + } + + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + route := Resolve(http.MethodGet, testCase.path) + + if route.Kind != testCase.kind { + t.Fatalf("Kind = %q, want %q", route.Kind, testCase.kind) + } + if route.Name != testCase.imageName { + t.Fatalf("Name = %q, want %q", route.Name, testCase.imageName) + } + if testCase.reference != "" && route.Reference != testCase.reference && route.Digest.String() != testCase.reference { + t.Fatalf("Reference = %q, want %q", route.Reference, testCase.reference) + } + }) + } +} + +func TestResolveDigestReferences(t *testing.T) { + digest := "sha256:" + strings.Repeat("ef", 32) + + t.Run("a manifest addressed by digest reports ByDigest", func(t *testing.T) { + route := Resolve(http.MethodGet, "/nginx/manifests/"+digest) + if !route.ByDigest { + t.Fatal("ByDigest = false, want true") + } + if route.Digest.String() != digest { + t.Fatalf("Digest = %q, want %q", route.Digest.String(), digest) + } + if route.Tag != "" { + t.Fatalf("Tag = %q, want empty", route.Tag) + } + }) + + t.Run("a manifest addressed by tag does not", func(t *testing.T) { + route := Resolve(http.MethodGet, "/nginx/manifests/1.25") + if route.ByDigest { + t.Fatal("ByDigest = true, want false") + } + if route.Tag != "1.25" { + t.Fatalf("Tag = %q, want 1.25", route.Tag) + } + }) +} diff --git a/internal/docker/version.go b/internal/docker/version.go new file mode 100644 index 0000000..3e49463 --- /dev/null +++ b/internal/docker/version.go @@ -0,0 +1,178 @@ +package docker + +import ( + "strconv" + "strings" +) + +// prereleaseMarkers are matched against whole words in a tag rather than as +// substrings, because a substring test reads "rc" out of "source" and "1.0-dev" +// out of nothing at all. Variant suffixes like "-alpine" or "-slim" name a base +// image and are deliberately absent: they are not prereleases of anything. +var prereleaseMarkers = map[string]bool{ + "alpha": true, + "beta": true, + "rc": true, + "dev": true, + "pre": true, + "snapshot": true, + "nightly": true, + "edge": true, +} + +// tag is a Docker tag read as loosely as one can be. Tags are arbitrary strings, +// so the only structure worth trusting is a leading dotted number, and whether +// there is one at all. +type tag struct { + raw string + versioned bool + core []int64 + suffix string +} + +func parseTag(raw string) tag { + lowered := strings.ToLower(strings.TrimSpace(raw)) + parsed := tag{raw: lowered} + + if lowered == "" || !isDigit(lowered[0]) { + return parsed + } + parsed.versioned = true + + end := 0 + for { + start := end + for end < len(lowered) && isDigit(lowered[end]) { + end++ + } + if end == start { + break + } + + number, err := strconv.ParseInt(lowered[start:end], 10, 64) + if err != nil { + number = 0 + } + parsed.core = append(parsed.core, number) + + // Only a dot with a digit behind it continues the number. A trailing dot + // belongs to the suffix, where it cannot be mistaken for a segment. + if end+1 < len(lowered) && lowered[end] == '.' && isDigit(lowered[end+1]) { + end++ + continue + } + break + } + + parsed.suffix = lowered[end:] + return parsed +} + +// words splits a suffix into the leading alphabetic run of each token, which is +// what a marker is matched against. +func words(text string) []string { + var found []string + + index := 0 + for index < len(text) { + for index < len(text) && !isLowerAlphanumeric(text[index]) { + index++ + } + start := index + for index < len(text) && isLowerAlphanumeric(text[index]) { + index++ + } + if start == index { + continue + } + + token := text[start:index] + end := 0 + for end < len(token) && !isDigit(token[end]) { + end++ + } + if end > 0 { + found = append(found, token[:end]) + } + } + return found +} + +// Compare orders tags. A tag that opens with a number is treated as a version +// and outranks every tag that does not, which puts "latest" and "alpine" +// underneath "1.25" instead of above it on a bytewise sort. +func Compare(a, b string) int { + left, right := parseTag(a), parseTag(b) + + if left.versioned != right.versioned { + if left.versioned { + return 1 + } + return -1 + } + if !left.versioned { + return strings.Compare(left.raw, right.raw) + } + + for index := 0; index < len(left.core) || index < len(right.core); index++ { + var leftPart, rightPart int64 + if index < len(left.core) { + leftPart = left.core[index] + } + if index < len(right.core) { + rightPart = right.core[index] + } + if leftPart != rightPart { + if leftPart < rightPart { + return -1 + } + return 1 + } + } + + // A bare version outranks the same version carrying anything else, whether + // that is a prerelease marker or a variant. + switch { + case left.suffix == "" && right.suffix == "": + return 0 + case left.suffix == "": + return 1 + case right.suffix == "": + return -1 + } + return strings.Compare(left.suffix, right.suffix) +} + +// Prerelease reads a versioned tag's suffix and an unversioned tag whole, so +// "1.0.0-rc1" and a bare "nightly" both report true while "latest" and +// "1.25-alpine" do not. +func Prerelease(raw string) bool { + parsed := parseTag(raw) + + subject := parsed.suffix + if !parsed.versioned { + subject = parsed.raw + } + + for _, word := range words(subject) { + if prereleaseMarkers[word] { + return true + } + } + return false +} + +// Core is the dotted number a tag opens with, which is what groups the variants +// and prereleases of one release together. A tag without one is its own base. +func Core(raw string) string { + parsed := parseTag(raw) + if !parsed.versioned { + return parsed.raw + } + + parts := make([]string, 0, len(parsed.core)) + for _, part := range parsed.core { + parts = append(parts, strconv.FormatInt(part, 10)) + } + return strings.Join(parts, ".") +} diff --git a/internal/docker/version_test.go b/internal/docker/version_test.go new file mode 100644 index 0000000..f842f86 --- /dev/null +++ b/internal/docker/version_test.go @@ -0,0 +1,163 @@ +package docker + +import ( + "testing" + + "arca/internal/format" +) + +func TestCompareOrdering(t *testing.T) { + cases := []struct { + lower string + higher string + }{ + {"1.0.0", "1.0.1"}, + {"1.0.0", "1.1.0"}, + {"1.0.0", "2.0.0"}, + {"1.2", "1.10"}, + {"1", "1.1"}, + {"1.9", "1.10"}, + + // The real tags from the Nexus instance this format was designed against. + {"0.1.8-unittest-staging", "0.1.9-swaggerui-staging"}, + + // A bare version outranks the same version with anything appended, + // whether that is a prerelease or a variant. + {"1.0.0-rc1", "1.0.0"}, + {"1.25-alpine", "1.25"}, + + // A tag that does not open with a number is not a version, so it sorts + // under every tag that is. A bytewise sort would put "latest" on top. + {"latest", "0.0.1"}, + {"alpine", "1.0"}, + {"stable", "2"}, + + // Two names order against each other bytewise, having nothing better. + {"alpine", "latest"}, + } + + for _, testCase := range cases { + t.Run(testCase.lower+" < "+testCase.higher, func(t *testing.T) { + if got := Compare(testCase.lower, testCase.higher); got >= 0 { + t.Fatalf("Compare(%q, %q) = %d, want < 0", testCase.lower, testCase.higher, got) + } + if got := Compare(testCase.higher, testCase.lower); got <= 0 { + t.Fatalf("Compare(%q, %q) = %d, want > 0", testCase.higher, testCase.lower, got) + } + }) + } +} + +func TestCompareEquality(t *testing.T) { + cases := [][2]string{ + {"1.0.0", "1.0.0"}, + {"1.25-ALPINE", "1.25-alpine"}, + {"LATEST", "latest"}, + {" 1.0 ", "1.0"}, + } + + for _, testCase := range cases { + t.Run(testCase[0]+" == "+testCase[1], func(t *testing.T) { + if got := Compare(testCase[0], testCase[1]); got != 0 { + t.Fatalf("Compare(%q, %q) = %d, want 0", testCase[0], testCase[1], got) + } + }) + } +} + +func TestPrerelease(t *testing.T) { + cases := []struct { + tag string + want bool + }{ + {"1.0.0-rc1", true}, + {"1.0.0-rc.1", true}, + {"1.0.0-alpha", true}, + {"1.0.0-beta.2", true}, + {"1.0.0-SNAPSHOT", true}, + {"2.1-dev", true}, + {"1.0.0-pre", true}, + {"nightly", true}, + {"edge", true}, + + {"1.0.0", false}, + {"latest", false}, + {"stable", false}, + {"1", false}, + + // A variant names a base image rather than a prerelease of anything. + {"1.25-alpine", false}, + {"3.20-slim", false}, + {"1.0-bookworm", false}, + + // The tags from the live Nexus instance. No marker list classifies these, + // which is why a docker repository defaults to the mixed policy. + {"0.1.8-unittest-staging", false}, + {"0.1.9-swaggerui-staging", false}, + + // A substring test would read "rc" out of the first and "dev" out of the + // second. Markers are matched as whole words for exactly this reason. + {"1.0-source", false}, + {"1.0-bundev", false}, + {"1.0-searchable", false}, + } + + for _, testCase := range cases { + t.Run(testCase.tag, func(t *testing.T) { + if got := Prerelease(testCase.tag); got != testCase.want { + t.Fatalf("Prerelease(%q) = %v, want %v", testCase.tag, got, testCase.want) + } + }) + } +} + +func TestCore(t *testing.T) { + cases := []struct { + tag string + want string + }{ + {"1.0.0", "1.0.0"}, + {"1.25-alpine", "1.25"}, + {"0.1.8-unittest-staging", "0.1.8"}, + {"1.0.0-rc.1", "1.0.0"}, + {"16", "16"}, + {"latest", "latest"}, + {"alpine", "alpine"}, + {"1.", "1"}, + } + + for _, testCase := range cases { + t.Run(testCase.tag, func(t *testing.T) { + if got := Core(testCase.tag); got != testCase.want { + t.Fatalf("Core(%q) = %q, want %q", testCase.tag, got, testCase.want) + } + }) + } +} + +// The version list and the badges on the artifact page are driven through the +// format seam, so the comparator has to hold up under the mix of versions, +// variants and moving tags one real image carries all at once. +func TestSortingAMixedTagSet(t *testing.T) { + tags := []string{"latest", "1.25", "1.25-alpine", "1.26-rc1", "1.24", "alpine"} + + sorted := format.SortVersions(Layout{}, tags) + want := []string{"alpine", "latest", "1.24", "1.25-alpine", "1.25", "1.26-rc1"} + + if len(sorted) != len(want) { + t.Fatalf("sorted %d tags, want %d", len(sorted), len(want)) + } + for index := range want { + if sorted[index] != want[index] { + t.Fatalf("sorted = %v, want %v", sorted, want) + } + } + + latest, release := format.LatestAndRelease(Layout{}, tags) + if latest != "1.26-rc1" { + t.Fatalf("latest = %q, want %q", latest, "1.26-rc1") + } + if release != "1.25" { + t.Fatalf("release = %q, want %q", release, "1.25") + } +} diff --git a/internal/format/format.go b/internal/format/format.go index c3ff96d..efbca67 100644 --- a/internal/format/format.go +++ b/internal/format/format.go @@ -4,6 +4,7 @@ const ( Maven2 = "maven2" NPM = "npm" P2 = "p2" + Docker = "docker" ) type Coordinates struct { diff --git a/internal/frontend/src/components/app-shell.tsx b/internal/frontend/src/components/app-shell.tsx index 0d11c79..a2e5dc6 100644 --- a/internal/frontend/src/components/app-shell.tsx +++ b/internal/frontend/src/components/app-shell.tsx @@ -35,6 +35,7 @@ import { Button } from '@/components/ui/button' import { ThemeToggle } from '@/components/theme-toggle' import { useRefreshSession, useSession } from '@/lib/session' import { api, post, type Repository } from '@/lib/api' +import { FormatIcon } from '@/components/ui/format-icon' import { categoriesQuery, groupRepositories, needsHeadings } from '@/lib/categories' import { cn } from '@/lib/utils' @@ -156,6 +157,7 @@ function RailContent({ onNavigate = () => {} }: { onNavigate?: () => void }) { > {group.repositories.map((repository) => ( + {repository.name} {repository.visibility === 'public' ? ( @@ -184,12 +186,12 @@ function RailContent({ onNavigate = () => {} }: { onNavigate?: () => void }) { ) : null} -
-

Repository endpoint

-

- {window.location.origin}/repository/ -

-
+ {/*
*/} + {/*

Repository endpoint

*/} + {/*

*/} + {/* {window.location.origin}/repository/*/} + {/*

*/} + {/*
*/} @@ -212,15 +214,16 @@ function BuildStamp() { .join('\n') return ( -

Arca {build.version} {built && !Number.isNaN(built.valueOf()) ? ` · ${built.toISOString().slice(0, 10)}` : null} -

+ ) } diff --git a/internal/frontend/src/components/copy-button.tsx b/internal/frontend/src/components/copy-button.tsx index b52999b..aef694a 100644 --- a/internal/frontend/src/components/copy-button.tsx +++ b/internal/frontend/src/components/copy-button.tsx @@ -66,7 +66,7 @@ export function StackedCoordinate({ namespace: string name: string }) { - const scope = format === 'npm' ? (namespace ? `@${namespace}` : '') : namespace + const scope = format === 'npm' && namespace ? `@${namespace}` : namespace return ( <> @@ -91,13 +91,17 @@ export function Coordinate({ version?: string | null className?: string }) { - const npm = format === 'npm' + // npm and docker fold the namespace into the name with a separator of their own, + // while Maven and p2 keep it as a distinct field joined by a colon. + const inlineNamespace = format === 'npm' || format === 'docker' + const prefix = format === 'npm' ? `@${namespace}/` : `${namespace}/` + const versionSeparator = format === 'npm' ? '@' : ':' return ( - {npm ? ( + {inlineNamespace ? ( namespace ? ( - @{namespace}/ + {prefix} ) : null ) : ( <> @@ -108,7 +112,7 @@ export function Coordinate({ {name} {version ? ( <> - {npm ? '@' : ':'} + {versionSeparator} {version} ) : null} diff --git a/internal/frontend/src/components/docker-manifest.tsx b/internal/frontend/src/components/docker-manifest.tsx new file mode 100644 index 0000000..8117220 --- /dev/null +++ b/internal/frontend/src/components/docker-manifest.tsx @@ -0,0 +1,381 @@ +import { useState, type ReactNode } from 'react' +import { useQuery } from '@tanstack/react-query' +import { ChevronLeft, ChevronRight, Layers, Package } from 'lucide-react' +import { Badge } from '@/components/ui/badge' +import { Card } from '@/components/ui/card' +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table' +import { ErrorBlock, LoadingBlock } from '@/components/ui/feedback' +import { CopyButton } from '@/components/copy-button' +import { api, type DockerChild, type DockerLayer, type DockerManifest } from '@/lib/api' +import { formatBytes, formatRelative, plural } from '@/lib/utils' + +function Field({ label, children }: { label: string; children: ReactNode }) { + return ( +
+

{label}

+
{children}
+
+ ) +} + +function Summary({ manifest }: { manifest: DockerManifest }) { + const platform = [manifest.os, manifest.architecture, manifest.variant].filter(Boolean).join('/') + + return ( + +
+ {manifest.isIndex ? 'multi-platform index' : 'image'} + {platform ? {platform} : null} + {manifest.isIndex ? ( + + {plural(manifest.children.filter((child) => !child.referenceType).length, 'platform')} + + ) : ( + {plural(manifest.layerCount, 'layer')} + )} +
+ +
+ {formatBytes(manifest.totalSize)} + {formatBytes(manifest.size)} + + {manifest.imageCreated ? formatRelative(manifest.imageCreated) : 'unknown'} + + + {manifest.mediaType} + +
+ +
+ + {manifest.digest} + + +
+ + ) +} + +// attestationLabel names a child that is not a platform. Every buildkit +// multi-platform build attaches one, and it declares a platform of unknown/unknown, +// so labelling it is the difference between provenance and an apparent broken build. +function attestationLabel(child: DockerChild): string { + if (!child.referenceType) return child.platform || 'unknown' + if (child.referenceType === 'attestation-manifest') return 'attestation' + return child.referenceType +} + +// Platforms is the part a plain file listing can never show: an index holds no +// layers of its own, only one child manifest per platform. +function Platforms({ + platforms, + onSelect, +}: { + platforms: DockerChild[] + onSelect: (digest: string) => void +}) { + return ( +
+

Platforms

+ + + + + Platform + Size + Layers + Digest + + + + {platforms.map((child) => ( + (child.indexed ? onSelect(child.digest) : undefined)} + className={child.indexed ? 'cursor-pointer' : undefined} + > + + {child.indexed ? ( + + + {attestationLabel(child)} + + ) : ( + attestationLabel(child) + )} + {child.referenceType ? ( + + not a platform + + ) : null} + + + {child.indexed ? formatBytes(child.totalSize) : formatBytes(child.size)} + + + {child.indexed ? child.layerCount : '-'} + + + {child.short} + + + ))} + +
+ +

+ {platforms.some((child) => !child.indexed) + ? 'A platform without a layer count has not been indexed here yet, which is normal while an index is still being pushed.' + : 'Open a platform to see its layers and the configuration it was built with. An index holds neither of its own.'} +

+
+ ) +} + +// A foreign layer's source comes from a pushed manifest, so it is not necessarily a +// URL that parses. The raw value is better than a crash. +function sourceHost(urls: string[]): string { + const [first] = urls + if (!first) return 'elsewhere' + try { + return new URL(first).host + } catch { + return first + } +} + +function LayerNote({ layer }: { layer: DockerLayer }) { + if (layer.foreign) { + return ( + + fetched from {sourceHost(layer.urls)} + + ) + } + if (!layer.stored) { + return ( + + missing + + ) + } + if (layer.sharedWith > 0) { + return ( + + shared with {plural(layer.sharedWith, 'image')} + + ) + } + return null +} + +function LayerTable({ manifest }: { manifest: DockerManifest }) { + const shared = manifest.layers.filter((layer) => layer.sharedWith > 0) + const unique = manifest.layers + .filter((layer) => layer.sharedWith === 0 && !layer.foreign) + .reduce((total, layer) => total + layer.size, 0) + + return ( +
+
+

Layers

+ {shared.length > 0 ? ( +

+ {formatBytes(unique)} of {formatBytes(manifest.totalSize)} is unique to this + tag, so deleting it reclaims only that much +

+ ) : null} +
+ + + + + # + Digest + Size + Media type + + + + {manifest.layers.map((layer) => ( + + + {layer.position + 1} + + +
+ {layer.short} + +
+ + + {formatBytes(layer.size)} + + + {layer.mediaType.replace('application/vnd.', '')} + + + ))} + +
+ +
+ ) +} + +function List({ label, values }: { label: string; values: string[] }) { + if (values.length === 0) return null + return ( + +
+ {values.map((value) => ( +

+ {value} +

+ ))} +
+ + ) +} + +function Configuration({ manifest }: { manifest: DockerManifest }) { + const config = manifest.config + if (!config) { + return ( +

+ The config blob for this image is not stored here, so its build settings cannot + be read. That is normal in a repository that was migrated rather than pushed. +

+ ) + } + + const labels = Object.entries(manifest.labels) + const declared = + config.entrypoint.length + + config.cmd.length + + config.env.length + + config.exposedPorts.length + + labels.length + + (config.workingDir ? 1 : 0) + + (config.user ? 1 : 0) + + // An image built by "docker import", or any scratch image, declares none of this. + // Saying so beats rendering an empty card. + if (declared === 0) { + return ( +
+

Configuration

+

+ This image declares no entrypoint, command, environment or labels. +

+
+ ) + } + + return ( +
+

Configuration

+ + + + {config.workingDir ? {config.workingDir} : null} + {config.user ? {config.user} : null} + + + {labels.length > 0 ? ( +
+

Labels

+
+ {labels.map(([key, value]) => ( +

+ {key} + = + {value} +

+ ))} +
+
+ ) : null} + +
+ ) +} + +// History is longer than the layer list, because an instruction that only changes +// metadata produces an entry with no filesystem layer behind it. +function History({ manifest }: { manifest: DockerManifest }) { + const history = manifest.config?.history ?? [] + if (history.length === 0) return null + + return ( +
+

Build history

+ + {history.map((step, index) => ( +
+ + {step.emptyLayer ? : } + + + {step.createdBy || step.comment || '(no instruction recorded)'} + + {step.emptyLayer ? ( + no layer + ) : null} +
+ ))} + +
+ ) +} + +export function DockerManifestView({ + repository, + namespace, + name, + reference, +}: { + repository: string + namespace: string + name: string + reference: string +}) { + const [selected, setSelected] = useState(null) + const showing = selected ?? reference + + const query = new URLSearchParams({ namespace, name, reference: showing }) + const { data, error, isLoading } = useQuery({ + queryKey: ['docker-manifest', repository, namespace, name, showing], + queryFn: () => api(`/repositories/${repository}/docker/manifests?${query}`), + enabled: showing.length > 0, + }) + + if (isLoading) return + if (error) return + if (!data) return null + + return ( +
+ {selected ? ( + + ) : null} + + {data.isIndex ? ( + + ) : ( + <> + + + + + )} +
+ ) +} diff --git a/internal/frontend/src/components/snippets/docker.tsx b/internal/frontend/src/components/snippets/docker.tsx new file mode 100644 index 0000000..95aff53 --- /dev/null +++ b/internal/frontend/src/components/snippets/docker.tsx @@ -0,0 +1,113 @@ +import { CodeBlock } from '@/components/copy-button' +import type { RepositoryDetail } from '@/lib/api' + +// A Docker client builds its own URLs from the image reference, so the registry is +// the bare host: no scheme and no path. The repository name is the first segment of +// the image instead, which is how one host serves several repositories on one port. +function registryHost(): string { + return window.location.host +} + +function imageReference(repository: string, image: string): string { + return `${registryHost()}/${repository}/${image}` +} + +function LoginBlock() { + return ( + + ) +} + +export function DockerUsage({ repository }: { repository: RepositoryDetail }) { + const host = registryHost() + const isProxy = repository.type === 'proxy' + const isPublic = repository.visibility === 'public' + const example = imageReference(repository.name, 'your-image') + + return ( +
+
+

Registry

+

+ {isProxy + ? `This repository is read-only. Pull through it and every image is fetched from the remote once, then served from here. The repository name is the first segment of the image, so images resolve under ${host}/${repository.name}/.` + : `Images live under ${host}/${repository.name}/. The repository name is part of the image reference, which is how one host serves several repositories without a port or a hostname of its own.`} +

+
+ +
+ {isPublic && isProxy ? null : } + + {isProxy ? null : ( + <> + + + + )} +
+ + {isPublic ? ( +

+ Pulling needs no credentials. + {isProxy ? '' : ' Pushing needs an API token from your account page.'} +

+ ) : ( +

+ This repository is private, so both pulling and pushing need a docker login + with an API token from your account page. +

+ )} +
+ ) +} + +export function DockerDependency({ + repository, + image, + tag, + digest, +}: { + repository: string + image: string + tag: string + digest?: string +}) { + const reference = imageReference(repository, image) + + return ( +
+

Pull

+
+
+ + +
+
+ {digest ? ( + + ) : null} + +
+
+ {digest ? ( +

+ A tag can be moved to another image. Pulling by digest is the only reference + that always resolves to these exact bytes. +

+ ) : null} +
+ ) +} diff --git a/internal/frontend/src/components/ui/badge.tsx b/internal/frontend/src/components/ui/badge.tsx index 38b2f64..31a4e81 100644 --- a/internal/frontend/src/components/ui/badge.tsx +++ b/internal/frontend/src/components/ui/badge.tsx @@ -1,6 +1,7 @@ import type * as React from 'react' import { cva, type VariantProps } from 'class-variance-authority' import type { RepositoryFormat } from '@/lib/api' +import { FormatIcon } from '@/components/ui/format-icon' import { formatLabel } from '@/lib/coordinates' import { cn } from '@/lib/utils' @@ -34,7 +35,10 @@ export function TypeBadge({ type }: { type: string }) { export function FormatBadge({ format }: { format: RepositoryFormat }) { return ( - {formatLabel(format)} + + + {formatLabel(format)} + ) } diff --git a/internal/frontend/src/components/ui/format-icon.tsx b/internal/frontend/src/components/ui/format-icon.tsx new file mode 100644 index 0000000..c998cf1 --- /dev/null +++ b/internal/frontend/src/components/ui/format-icon.tsx @@ -0,0 +1,37 @@ +import type { RepositoryFormat } from '@/lib/api' +import { formatLabel } from '@/lib/coordinates' +import { cn } from '@/lib/utils' + +// Brand marks from Simple Icons (CC0), inlined rather than fetched so the bundle stays +// self-contained. They identify each format at a glance, which a generic box cannot: +// the whole point of the icon is that a Docker whale is recognised before it is read. +const paths: Record = { + maven2: + 'M4.237.001c-.312-.013-.665.072-.828.457-.158.374-.283 1.188-.34 2.276l1.223.591c-.02-.737.007-1.43.076-2.066-.026.299-.056.96.006 2.039.019.342.049.725.088 1.15.002.024.002.047.007.069a45.485 45.485 0 0 0 .309 2.412c.057.368.126.752.195 1.16l-.01.01c.014.01.015.018.014.023l.03.16c.03.162.06.328.093.494l.108.553.056.289a61.72 61.72 0 0 0 .457 2.068c.09.382.186.78.287 1.186.098.386.199.783.309 1.193.096.362.199.735.303 1.117.003.018.012.036.015.055a145.826 145.826 0 0 0 .34 1.185l.049.174c.078.261.158.533.242.805a4.2 4.2 0 0 1-.293-.135l-.19-.654c-.02-.077-.042-.148-.062-.225l-.002-.004-.004-.002c-.087-.3-.17-.607-.257-.916-.023-.087-.044-.173-.069-.263l-.314-1.178c-.1-.381-.194-.765-.29-1.154-.094-.39-.185-.78-.277-1.172-.093-.401-.181-.8-.265-1.203-.085-.396-.161-.798-.24-1.193a50.315 50.315 0 0 1-.211-1.17c-.004-.013-.006-.03-.01-.041l.004-.002c-.057-.386-.116-.77-.174-1.15a60.905 60.905 0 0 1-.154-1.204 27.447 27.447 0 0 1-.172-2.41l-1.22-.59c-.004.074-.01.15-.013.23-.012.294-.02.605-.023.93a45.3 45.3 0 0 0 .006 1.157c.009.37.025.755.045 1.148.02.336.042.675.07 1.022l.002.039.006.004c.003.023.007.05.006.076.033.368.064.739.107 1.115a34.493 34.493 0 0 0 .303 2.125c.01.064.024.131.035.195a23.418 23.418 0 0 0 .547 2.32c.07.237.14.464.21.68.063.182.13.365.194.545.155.422.327.832.512 1.232l.006.004a.318.318 0 0 0 .02.05c.225.485.475.95.755 1.395.01.013.02.033.03.047-.455-.183-1.259-.098-1.253-.097.83.288 1.557.64 2.016 1.175-.183.2-.523.352-.953.477.594.064.924-.039 1.045-.092-.31.26-.483.732-.635 1.24.35-.57.696-.949 1.033-1.094.078.258.162.524.244.788A147.532 147.532 0 0 0 5.157 24a.56.56 0 0 0 .43-.312c.13-.282.83-1.775 1.908-3.875.413 1.303.88 2.679 1.386 4.109a.494.494 0 0 0 .076-.465 103.735 103.735 0 0 1-1.308-3.945c.154-.299.316-.612.484-.932.125.04.255.094.389.155.203.186.352.491.482.84a1.515 1.515 0 0 0-.334-1.098c1.335.258 2.547.09 3.287-.81a3.97 3.97 0 0 0 .192-.258c-.325.304-.682.404-1.313.273.996-.281 1.523-.617 2.035-1.22.12-.145.244-.303.371-.48-.943.722-1.927.822-2.9.493l-.045-.018c.914.02 2.203-.474 3.092-1.189.41-.33.796-.73 1.17-1.21.28-.359.55-.76.82-1.216.234-.393.468-.824.7-1.293a2.83 2.83 0 0 1-.74.137l-.144.008c-.048.002-.093 0-.146.002.885-.198 1.5-.74 1.994-1.447-.24.117-.628.262-1.07.297-.058.006-.12.006-.182.006-.013-.002-.028 0-.047-.002.306-.078.574-.178.81-.309a3.363 3.363 0 0 0 .358-.236c.044-.037.088-.07.13-.106.099-.086.193-.18.28-.287.028-.034.056-.063.08-.098.036-.05.073-.098.104-.146a8.388 8.388 0 0 0 .51-.828c.015-.031.032-.057.046-.088.04-.084.08-.16.11-.227.042-.099.074-.179.092-.238a.515.515 0 0 1-.108.051c-.273.112-.727.187-1.086.201-.004 0-.008 0-.013.004h-.067c.72-.214 1.067-.45 1.422-.818a13.883 13.883 0 0 0 1.154-1.428c.264-.37.505-.738.692-1.072a6.5 6.5 0 0 0 .298-.592c.066-.157.122-.305.172-.45-.466.01-.986.011-1.48 0 .495.01 1.015.007 1.484-.005.5-1.485.063-2.262.063-2.262s-.526-1.212-1.4-.851c-.426.175-1.172.73-2.083 1.56l.514 1.45a17.561 17.561 0 0 1 1.703-1.602c-.257.22-.807.726-1.615 1.644-.256.29-.537.624-.844.997-.017.02-.035.038-.047.06a51.435 51.435 0 0 0-1.666 2.187c-.248.34-.498.704-.765 1.088h-.016c.002.02-.004.028-.01.032l-.101.152c-.104.155-.213.31-.318.47l-.352.534c-.061.09-.124.181-.186.277-.184.282-.367.573-.558.873a97.351 97.351 0 0 0-1.428 2.338 96.866 96.866 0 0 0-1.341 2.343c-.012.017-.02.04-.034.057a197.256 197.256 0 0 0-.668 1.223l-.097.181c-.17.318-.346.642-.52.979 0 .004-.005.008-.006.013-.026.048-.05.093-.072.141-.117.222-.218.424-.45.87a1.352 1.352 0 0 0-.233-.182l.345-.65c.047-.089.096-.177.143-.27l.04-.077.546-1.001.13-.233v-.006l-.001-.006c.169-.31.345-.62.52-.94.051-.087.102-.173.153-.265.224-.395.454-.794.684-1.197a91.685 91.685 0 0 1 2.135-3.504c.247-.386.503-.77.754-1.152.092-.138.182-.272.279-.41a72.9 72.9 0 0 1 .48-.701c.007-.012.019-.024.026-.037h.006c.26-.356.517-.713.773-1.065.278-.373.554-.735.83-1.09a31.075 31.075 0 0 1 1.777-2.075l-.515-1.446c-.06.057-.126.116-.192.178a32.37 32.37 0 0 0-.758.729c-.295.294-.597.606-.912.935a46.032 46.032 0 0 0-1.632 1.838l-.03.033.002.008c-.017.02-.033.044-.054.064-.266.323-.538.649-.801.985a39.105 39.105 0 0 0-1.445 1.95c-.043.06-.085.126-.127.186a26.458 26.458 0 0 0-1.403 2.303c-.13.247-.256.485-.37.715-.096.195-.187.395-.278.591-.21.463-.398.93-.566 1.399l.002.006a.36.36 0 0 0-.026.058c-.108.303-.203.608-.29.914-.14.174-.302.325-.483.46a3.505 3.505 0 0 0-.131-.153 5.148 5.148 0 0 0 .824-2.211 6.4 6.4 0 0 0-.016-1.488c-.046-.4-.126-.82-.238-1.274-.097-.393-.217-.81-.363-1.248-.091.185-.22.367-.379.545l-.086.094c-.029.032-.06.06-.092.094.434-.674.486-1.397.358-2.148a2.722 2.722 0 0 1-.49.85c-.033.038-.072.077-.11.116-.01.007-.019.018-.033.028.144-.24.25-.467.318-.698a1.29 1.29 0 0 0 .04-.146 2.85 2.85 0 0 0 .038-.225l.018-.146a2.11 2.11 0 0 0-.002-.354c-.003-.04-.004-.076-.01-.113-.01-.055-.016-.105-.027-.154a7.416 7.416 0 0 0-.193-.84c-.01-.028-.015-.056-.026-.084-.027-.079-.048-.149-.072-.209a2.1 2.1 0 0 0-.09-.209.455.455 0 0 1-.035.1c-.102.24-.34.57-.557.8-.003.003-.007.005-.007.01l-.04.043c.318-.58.39-.946.385-1.398a12.274 12.274 0 0 0-.16-1.615 10.68 10.68 0 0 0-.232-1.104 5.853 5.853 0 0 0-.18-.558 6.337 6.337 0 0 0-.172-.391 26.18 26.18 0 0 0 .002-.004C5.576.341 4.82.124 4.82.124s-.27-.11-.582-.123zm3.38 15.783l.032.082v.002c-.06.033-.116.067-.178.097-.012.004-.024.012-.039.018a2.41 2.41 0 0 0 .186-.2zm-.603 1.626c.13.136.25.242.354.32l.07.227a1.866 1.866 0 0 0-.246.053l-.03-.098c-.024-.084-.048-.17-.076-.257l-.021-.073zm.26.875a2.34 2.34 0 0 1 .271.01l.07.229a.778.778 0 0 1 .247-.004l-.326.627a127.643 127.643 0 0 1-.262-.862z', + npm: + 'M1.763 0C.786 0 0 .786 0 1.763v20.474C0 23.214.786 24 1.763 24h20.474c.977 0 1.763-.786 1.763-1.763V1.763C24 .786 23.214 0 22.237 0zM5.13 5.323l13.837.019-.009 13.836h-3.464l.01-10.382h-3.456L12.04 19.17H5.113z', + p2: + 'M11.109.024a15.58 15.58 0 00-.737.023C6.728.361 3.469 2.517 1.579 5.86A12.53 12.53 0 00.021 11.11c-.04.517-.02 1.745.035 2.208.306 2.682 1.353 5.06 3.07 6.965 1.962 2.173 4.586 3.467 7.437 3.663.42.032 1.043.04 1.02.012a2.404 2.404 0 00-.338-.074c-1.674-.33-3.388-1.13-4.777-2.232a12.344 12.344 0 01-2.45-2.636A12.387 12.387 0 011.884 12.5a12.413 12.413 0 01.56-4.274c.785-2.522 2.37-4.726 4.475-6.228A11.073 11.073 0 0111.156.122l.443-.098zm1.474.51C10.646.65 8.807 1.299 7.301 2.4 5.426 3.77 3.995 5.644 3.22 7.746c-.145.397-.282.82-.282.879 0 .012 3.828.024 10.31.024 8.463 0 10.315-.008 10.315-.036 0-.047-.153-.525-.283-.878-.153-.42-.576-1.31-.82-1.722-.4-.683-.91-1.373-1.474-1.992-1.65-1.82-3.593-2.934-5.82-3.334-.785-.141-1.8-.2-2.585-.153zM23.83 9.97c-.02 0-4.792 0-10.609.004l-10.573.008-.011.059c-.036.16-.134 1.081-.134 1.242 0 .028 1.785.032 10.746.032H24v-.075c0-.102-.07-.791-.106-1.054-.02-.16-.04-.216-.063-.216zm-10.573 2.635c-9.37-.004-10.73 0-10.742.035-.02.04.024.557.075.973.02.157.035.298.035.314 0 .027 2.137.035 10.624.035h10.624l.024-.188c.043-.326.102-.97.094-1.067l-.008-.094zm.003 2.718c-8.882 0-10.321.004-10.321.035 0 .02.054.208.12.42a11.122 11.122 0 002.072 3.741c.282.342.945 1.036 1.228 1.287 1.568 1.4 3.247 2.216 5.18 2.53.605.094.886.113 1.75.11.91 0 1.297-.032 2.023-.177 2.11-.416 3.914-1.451 5.53-3.17 1.267-1.348 2.106-2.76 2.628-4.41l.117-.366z', + docker: + 'M13.983 11.078h2.119a.186.186 0 00.186-.185V9.006a.186.186 0 00-.186-.186h-2.119a.185.185 0 00-.185.185v1.888c0 .102.083.185.185.185m-2.954-5.43h2.118a.186.186 0 00.186-.186V3.574a.186.186 0 00-.186-.185h-2.118a.185.185 0 00-.185.185v1.888c0 .102.082.185.185.185m0 2.716h2.118a.187.187 0 00.186-.186V6.29a.186.186 0 00-.186-.185h-2.118a.185.185 0 00-.185.185v1.887c0 .102.082.185.185.186m-2.93 0h2.12a.186.186 0 00.184-.186V6.29a.185.185 0 00-.185-.185H8.1a.185.185 0 00-.185.185v1.887c0 .102.083.185.185.186m-2.964 0h2.119a.186.186 0 00.185-.186V6.29a.185.185 0 00-.185-.185H5.136a.186.186 0 00-.186.185v1.887c0 .102.084.185.186.186m5.893 2.715h2.118a.186.186 0 00.186-.185V9.006a.186.186 0 00-.186-.186h-2.118a.185.185 0 00-.185.185v1.888c0 .102.082.185.185.185m-2.93 0h2.12a.185.185 0 00.184-.185V9.006a.185.185 0 00-.184-.186h-2.12a.185.185 0 00-.184.185v1.888c0 .102.083.185.185.185m-2.964 0h2.119a.185.185 0 00.185-.185V9.006a.185.185 0 00-.184-.186h-2.12a.186.186 0 00-.186.186v1.887c0 .102.084.185.186.185m-2.92 0h2.12a.185.185 0 00.184-.185V9.006a.185.185 0 00-.184-.186h-2.12a.185.185 0 00-.184.185v1.888c0 .102.082.185.185.185M23.763 9.89c-.065-.051-.672-.51-1.954-.51-.338.001-.676.03-1.01.087-.248-1.7-1.653-2.53-1.716-2.566l-.344-.199-.226.327c-.284.438-.49.922-.612 1.43-.23.97-.09 1.882.403 2.661-.595.332-1.55.413-1.744.42H.751a.751.751 0 00-.75.748 11.376 11.376 0 00.692 4.062c.545 1.428 1.355 2.48 2.41 3.124 1.18.723 3.1 1.137 5.275 1.137.983.003 1.963-.086 2.93-.266a12.248 12.248 0 003.823-1.389c.98-.567 1.86-1.288 2.61-2.136 1.252-1.418 1.998-2.997 2.553-4.4h.221c1.372 0 2.215-.549 2.68-1.009.309-.293.55-.65.707-1.046l.098-.288Z', +} + +export function FormatIcon({ + format, + className, +}: { + format: RepositoryFormat + className?: string +}) { + return ( + + + + ) +} diff --git a/internal/frontend/src/lib/api.ts b/internal/frontend/src/lib/api.ts index 6dcded0..0d24cb3 100644 --- a/internal/frontend/src/lib/api.ts +++ b/internal/frontend/src/lib/api.ts @@ -63,7 +63,7 @@ export type Permission = 'read' | 'write' | 'admin' export type Policy = 'release' | 'snapshot' | 'mixed' export type Visibility = 'public' | 'private' export type RepositoryType = 'hosted' | 'proxy' | 'group' -export type RepositoryFormat = 'maven2' | 'npm' | 'p2' +export type RepositoryFormat = 'maven2' | 'npm' | 'p2' | 'docker' export type MigrationState = 'idle' | 'running' | 'done' | 'cancelled' | 'failed' export type MigrationRepositoryState = 'pending' | 'running' | 'done' | 'failed' | 'skipped' @@ -80,6 +80,7 @@ export interface MigrationRepository { copied: number present: number failed: number + untranslatable: number copyingAssets: boolean } @@ -201,6 +202,90 @@ export interface ArtifactDetail { versions: Array<{ version: string; isSnapshot: boolean; updatedAt: number; createdAt: number }> } +export interface DockerLayer { + digest: string + short: string + mediaType: string + size: number + position: number + sharedWith: number + foreign: boolean + urls: string[] + stored: boolean +} + +export interface DockerChild { + digest: string + short: string + mediaType: string + platform: string + size: number + totalSize: number + os: string + architecture: string + variant: string + layerCount: number + indexed: boolean + referenceType: string +} + +export interface DockerHistoryStep { + created: number + createdBy: string + comment: string + emptyLayer: boolean +} + +export interface DockerConfig { + digest: string + size: number + user: string + workingDir: string + entrypoint: string[] + cmd: string[] + env: string[] + exposedPorts: string[] + history: DockerHistoryStep[] +} + +export interface DockerManifest { + repository: string + namespace: string + name: string + image: string + reference: string + digest: string + short: string + mediaType: string + size: number + totalSize: number + layerCount: number + isIndex: boolean + os: string + architecture: string + variant: string + imageCreated: number + labels: Record + annotations: Record + layers: DockerLayer[] + children: DockerChild[] + config: DockerConfig | null + pullBy: { tag: string; digest: string } +} + +export interface DockerReindexReport { + repositories: Array<{ repository: string; manifests: number; failed: number }> +} + +export interface DockerSweepReport { + repositories: Array<{ + repository: string + blobs: number + manifests: number + bytes: number + }> +} + export interface ApiToken { id: string name: string diff --git a/internal/frontend/src/lib/coordinates.ts b/internal/frontend/src/lib/coordinates.ts index 1aa9466..fcc7b96 100644 --- a/internal/frontend/src/lib/coordinates.ts +++ b/internal/frontend/src/lib/coordinates.ts @@ -6,6 +6,7 @@ export function packageLabel( name: string, ): string { if (format === 'npm') return namespace ? `@${namespace}/${name}` : name + if (format === 'docker') return namespace ? `${namespace}/${name}` : name return `${namespace}:${name}` } @@ -17,11 +18,15 @@ export function coordinateLabel( ): string { const label = packageLabel(format, namespace, name) if (!version) return label - return format === 'npm' ? `${label}@${version}` : `${label}:${version}` + if (format === 'npm') return `${label}@${version}` + // A docker tag is joined with a colon, the same as Maven, but the separator + // means something different: it names a moving pointer rather than a version. + return `${label}:${version}` } export function formatLabel(format: RepositoryFormat): string { if (format === 'npm') return 'npm' if (format === 'p2') return 'p2' + if (format === 'docker') return 'Docker' return 'Maven' } diff --git a/internal/frontend/src/routes/admin-maintenance.tsx b/internal/frontend/src/routes/admin-maintenance.tsx index 8cdf6a3..f7026f1 100644 --- a/internal/frontend/src/routes/admin-maintenance.tsx +++ b/internal/frontend/src/routes/admin-maintenance.tsx @@ -16,10 +16,13 @@ import { type MigrationRepository, type MigrationRequest, type MigrationStatus, + type DockerReindexReport, + type DockerSweepReport, type RepositoryFormat, type Visibility, } from '@/lib/api' import { useSession } from '@/lib/session' +import { formatBytes } from '@/lib/utils' const emptyForm: MigrationRequest = { url: '', @@ -109,6 +112,7 @@ function Progress({ entry }: { entry: MigrationRepository }) { {entry.copied} copied {entry.present > 0 ? `, ${entry.present} already present` : ''} {entry.failed > 0 ? `, ${entry.failed} failed` : ''} + {entry.untranslatable > 0 ? `, ${entry.untranslatable} not applicable here` : ''} ) } @@ -119,14 +123,18 @@ function RunSummary({ status }: { status: MigrationStatus }) { copied: sum.copied + entry.copied, present: sum.present + entry.present, failed: sum.failed + entry.failed, + untranslatable: sum.untranslatable + entry.untranslatable, }), - { copied: 0, present: 0, failed: 0 }, + { copied: 0, present: 0, failed: 0, untranslatable: 0 }, ) return (

{totals.copied} files copied, {totals.present} already present {totals.failed > 0 ? `, ${totals.failed} failed` : ''} + {totals.untranslatable > 0 + ? `, ${totals.untranslatable} paths this server has no place for` + : ''} {status.current ? `, currently slurping ${status.current}...` : ''}

) @@ -321,6 +329,157 @@ function statusVariant(state: MigrationStatus['state']) { return 'neutral' as const } +// A docker repository is the one format that cannot reclaim space on delete: a tag's +// layers are shared, so removing one leaves them behind on purpose. This is where +// that decision gets revisited. +function DockerSweepCard() { + const client = useQueryClient() + const [report, setReport] = useState(null) + const [swept, setSwept] = useState(false) + const [reindexed, setReindexed] = useState(null) + + const repositories = useQuery({ + queryKey: ['repositories'], + queryFn: () => api<{ repositories: Array<{ format: RepositoryFormat }> }>('/repositories'), + }) + + const preview = useMutation({ + mutationFn: () => api('/admin/docker/sweep'), + onSuccess: (result) => { + setReport(result) + setSwept(false) + }, + }) + + // Rebuilding reads every stored manifest afresh. It repairs a migration that was + // interrupted, or one run before this server knew how to index, without recopying. + const reindex = useMutation({ + mutationFn: () => post('/admin/docker/reindex'), + onSuccess: (result) => setReindexed(result), + }) + + const sweep = useMutation({ + mutationFn: () => post('/admin/docker/sweep'), + onSuccess: (result) => { + setReport(result) + setSwept(true) + client.invalidateQueries({ queryKey: ['repositories'] }) + }, + }) + + const hasDocker = (repositories.data?.repositories ?? []).some( + (entry) => entry.format === 'docker', + ) + if (repositories.isLoading || !hasDocker) return null + + const entries = (report?.repositories ?? []).filter( + (entry) => entry.blobs > 0 || entry.manifests > 0, + ) + const error = (preview.error ?? sweep.error ?? reindex.error) as ApiError | null + + return ( + + + Docker housekeeping + + Reclaiming removes layers and untagged manifests no tag can reach any more. A + layer shared with another tag is kept, which is why deleting a tag reclaims less + than its size; it also runs on its own every few hours. Rebuilding metadata + re-reads every stored manifest, which repairs a migration that stopped halfway. + + + +
+ {error ? : null} + +
+ + {report && !swept && entries.length > 0 ? ( + + ) : null} + +
+ + {reindexed ? ( +

+ Read {reindexed.repositories.reduce((sum, entry) => sum + entry.manifests, 0)} manifests + {reindexed.repositories.some((entry) => entry.failed > 0) + ? `, ${reindexed.repositories.reduce((sum, entry) => sum + entry.failed, 0)} could not be read` + : ''} + . +

+ ) : null} + + {report ? ( + entries.length === 0 ? ( +

+ Nothing to reclaim. Every layer and manifest here is still reachable from a + tag. +

+ ) : ( +
+

+ {swept + ? 'Reclaimed.' + : 'Nothing has been removed yet.'} +

+
+ + + + Repository + Layers + Manifests + {swept ? Reclaimed : null} + + + + {entries.map((entry) => ( + + {entry.repository} + + {entry.blobs} + + + {entry.manifests} + + {swept ? ( + + {formatBytes(entry.bytes)} + + ) : null} + + ))} + +
+
+
+ ) + ) : null} +
+ + ) +} + export function AdminMaintenanceRoute() { const { user } = useSession() if (user?.role !== 'admin') return @@ -333,6 +492,7 @@ export function AdminMaintenanceRoute() { description="One-off operations on this instance." /> + ) } diff --git a/internal/frontend/src/routes/admin-repositories.tsx b/internal/frontend/src/routes/admin-repositories.tsx index 46b9f3f..a3a3d54 100644 --- a/internal/frontend/src/routes/admin-repositories.tsx +++ b/internal/frontend/src/routes/admin-repositories.tsx @@ -206,6 +206,7 @@ function CreateRepositoryDialog() { const isGroup = type === 'group' const isNPM = format === 'npm' const isP2 = format === 'p2' + const isDocker = format === 'docker' const existing = useQuery({ queryKey: ['repositories'], @@ -267,7 +268,9 @@ function CreateRepositoryDialog() { New repository - {isNPM ? 'npm registry' : 'Maven 2 layout'}, served at /repository/{name || 'name'} + {isDocker + ? `Docker registry, pulled as ${window.location.host}/${name || 'name'}/` + : `${isNPM ? 'npm registry' : isP2 ? 'p2 update site' : 'Maven 2 layout'}, served at /repository/${name || 'name'}`} @@ -289,6 +292,7 @@ function CreateRepositoryDialog() { + @@ -319,7 +323,9 @@ function CreateRepositoryDialog() { ? 'The registry base URL this repository mirrors.' : isP2 ? 'The host root to mirror, without a path. Update sites point at children above their own directory, which only resolves from the root.' - : 'The Maven 2 base URL this repository mirrors.' + : isDocker + ? 'The registry to pull through. Docker Hub needs no path.' + : 'The Maven 2 base URL this repository mirrors.' } > @@ -627,9 +635,13 @@ function SettingsDialog({ repository }: { repository: Repository }) { className="mt-0.5" /> - Allow overwriting existing releases + {repository.format === 'docker' + ? 'Allow tags to be moved to another image' + : 'Allow overwriting existing releases'} - Off by default so a published version can never change underneath a build. + {repository.format === 'docker' + ? 'Normal Docker practice, since a tag like latest is meant to move. Turn it off to make every tag here permanent.' + : 'Off by default so a published version can never change underneath a build.'} diff --git a/internal/frontend/src/routes/artifact.tsx b/internal/frontend/src/routes/artifact.tsx index b99344c..c2b3c14 100644 --- a/internal/frontend/src/routes/artifact.tsx +++ b/internal/frontend/src/routes/artifact.tsx @@ -11,14 +11,17 @@ import { CopyButton } from '@/components/copy-button' import { DeleteArtifactDialog } from '@/components/delete-artifact' import { MavenDependency, MavenSetup } from '@/components/snippets/maven' import { NPMDependency, NPMSetup } from '@/components/snippets/npm' +import { DockerDependency, DockerUsage } from '@/components/snippets/docker' +import { DockerManifestView } from '@/components/docker-manifest' import { api, type ArtifactDetail, type BrowseEntry, + type DockerManifest, type RepositoryDetail, type RepositoryFormat, } from '@/lib/api' -import { coordinateLabel } from '@/lib/coordinates' +import { coordinateLabel, packageLabel } from '@/lib/coordinates' import { cn, formatBytes, formatRelative, plural } from '@/lib/utils' function Files({ @@ -97,18 +100,34 @@ function Snippets({ namespace, name, version, + digest, }: { format: RepositoryFormat repository: string namespace: string name: string version: string + digest?: string }) { const { data } = useQuery({ queryKey: ['repository', repository], queryFn: () => api(`/repositories/${repository}`), }) + if (format === 'docker') { + return ( +
+ + {data ? : null} +
+ ) + } + if (format === 'npm') { return (
@@ -150,16 +169,33 @@ export function ArtifactRoute() { enabled: name.length > 0, }) + // The pull snippets want the manifest digest, and the manifest tab wants the whole + // record. They share a query key, so this costs one request rather than two. + const reference = selected ?? data?.versions[0]?.version ?? '' + const manifestQuery = new URLSearchParams({ namespace, name, reference }) + const manifest = useQuery({ + queryKey: ['docker-manifest', repository, namespace, name, reference], + queryFn: () => + api(`/repositories/${repository}/docker/manifests?${manifestQuery}`), + enabled: data?.format === 'docker' && reference.length > 0, + }) + if (!name) return if (isLoading) return if (error) return if (!data) return null - const version = selected ?? data.versions[0]?.version ?? '' + const version = reference const format = data.format - const npm = format === 'npm' + const docker = format === 'docker' const coordinates = coordinateLabel(format, namespace, name, version) + // npm and docker both write the namespace into the name, so the heading is one + // line; Maven and p2 keep it above. The separator before a version differs again. + const inlineNamespace = format === 'npm' || docker + const namespacePrefix = format === 'npm' ? `@${namespace}/` : `${namespace}/` + const versionSeparator = format === 'npm' ? '@' : ':' + return ( <>
@@ -172,9 +208,9 @@ export function ArtifactRoute() {
- {npm ? ( + {inlineNamespace ? (

- {namespace ? @{namespace}/ : null} + {namespace ? {namespacePrefix} : null} {name}

) : ( @@ -191,14 +227,16 @@ export function ArtifactRoute() { {data.latest && data.latest !== data.release ? ( latest {data.latest} ) : null} - {plural(data.versions.length, 'version')} + + {plural(data.versions.length, docker ? 'tag' : 'version')} +
- {npm ? ( + {inlineNamespace ? ( namespace ? ( - @{namespace}/ + {namespacePrefix} ) : null ) : ( <> @@ -207,7 +245,7 @@ export function ArtifactRoute() { )} {name} - {npm ? '@' : ':'} + {versionSeparator} {version} @@ -216,7 +254,9 @@ export function ArtifactRoute() {
-

Versions

+

+ {docker ? 'Tags' : 'Versions'} +

{data.versions.map((entry) => (
diff --git a/internal/frontend/src/routes/repositories.tsx b/internal/frontend/src/routes/repositories.tsx index 2f74d49..44f8bf1 100644 --- a/internal/frontend/src/routes/repositories.tsx +++ b/internal/frontend/src/routes/repositories.tsx @@ -2,6 +2,7 @@ import { Link } from 'react-router-dom' import { useQuery } from '@tanstack/react-query' import { Globe, Lock } from 'lucide-react' import { PolicyBadge } from '@/components/ui/badge' +import { FormatIcon } from '@/components/ui/format-icon' import { Card } from '@/components/ui/card' import { EmptyState, ErrorBlock, LoadingBlock, PageHeading } from '@/components/ui/feedback' import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from '@/components/ui/table' @@ -32,8 +33,9 @@ function RepositoryTable({ repositories }: { repositories: Repository[] }) { + {repository.name} {repository.description ? ( diff --git a/internal/frontend/src/routes/repository.tsx b/internal/frontend/src/routes/repository.tsx index 1255091..626575c 100644 --- a/internal/frontend/src/routes/repository.tsx +++ b/internal/frontend/src/routes/repository.tsx @@ -11,6 +11,7 @@ import { Coordinate, CopyButton } from '@/components/copy-button' import { MavenUsage } from '@/components/snippets/maven' import { NPMUsage } from '@/components/snippets/npm' import { P2Usage } from '@/components/snippets/p2' +import { DockerUsage } from '@/components/snippets/docker' import { api, type ArtifactSummary, @@ -88,9 +89,11 @@ function Browser({ ) @@ -191,16 +194,22 @@ function Overview({ if (artifacts.length === 0) { return isProxy ? ( ) : ( ) @@ -211,9 +220,9 @@ function Overview({ - Artifact + {format === 'docker' ? 'Image' : 'Artifact'} Latest - Versions + {format === 'docker' ? 'Tags' : 'Versions'} Updated @@ -251,9 +260,23 @@ function Overview({ function Usage({ repository }: { repository: RepositoryDetail }) { if (repository.format === 'npm') return if (repository.format === 'p2') return + if (repository.format === 'docker') return return } +// dockerOrDefault keeps the stat labels honest per format without three nested +// ternaries at each call site. +function dockerOrDefault( + isDocker: boolean, + isProxy: boolean, + docker: string, + proxy: string, + hosted: string, +): string { + if (isDocker) return docker + return isProxy ? proxy : hosted +} + export function RepositoryRoute() { const params = useParams() const name = params.name! @@ -268,7 +291,12 @@ export function RepositoryRoute() { if (error) return if (!data) return null - const endpoint = `${window.location.origin}/repository/${data.name}` + const docker = data.format === 'docker' + // A Docker client builds its own URLs from the image reference, so the thing to + // copy is the host and repository prefix, not the path the other formats serve on. + const endpoint = docker + ? `${window.location.host}/${data.name}/` + : `${window.location.origin}/repository/${data.name}` return ( <> @@ -300,6 +328,17 @@ export function RepositoryRoute() { + {docker ? ( +

+ Images are addressed as{' '} + + {window.location.host}/{data.name}/<image>:<tag> + + . The repository name is part of the reference, which is how one host serves + several repositories without a port or a hostname of its own. +

+ ) : null} + {data.type === 'proxy' ? (

Mirrors {data.remoteUrl}. Artifacts are @@ -311,11 +350,14 @@ export function RepositoryRoute() { ) : null}

- +
diff --git a/internal/migrate/docker.go b/internal/migrate/docker.go new file mode 100644 index 0000000..14a6caa --- /dev/null +++ b/internal/migrate/docker.go @@ -0,0 +1,66 @@ +package migrate + +import ( + "strings" + + "arca/internal/docker" +) + +// Nexus lays a docker repository out as the registry API addresses it, under a v2 +// prefix, and stores blobs once on a shared path rather than per image. Verified +// against a Nexus 3.70 instance, which serves only the shared shape; the per-image one +// is kept because other versions may differ and it costs a case. +// +// v2/-/blobs/sha256: the shared blob store +// v2//blobs/sha256: a per-image blob +// v2//manifests/sha256: a manifest by digest +// v2//manifests/ a tag, which only the components walk lists +const ( + nexusRoot = "v2/" + nexusSharedBlobs = "-" + nexusBlobs = "blobs" + nexusManifests = "manifests" +) + +// DockerPath maps a Nexus docker path onto this server's layout. It reports false for +// anything it does not recognise, so an unexpected shape is skipped and counted rather +// than stored somewhere nothing will read it back from. +func DockerPath(path string) (string, bool) { + rest, found := strings.CutPrefix(strings.TrimPrefix(path, "/"), nexusRoot) + if !found { + return "", false + } + + cut := strings.LastIndex(rest, "/"+nexusBlobs+"/") + if cut >= 0 { + // Every blob lands in one store regardless of which image Nexus filed it + // under, because a digest names the same bytes either way. + digest, ok := docker.ParseDigest(rest[cut+len(nexusBlobs)+2:]) + if !ok { + return "", false + } + image := rest[:cut] + if image != nexusSharedBlobs && !docker.IsValidName(image) { + return "", false + } + return docker.BlobPath(digest), true + } + + cut = strings.LastIndex(rest, "/"+nexusManifests+"/") + if cut < 0 { + return "", false + } + + image, reference := rest[:cut], rest[cut+len(nexusManifests)+2:] + if !docker.IsValidName(image) { + return "", false + } + + if digest, ok := docker.ParseDigest(reference); ok { + return docker.ManifestPath(image, digest), true + } + if docker.IsValidTag(reference) { + return docker.TagPath(image, reference), true + } + return "", false +} diff --git a/internal/migrate/docker_test.go b/internal/migrate/docker_test.go new file mode 100644 index 0000000..91e6928 --- /dev/null +++ b/internal/migrate/docker_test.go @@ -0,0 +1,91 @@ +package migrate + +import ( + "strings" + "testing" +) + +func TestDockerPath(t *testing.T) { + hex := strings.Repeat("ab", 32) + digest := "sha256:" + hex + + cases := []struct { + name string + path string + want string + ok bool + }{ + // The shared blob store is what a real Nexus 3.70 uses, and the only shape its + // asset listing showed. + {name: "a shared blob", path: "v2/-/blobs/" + digest, want: "_blobs/sha256/" + hex, ok: true}, + {name: "a leading slash is tolerated", path: "/v2/-/blobs/" + digest, want: "_blobs/sha256/" + hex, ok: true}, + + // Kept because other versions may file blobs per image, and a digest names the + // same bytes either way. + {name: "a per-image blob", path: "v2/nginx/blobs/" + digest, want: "_blobs/sha256/" + hex, ok: true}, + {name: "a per-image blob under a namespace", path: "v2/team/api/blobs/" + digest, want: "_blobs/sha256/" + hex, ok: true}, + + { + name: "a manifest by digest", path: "v2/team/api/manifests/" + digest, + want: "team/api/_manifests/sha256/" + hex, ok: true, + }, + { + name: "a manifest by tag", path: "v2/team/api/manifests/1.0", + want: "team/api/1.0/manifest.json", ok: true, + }, + { + name: "a deep image name", path: "v2/a/b/c/manifests/latest", + want: "a/b/c/latest/manifest.json", ok: true, + }, + + {name: "nothing", path: ""}, + {name: "no v2 prefix", path: "team/api/manifests/1.0"}, + {name: "the tag list", path: "v2/team/api/tags/list"}, + {name: "an unknown verb", path: "v2/team/api/layers/1"}, + {name: "a nameless manifest", path: "v2/manifests/1.0"}, + {name: "an uppercase image", path: "v2/TEAM/api/manifests/1.0"}, + {name: "an unusable digest", path: "v2/-/blobs/sha256:short"}, + {name: "an unusable algorithm", path: "v2/-/blobs/md5:" + strings.Repeat("ab", 16)}, + {name: "an invalid tag", path: "v2/team/api/manifests/.hidden"}, + {name: "a traversal attempt", path: "v2/team/api/manifests/sha256:../../etc/passwd"}, + } + + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + got, ok := DockerPath(testCase.path) + if ok != testCase.ok { + t.Fatalf("DockerPath(%q) ok = %v, want %v", testCase.path, ok, testCase.ok) + } + if ok && got != testCase.want { + t.Fatalf("DockerPath(%q) = %q, want %q", testCase.path, got, testCase.want) + } + }) + } +} + +// Only docker rewrites paths. Every other format stores what Nexus stored, and quietly +// changing one of those would move artifacts out from under the builds that resolve them. +func TestTranslatePathLeavesOtherFormatsAlone(t *testing.T) { + cases := []struct { + format string + path string + }{ + {"maven2", "com/example/app/1.0/app-1.0.jar"}, + {"npm", "@scope/package/1.0.0/package-1.0.0.tgz"}, + {"p2", "logging/2.0/plugins/slf4j.api_2.0.16.jar"}, + } + + for _, testCase := range cases { + t.Run(testCase.format, func(t *testing.T) { + decision := Decision{Format: testCase.format} + + got, ok := decision.TranslatePath(testCase.path) + if !ok || got != testCase.path { + t.Fatalf("TranslatePath(%q) = (%q, %v), want it unchanged", testCase.path, got, ok) + } + if decision.CopyComponents() { + t.Fatal("a non-docker format asked for the component walk") + } + }) + } +} diff --git a/internal/migrate/plan.go b/internal/migrate/plan.go index ad1bd78..e519dba 100644 --- a/internal/migrate/plan.go +++ b/internal/migrate/plan.go @@ -33,6 +33,24 @@ type Decision struct { Reason string } +// TranslatePath maps a source path onto the layout this server stores. Most formats +// store what Nexus stored, so the default is the path unchanged; docker addresses its +// content entirely differently at both ends. +func (d Decision) TranslatePath(path string) (string, bool) { + if d.Format == format.Docker { + return DockerPath(path) + } + return path, true +} + +// CopyComponents reports whether the source's component listing has to be walked as +// well as its assets. Only docker needs it, and it needs it badly: the asset list names +// no tags at all, so assets alone copy every byte of every image and leave none of them +// pullable. +func (d Decision) CopyComponents() bool { + return d.CopyAssets && d.Format == format.Docker +} + func (d Decision) Name() string { return d.Target } func (d Decision) SourceName() string { return d.Source.Name } @@ -94,10 +112,7 @@ func decide(repository nexus.Repository, options Options) Decision { // Membership is not migrated: arca groups exist only for p2, and which // members a group should carry is a decision worth making by hand. if repository.IsGroup() { - decision.Reason = fmt.Sprintf( - "groups are not migrated; recreate it as a p2 group, or point clients at the members directly (%s)", - strings.Join(repository.Members(), ", "), - ) + decision.Reason = groupReason(repository) return decision } @@ -131,12 +146,29 @@ func decide(repository nexus.Repository, options Options) Decision { return decision } +// groupReason explains a skipped group. The member list is often unavailable: the +// repository endpoint returns empty attributes on some Nexus versions, so claiming a +// group has no members would be worse than saying they could not be read. +func groupReason(repository nexus.Repository) string { + members := repository.Members() + if len(members) == 0 { + return "groups are not migrated, and this Nexus did not report which repositories this one " + + "contains; recreate it by hand, or point clients at the members directly" + } + return fmt.Sprintf( + "groups are not migrated; recreate it as a p2 group, or point clients at the members directly (%s)", + strings.Join(members, ", "), + ) +} + func formatFor(nexusFormat string, options Options) (string, bool) { switch nexusFormat { case "maven2": return format.Maven2, true case "npm": return format.NPM, true + case "docker": + return format.Docker, true case "raw": return options.rawFormat(), true default: @@ -146,7 +178,15 @@ func formatFor(nexusFormat string, options Options) (string, bool) { // policyFor carries a Maven repository's version policy across. Nexus has no // equivalent for the other formats, which take arca's default. +// +// Docker is the exception: its tags carry arbitrary suffixes, and real ones like +// 0.1.9-swaggerui-staging are neither a release nor a prerelease by any rule worth +// writing, so a mixed policy is the only honest choice. func policyFor(repository nexus.Repository) string { + if repository.Format == "docker" { + return models.PolicyMixed + } + switch strings.ToUpper(repository.VersionPolicy()) { case "SNAPSHOT": return models.PolicySnapshot diff --git a/internal/migrate/plan_test.go b/internal/migrate/plan_test.go index 8a52209..0adc187 100644 --- a/internal/migrate/plan_test.go +++ b/internal/migrate/plan_test.go @@ -55,8 +55,12 @@ func TestPlanDecides(t *testing.T) { action: ActionSkip, reason: "maven-central, maven-snapshots", }, { - name: "an unsupported format is skipped", source: repository("docker-hosted", "docker", "hosted"), - action: ActionSkip, reason: "no docker format", + name: "an unsupported format is skipped", source: repository("gems", "rubygems", "hosted"), + action: ActionSkip, reason: "no rubygems format", + }, + { + name: "a docker repository carries across", source: repository("docker", "docker", "hosted"), + action: ActionCreate, wantFormat: "docker", wantType: "hosted", copies: true, }, { name: "a proxy without an upstream is skipped", source: orphan, diff --git a/internal/migrate/run.go b/internal/migrate/run.go index fc0e45d..f1b6408 100644 --- a/internal/migrate/run.go +++ b/internal/migrate/run.go @@ -22,6 +22,10 @@ type Destination interface { CreateRepository(ctx context.Context, decision Decision, visibility string) error HasAsset(ctx context.Context, repository, path string) (bool, error) PutAsset(ctx context.Context, repository, path string, size int64, body io.Reader) error + // Settled runs once a repository's content is all in place, for a format whose + // metadata cannot be built as the files arrive. Docker needs it: a manifest + // references blobs that the copy order gives no guarantee of having seen yet. + Settled(ctx context.Context, decision Decision) error } // Result records what happened to one repository. @@ -32,7 +36,10 @@ type Result struct { Copied int Skipped int Failed int - Errors []string + // Untranslatable counts source paths this server has no place for. Reporting them + // is what stops a copy claiming to be complete when it silently dropped content. + Untranslatable int + Errors []string } type Reporter interface { @@ -95,6 +102,17 @@ func (r *Runner) applyOne(ctx context.Context, decision Decision) Result { } r.copyAssets(ctx, decision, &result) + if decision.CopyComponents() { + r.copyComponents(ctx, decision, &result) + } + + // Settling comes last for the same reason it exists: only now is every blob a + // manifest might reference actually present. + if err := r.Target.Settled(ctx, decision); err != nil { + result.Failed++ + result.Errors = append(result.Errors, "building metadata: "+err.Error()) + } + return result } @@ -114,24 +132,12 @@ func (r *Runner) copyAssets(ctx context.Context, decision Decision, result *Resu go func() { defer workers.Done() for asset := range queue { - copied, err := r.copyAsset(ctx, decision.Name(), asset) + // The transfer happens outside the lock. Holding it across a + // download would serialise the pool this exists to parallelise. + copied, err := r.copyAsset(ctx, decision, asset) mutex.Lock() - switch { - case err != nil: - result.Failed++ - result.Errors = append(result.Errors, asset.Path+": "+err.Error()) - if r.Reporter != nil { - r.Reporter.Problem(decision.Name(), asset.Path, err) - } - case copied: - result.Copied++ - default: - result.Skipped++ - } - if r.Reporter != nil { - r.Reporter.Progress(decision.Name(), result.Copied, result.Skipped, result.Failed) - } + r.record(decision, asset, copied, err, result) mutex.Unlock() } }() @@ -152,10 +158,71 @@ func (r *Runner) copyAssets(ctx context.Context, decision Decision, result *Resu } } +// copyComponents walks the source grouped by version. It exists for docker, whose tags +// appear in no other listing, and it copies only the assets the asset walk could not +// have seen. +func (r *Runner) copyComponents(ctx context.Context, decision Decision, result *Result) { + err := r.Source.Components(ctx, decision.SourceName(), func(page []nexus.Component) error { + if ctx.Err() != nil { + return ctx.Err() + } + + for _, component := range page { + for _, asset := range component.Assets { + r.recordCopy(ctx, decision, asset, result) + } + } + return nil + }) + + if err != nil { + result.Failed++ + result.Errors = append(result.Errors, err.Error()) + } +} + +// recordCopy transfers one asset and books the outcome. Only the components walk uses +// it, which runs on one goroutine, so it needs no lock of its own. +func (r *Runner) recordCopy(ctx context.Context, decision Decision, asset nexus.Asset, result *Result) { + copied, err := r.copyAsset(ctx, decision, asset) + r.record(decision, asset, copied, err, result) +} + +// record books one outcome. Callers hold whatever lock the result needs. +func (r *Runner) record(decision Decision, asset nexus.Asset, copied bool, err error, result *Result) { + switch { + case errors.Is(err, errUntranslatable): + result.Untranslatable++ + case err != nil: + result.Failed++ + result.Errors = append(result.Errors, asset.Path+": "+err.Error()) + if r.Reporter != nil { + r.Reporter.Problem(decision.Name(), asset.Path, err) + } + case copied: + result.Copied++ + default: + result.Skipped++ + } + + if r.Reporter != nil { + r.Reporter.Progress(decision.Name(), result.Copied, result.Skipped, result.Failed) + } +} + +var errUntranslatable = errors.New("migrate: this server has no place for that path") + // copyAsset reports whether anything was transferred. An asset already in arca // is left alone so a rerun costs one HEAD instead of a download. -func (r *Runner) copyAsset(ctx context.Context, repository string, asset nexus.Asset) (bool, error) { - present, err := r.Target.HasAsset(ctx, repository, asset.Path) +func (r *Runner) copyAsset(ctx context.Context, decision Decision, asset nexus.Asset) (bool, error) { + path, ok := decision.TranslatePath(asset.Path) + if !ok { + return false, errUntranslatable + } + + repository := decision.Name() + + present, err := r.Target.HasAsset(ctx, repository, path) if err != nil { return false, err } @@ -169,7 +236,7 @@ func (r *Runner) copyAsset(ctx context.Context, repository string, asset nexus.A } defer body.Close() - if err := r.Target.PutAsset(ctx, repository, asset.Path, asset.FileSize, body); err != nil { + if err := r.Target.PutAsset(ctx, repository, path, asset.FileSize, body); err != nil { return false, err } return true, nil diff --git a/internal/nexus/client.go b/internal/nexus/client.go index 4c5493d..a849987 100644 --- a/internal/nexus/client.go +++ b/internal/nexus/client.go @@ -80,9 +80,23 @@ type Asset struct { } `json:"checksum"` } -type assetPage struct { - Items []Asset `json:"items"` - ContinuationToken string `json:"continuationToken"` +// Component is one versioned thing, with its assets nested. For docker this is the +// only endpoint that names tags at all: the asset list reports manifests by digest and +// nothing else, so a migration driven off assets alone copies every byte and produces +// no tags. +type Component struct { + ID string `json:"id"` + Group string `json:"group"` + Name string `json:"name"` + Version string `json:"version"` + Repository string `json:"repository"` + Format string `json:"format"` + Assets []Asset `json:"assets"` +} + +type page[T any] struct { + Items []T `json:"items"` + ContinuationToken string `json:"continuationToken"` } func (c *Client) get(ctx context.Context, path string, query url.Values) (*http.Response, error) { @@ -135,6 +149,17 @@ func (c *Client) Repositories(ctx context.Context) ([]Repository, error) { // Nexus pages with. The callback runs per page so a large repository never has // to be held in memory at once. func (c *Client) Assets(ctx context.Context, repository string, visit func([]Asset) error) error { + return walk(ctx, c, "/service/rest/v1/assets", repository, "assets", visit) +} + +// Components walks the same repository grouped by version. Both walks are needed for +// docker, whose two endpoints expose disjoint views: assets hold the blobs and the +// digest-addressed manifests, components hold the tags. +func (c *Client) Components(ctx context.Context, repository string, visit func([]Component) error) error { + return walk(ctx, c, "/service/rest/v1/components", repository, "components", visit) +} + +func walk[T any](ctx context.Context, c *Client, path, repository, subject string, visit func([]T) error) error { token := "" for { @@ -143,25 +168,25 @@ func (c *Client) Assets(ctx context.Context, repository string, visit func([]Ass query.Set("continuationToken", token) } - response, err := c.get(ctx, "/service/rest/v1/assets", query) + response, err := c.get(ctx, path, query) if err != nil { return err } - var page assetPage - err = json.NewDecoder(response.Body).Decode(&page) + var current page[T] + err = json.NewDecoder(response.Body).Decode(¤t) response.Body.Close() if err != nil { - return fmt.Errorf("nexus: reading assets of %s: %w", repository, err) + return fmt.Errorf("nexus: reading %s of %s: %w", subject, repository, err) } - if err := visit(page.Items); err != nil { + if err := visit(current.Items); err != nil { return err } - if page.ContinuationToken == "" { + if current.ContinuationToken == "" { return nil } - token = page.ContinuationToken + token = current.ContinuationToken } } diff --git a/internal/proxy/client.go b/internal/proxy/client.go index 1e03143..52fe729 100644 --- a/internal/proxy/client.go +++ b/internal/proxy/client.go @@ -45,7 +45,8 @@ func (r *Response) Close() { } type Client struct { - http *http.Client + http *http.Client + tokens *tokenCache } func NewClient(timeout time.Duration) *Client { @@ -53,7 +54,13 @@ func NewClient(timeout time.Duration) *Client { transport.MaxIdleConnsPerHost = 16 transport.ResponseHeaderTimeout = 30 * time.Second - return &Client{http: &http.Client{Timeout: timeout, Transport: transport}} + // Redirects are followed, which container registries rely on: a blob GET is + // answered with a 307 to object storage. Go strips the Authorization header when + // a redirect crosses hosts, so the token never reaches the CDN. + return &Client{ + http: &http.Client{Timeout: timeout, Transport: transport}, + tokens: newTokenCache(), + } } func NormalizeRemoteURL(raw string) (string, error) { @@ -80,6 +87,46 @@ func (c *Client) Fetch(ctx context.Context, remote Remote, path string, conditio } target.Path = strings.TrimSuffix(target.Path, "/") + "/" + path + response, err := c.send(ctx, remote, target, conditional, "") + if err != nil { + return nil, err + } + + // A container registry answers an unauthenticated request with a challenge to a + // separate token service rather than accepting credentials itself. One retry with + // the issued token is the whole of that protocol. + if response.StatusCode == http.StatusUnauthorized { + answer, parseErr := parseChallenge(response.Header.Get("WWW-Authenticate")) + if parseErr == nil { + drain(response) + + token, tokenErr := c.token(ctx, remote, answer) + if tokenErr != nil { + return nil, tokenErr + } + if response, err = c.send(ctx, remote, target, conditional, token); err != nil { + return nil, err + } + } + } + + result := &Response{ + Status: response.StatusCode, + ContentType: response.Header.Get("Content-Type"), + ETag: response.Header.Get("ETag"), + LastModified: response.Header.Get("Last-Modified"), + } + + if response.StatusCode == http.StatusOK { + result.Body = response.Body + } else { + drain(response) + } + + return result, nil +} + +func (c *Client) send(ctx context.Context, remote Remote, target *url.URL, conditional Conditional, token string) (*http.Response, error) { request, err := http.NewRequestWithContext(ctx, http.MethodGet, target.String(), nil) if err != nil { return nil, fmt.Errorf("build upstream request: %w", err) @@ -92,7 +139,10 @@ func (c *Client) Fetch(ctx context.Context, remote Remote, path string, conditio request.Header.Set("User-Agent", userAgent) request.Header.Set("Accept", accept) - if remote.Username != "" || remote.Password != "" { + switch { + case token != "": + request.Header.Set("Authorization", "Bearer "+token) + case remote.Username != "" || remote.Password != "": request.SetBasicAuth(remote.Username, remote.Password) } if conditional.ETag != "" { @@ -106,20 +156,10 @@ func (c *Client) Fetch(ctx context.Context, remote Remote, path string, conditio if err != nil { return nil, fmt.Errorf("fetch %s: %w", target.Redacted(), err) } + return response, nil +} - result := &Response{ - Status: response.StatusCode, - ContentType: response.Header.Get("Content-Type"), - ETag: response.Header.Get("ETag"), - LastModified: response.Header.Get("Last-Modified"), - } - - if response.StatusCode == http.StatusOK { - result.Body = response.Body - } else { - _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096)) - response.Body.Close() - } - - return result, nil +func drain(response *http.Response) { + _, _ = io.Copy(io.Discard, io.LimitReader(response.Body, 4096)) + response.Body.Close() } diff --git a/internal/proxy/token.go b/internal/proxy/token.go new file mode 100644 index 0000000..1d3945a --- /dev/null +++ b/internal/proxy/token.go @@ -0,0 +1,214 @@ +package proxy + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "sync" + "time" +) + +// Container registries do not accept credentials directly. They answer an +// unauthenticated request with a Bearer challenge naming a separate token service, +// and the client exchanges its credentials there for a short-lived token scoped to +// one repository. Docker Hub is the common case: registry-1.docker.io challenges to +// auth.docker.io. +// +// The exchange is what makes anonymous pulls work too: with no credentials the token +// service still issues a token, it just carries fewer rights. + +const ( + // tokenSkew expires a cached token early, so one is never presented in the + // moment it stops being valid. + tokenSkew = 30 * time.Second + // tokenLifetime is what a service that reports no expiry is assumed to grant. + tokenLifetime = 5 * time.Minute + maxTokenBytes = 1 << 20 +) + +var errNoChallenge = errors.New("proxy: the response carries no bearer challenge") + +type challenge struct { + Realm string + Service string + Scope string +} + +// parseChallenge reads a WWW-Authenticate header. Only the Bearer scheme is handled: +// a Basic challenge means the credentials were already sent and rejected, which is not +// something a retry can fix. +func parseChallenge(header string) (challenge, error) { + rest, found := cutPrefixFold(header, "bearer ") + if !found { + return challenge{}, errNoChallenge + } + + parsed := challenge{} + for _, parameter := range splitParameters(rest) { + key, value, found := strings.Cut(parameter, "=") + if !found { + continue + } + + value = strings.Trim(strings.TrimSpace(value), `"`) + switch strings.ToLower(strings.TrimSpace(key)) { + case "realm": + parsed.Realm = value + case "service": + parsed.Service = value + case "scope": + parsed.Scope = value + } + } + + if parsed.Realm == "" { + return challenge{}, errNoChallenge + } + return parsed, nil +} + +func cutPrefixFold(value, prefix string) (string, bool) { + if len(value) < len(prefix) || !strings.EqualFold(value[:len(prefix)], prefix) { + return "", false + } + return value[len(prefix):], true +} + +// splitParameters splits on commas that are not inside a quoted value, because a +// scope routinely contains one: "repository:a:pull,push". +func splitParameters(value string) []string { + var parameters []string + quoted := false + start := 0 + + for index := 0; index < len(value); index++ { + switch value[index] { + case '"': + quoted = !quoted + case ',': + if !quoted { + parameters = append(parameters, value[start:index]) + start = index + 1 + } + } + } + return append(parameters, value[start:]) +} + +func (c challenge) key(remote Remote) string { + return remote.BaseURL + "|" + c.Realm + "|" + c.Service + "|" + c.Scope +} + +type cachedToken struct { + token string + expires time.Time +} + +type tokenCache struct { + mutex sync.Mutex + tokens map[string]cachedToken +} + +func newTokenCache() *tokenCache { return &tokenCache{tokens: map[string]cachedToken{}} } + +func (t *tokenCache) get(key string) (string, bool) { + t.mutex.Lock() + defer t.mutex.Unlock() + + entry, ok := t.tokens[key] + if !ok || time.Now().After(entry.expires) { + return "", false + } + return entry.token, true +} + +func (t *tokenCache) put(key, token string, lifetime time.Duration) { + t.mutex.Lock() + defer t.mutex.Unlock() + + t.tokens[key] = cachedToken{token: token, expires: time.Now().Add(lifetime - tokenSkew)} +} + +type tokenResponse struct { + Token string `json:"token"` + AccessToken string `json:"access_token"` + ExpiresIn int `json:"expires_in"` +} + +func (t tokenResponse) value() string { + if t.Token != "" { + return t.Token + } + return t.AccessToken +} + +func (t tokenResponse) lifetime() time.Duration { + if t.ExpiresIn <= 0 { + return tokenLifetime + } + return time.Duration(t.ExpiresIn) * time.Second +} + +// token answers a challenge, from cache when it can. Credentials are only ever sent +// to an HTTPS realm: the realm is named by the upstream's own response, so anything +// less would let a compromised or impersonated registry collect them in the clear. +func (c *Client) token(ctx context.Context, remote Remote, answer challenge) (string, error) { + key := answer.key(remote) + if cached, ok := c.tokens.get(key); ok { + return cached, nil + } + + realm, err := url.Parse(answer.Realm) + if err != nil || realm.Host == "" { + return "", fmt.Errorf("proxy: %q is not a usable token realm", answer.Realm) + } + credentialled := remote.Username != "" || remote.Password != "" + if credentialled && realm.Scheme != "https" { + return "", fmt.Errorf("proxy: refusing to send credentials to the plain-text token realm %s", realm.Host) + } + + query := realm.Query() + if answer.Service != "" { + query.Set("service", answer.Service) + } + if answer.Scope != "" { + query.Set("scope", answer.Scope) + } + realm.RawQuery = query.Encode() + + request, err := http.NewRequestWithContext(ctx, http.MethodGet, realm.String(), nil) + if err != nil { + return "", fmt.Errorf("proxy: build token request: %w", err) + } + request.Header.Set("User-Agent", userAgent) + request.Header.Set("Accept", "application/json") + if credentialled { + request.SetBasicAuth(remote.Username, remote.Password) + } + + response, err := c.http.Do(request) + if err != nil { + return "", fmt.Errorf("proxy: fetch a token from %s: %w", realm.Host, err) + } + defer response.Body.Close() + + if response.StatusCode != http.StatusOK { + return "", fmt.Errorf("proxy: %s answered %d for a token", realm.Host, response.StatusCode) + } + + var issued tokenResponse + if err := json.NewDecoder(io.LimitReader(response.Body, maxTokenBytes)).Decode(&issued); err != nil { + return "", fmt.Errorf("proxy: reading a token from %s: %w", realm.Host, err) + } + if issued.value() == "" { + return "", fmt.Errorf("proxy: %s issued an empty token", realm.Host) + } + + c.tokens.put(key, issued.value(), issued.lifetime()) + return issued.value(), nil +} diff --git a/internal/server/api_docker.go b/internal/server/api_docker.go new file mode 100644 index 0000000..67846ca --- /dev/null +++ b/internal/server/api_docker.go @@ -0,0 +1,436 @@ +package server + +import ( + "encoding/json" + "io" + "net/http" + "strings" + + "github.com/charmbracelet/log" + + "arca/internal/docker" + "arca/internal/format" + "arca/internal/store/models" +) + +type dockerLayerResponse struct { + Digest string `json:"digest"` + Short string `json:"short"` + MediaType string `json:"mediaType"` + Size int64 `json:"size"` + Position int64 `json:"position"` + // SharedWith counts the other manifests here that need this layer, so the UI + // can explain why removing one tag reclaims less than its total size. + SharedWith int `json:"sharedWith"` + Foreign bool `json:"foreign"` + URLs []string `json:"urls"` + Stored bool `json:"stored"` +} + +type dockerChildResponse struct { + Digest string `json:"digest"` + Short string `json:"short"` + MediaType string `json:"mediaType"` + Platform string `json:"platform"` + Size int64 `json:"size"` + TotalSize int64 `json:"totalSize"` + OS string `json:"os"` + Architecture string `json:"architecture"` + Variant string `json:"variant"` + LayerCount int `json:"layerCount"` + Indexed bool `json:"indexed"` + // ReferenceType names a child that is not a platform at all. buildkit attaches + // provenance and SBOM attestations to every multi-platform build, and they + // declare a platform of unknown/unknown, so without this they read as broken. + ReferenceType string `json:"referenceType"` +} + +type dockerHistoryResponse struct { + Created int64 `json:"created"` + CreatedBy string `json:"createdBy"` + Comment string `json:"comment"` + EmptyLayer bool `json:"emptyLayer"` +} + +type dockerConfigResponse struct { + Digest string `json:"digest"` + Size int64 `json:"size"` + User string `json:"user"` + WorkingDir string `json:"workingDir"` + Entrypoint []string `json:"entrypoint"` + Cmd []string `json:"cmd"` + Env []string `json:"env"` + ExposedPorts []string `json:"exposedPorts"` + History []dockerHistoryResponse `json:"history"` +} + +type dockerManifestResponse struct { + Repository string `json:"repository"` + Namespace string `json:"namespace"` + Name string `json:"name"` + Image string `json:"image"` + Reference string `json:"reference"` + Digest string `json:"digest"` + Short string `json:"short"` + MediaType string `json:"mediaType"` + Size int64 `json:"size"` + TotalSize int64 `json:"totalSize"` + LayerCount int `json:"layerCount"` + IsIndex bool `json:"isIndex"` + OS string `json:"os"` + Architecture string `json:"architecture"` + Variant string `json:"variant"` + ImageCreated int64 `json:"imageCreated"` + Labels map[string]string `json:"labels"` + Annotations map[string]string `json:"annotations"` + Layers []dockerLayerResponse `json:"layers"` + Children []dockerChildResponse `json:"children"` + Config *dockerConfigResponse `json:"config"` + PullBy map[string]string `json:"pullBy"` +} + +// routeDockerManifest describes one tag or digest: its layers, how much of each is +// shared, the platforms of an index, and the config the image was built with. It is +// what the file listing cannot be for docker, where a version's only file is a +// manifest of a few kilobytes and its real weight lives in shared blobs. +func (s *Server) routeDockerManifest(writer http.ResponseWriter, request *http.Request) error { + repository, _, err := s.loadAccessible(request, models.PermissionRead) + if err != nil { + return err + } + if repository.Format != format.Docker { + return fail(http.StatusBadRequest, "This is not a docker repository") + } + + namespace, name, err := requireCoordinates(request) + if err != nil { + return err + } + reference, err := requireText(request.URL.Query().Get("reference"), "reference", 200) + if err != nil { + return err + } + + image := docker.ImageName(namespace, name) + digest, err := s.resolveDockerReference(repository, image, reference) + if err != nil { + return err + } + + record, err := s.store.DockerManifest(repository.ID, digest.String()) + if err != nil { + return fail(http.StatusNotFound, "That manifest has not been indexed") + } + var config *docker.Config + + references, err := s.store.DockerReferences(repository.ID, digest.String()) + if err != nil { + return err + } + + response := dockerManifestResponse{ + Repository: repository.Name, + Namespace: namespace, + Name: name, + Image: image, + Reference: reference, + Digest: record.Digest, + Short: digest.Short(), + MediaType: record.MediaType, + Size: record.Size, + TotalSize: record.TotalSize, + LayerCount: record.LayerCount, + IsIndex: record.IsIndex(), + OS: record.OS, + Architecture: record.Architecture, + Variant: record.Variant, + ImageCreated: record.ImageCreated, + Labels: decodeJSONMap(record.Labels), + Annotations: decodeJSONMap(record.Annotations), + Layers: []dockerLayerResponse{}, + Children: []dockerChildResponse{}, + PullBy: map[string]string{ + "tag": repository.Name + "/" + image + ":" + reference, + "digest": repository.Name + "/" + image + "@" + record.Digest, + }, + } + + if response.IsIndex { + response.Children, err = s.describeDockerChildren(repository, references) + if err == nil { + s.backfillDockerIndexTime(repository, record, &response) + } + } else { + response.Layers, err = s.describeDockerLayers(repository, references) + if err == nil { + response.Config, config = s.describeDockerConfig(repository, references) + } + } + if err != nil { + return err + } + + // A proxy fetches a manifest before the config blob it points at, so indexing had + // nothing to read the platform from. Filling it in here, and writing it back, is + // what stops a proxied image reporting an unknown architecture for ever. + if config != nil { + s.backfillDockerPlatform(repository, record, *config, &response) + } + + writeJSON(writer, http.StatusOK, response) + return nil +} + +// resolveDockerReference turns a tag or a digest into the digest to look up. A tag +// resolves through the asset that holds its manifest, whose own SHA256 is that +// digest, so no tag table is needed. +func (s *Server) resolveDockerReference(repository *models.Repository, image, reference string) (docker.Digest, error) { + if digest, ok := docker.ParseDigest(reference); ok { + return digest, nil + } + if !docker.IsValidTag(reference) { + return docker.Digest{}, fail(http.StatusBadRequest, "reference must be a tag or a digest") + } + + asset, err := s.store.FindAsset(repository.ID, docker.TagPath(image, reference)) + if err != nil { + return docker.Digest{}, fail(http.StatusNotFound, "There is no tag %q on %s", reference, image) + } + return docker.SHA256(asset.SHA256), nil +} + +func (s *Server) describeDockerLayers(repository *models.Repository, references []models.DockerReference) ([]dockerLayerResponse, error) { + layers := make([]dockerLayerResponse, 0, len(references)) + + digests := make([]string, 0, len(references)) + for _, reference := range references { + if reference.Kind == models.DockerReferenceLayer { + digests = append(digests, reference.ChildDigest) + } + } + + sharing, err := s.store.DockerBlobSharing(repository.ID, digests) + if err != nil { + return nil, err + } + + for _, reference := range references { + if reference.Kind != models.DockerReferenceLayer { + continue + } + + layer := dockerLayerResponse{ + Digest: reference.ChildDigest, + MediaType: reference.MediaType, + Size: reference.Size, + Position: reference.Position, + Foreign: reference.IsForeign(), + URLs: splitLines(reference.URLs), + // The count includes this manifest, so what the UI wants is the rest. + SharedWith: max(sharing[reference.ChildDigest]-1, 0), + } + if digest, ok := docker.ParseDigest(reference.ChildDigest); ok { + layer.Short = digest.Short() + layer.Stored = s.dockerBlobStored(repository, digest) + } + + layers = append(layers, layer) + } + return layers, nil +} + +// dockerBlobStored separates a layer this server holds from one it only knows +// about, which is the difference between a foreign layer and a broken one. +func (s *Server) dockerBlobStored(repository *models.Repository, digest docker.Digest) bool { + _, err := s.store.FindAsset(repository.ID, docker.BlobPath(digest)) + return err == nil +} + +func (s *Server) describeDockerChildren(repository *models.Repository, references []models.DockerReference) ([]dockerChildResponse, error) { + digests := make([]string, 0, len(references)) + for _, reference := range references { + if reference.Kind == models.DockerReferenceManifest { + digests = append(digests, reference.ChildDigest) + } + } + + indexed, err := s.store.DockerManifestsByDigest(repository.ID, digests) + if err != nil { + return nil, err + } + + children := make([]dockerChildResponse, 0, len(digests)) + for _, reference := range references { + if reference.Kind != models.DockerReferenceManifest { + continue + } + + child := dockerChildResponse{ + Digest: reference.ChildDigest, + MediaType: reference.MediaType, + Platform: reference.Platform, + Size: reference.Size, + ReferenceType: decodeJSONMap(reference.Annotations)[docker.AnnotationReferenceType], + } + if digest, ok := docker.ParseDigest(reference.ChildDigest); ok { + child.Short = digest.Short() + } + + // A child that has not been indexed is normal while an index is still being + // pushed, and permanent in a partially migrated repository, so it is + // reported as unindexed rather than left out. + if record, ok := indexed[reference.ChildDigest]; ok { + child.Indexed = true + child.TotalSize = record.TotalSize + child.OS = record.OS + child.Architecture = record.Architecture + child.Variant = record.Variant + child.LayerCount = record.LayerCount + } + + children = append(children, child) + } + return children, nil +} + +// describeDockerConfig reads the build-time settings out of the config blob. It is +// read here rather than stored, because it is small, already local, and duplicating +// it into a column would only give it a second chance to go stale. +// backfillDockerPlatform repairs a record indexed before its config blob arrived. It +// writes during a read, which is worth it: the alternative is a permanently blank +// platform on every proxied image, and the value is derived rather than authored so +// recomputing it costs nothing but the one update. +func (s *Server) backfillDockerPlatform(repository *models.Repository, record *models.DockerManifest, config docker.Config, response *dockerManifestResponse) { + if record.Architecture != "" || config.Architecture == "" { + return + } + + record.Architecture = config.Architecture + record.OS = config.OS + record.Variant = config.Variant + record.ImageCreated = config.Created + record.Labels = encodeJSONMap(config.Labels) + + response.Architecture = record.Architecture + response.OS = record.OS + response.Variant = record.Variant + response.ImageCreated = record.ImageCreated + response.Labels = config.Labels + + if err := s.store.UpdateDockerManifestPlatform(record); err != nil { + log.Warnf("recording the platform of %s/%s failed: %v", repository.Name, record.Digest, err) + } +} + +// backfillDockerIndexTime is the same repair one level up. An index has no config, so +// it takes its build time from its children, and on a proxy those are fetched after the +// index itself and so were not indexed when it was. +func (s *Server) backfillDockerIndexTime(repository *models.Repository, record *models.DockerManifest, response *dockerManifestResponse) { + if record.ImageCreated != 0 { + return + } + + var newest int64 + for _, child := range response.Children { + if indexed, err := s.store.DockerManifest(repository.ID, child.Digest); err == nil { + newest = max(newest, indexed.ImageCreated) + } + } + if newest == 0 { + return + } + + record.ImageCreated = newest + response.ImageCreated = newest + + if err := s.store.UpdateDockerManifestPlatform(record); err != nil { + log.Warnf("recording the build time of %s/%s failed: %v", repository.Name, record.Digest, err) + } +} + +func (s *Server) describeDockerConfig(repository *models.Repository, references []models.DockerReference) (*dockerConfigResponse, *docker.Config) { + var entry models.DockerReference + for _, reference := range references { + if reference.Kind == models.DockerReferenceConfig { + entry = reference + break + } + } + if entry.ChildDigest == "" { + return nil, nil + } + + digest, ok := docker.ParseDigest(entry.ChildDigest) + if !ok { + return nil, nil + } + + asset, err := s.store.FindAsset(repository.ID, docker.BlobPath(digest)) + if err != nil { + return nil, nil + } + + file, _, err := s.blobs.Open(asset.StorageKey) + if err != nil { + return nil, nil + } + defer file.Close() + + document, err := io.ReadAll(io.LimitReader(file, maxManifestBytes)) + if err != nil { + return nil, nil + } + + config, err := docker.ParseConfig(document) + if err != nil { + return nil, nil + } + + response := &dockerConfigResponse{ + Digest: entry.ChildDigest, + Size: entry.Size, + User: config.User, + WorkingDir: config.WorkingDir, + Entrypoint: orEmpty(config.Entrypoint), + Cmd: orEmpty(config.Cmd), + Env: orEmpty(config.Env), + ExposedPorts: orEmpty(config.ExposedPorts), + History: []dockerHistoryResponse{}, + } + for _, step := range config.History { + response.History = append(response.History, dockerHistoryResponse{ + Created: step.Created, + CreatedBy: step.CreatedBy, + Comment: step.Comment, + EmptyLayer: step.EmptyLayer, + }) + } + return response, &config +} + +func decodeJSONMap(encoded string) map[string]string { + values := map[string]string{} + if encoded == "" { + return values + } + if err := json.Unmarshal([]byte(encoded), &values); err != nil { + return map[string]string{} + } + return values +} + +func splitLines(value string) []string { + if value == "" { + return []string{} + } + return strings.Split(value, "\n") +} + +// orEmpty keeps a JSON array out of being null, which the UI would have to guard +// every read against. +func orEmpty(values []string) []string { + if values == nil { + return []string{} + } + return values +} diff --git a/internal/server/api_maintenance.go b/internal/server/api_maintenance.go index db9d3b0..d3f16c6 100644 --- a/internal/server/api_maintenance.go +++ b/internal/server/api_maintenance.go @@ -44,7 +44,7 @@ func (s *Server) plan(ctx context.Context, body migrationRequest) ([]migrate.Dec } } if body.RawFormat != "" { - if _, err := requireOneOf(body.RawFormat, formats, "rawFormat"); err != nil { + if _, err := requireOneOf(body.RawFormat, rawFormats, "rawFormat"); err != nil { return nil, err } } @@ -176,6 +176,53 @@ func (s *Server) runMigration(ctx context.Context, cancel context.CancelFunc, ru } } +// routeDockerSweepPreview reports what a sweep would reclaim without touching +// anything, so an administrator can see the number before agreeing to it. +func (s *Server) routeDockerSweepPreview(writer http.ResponseWriter, request *http.Request) error { + if _, err := requireAdmin(request); err != nil { + return err + } + + results, err := s.sweepDockerRepositories(false) + if err != nil { + return err + } + + writeJSON(writer, http.StatusOK, map[string]any{"repositories": results}) + return nil +} + +func (s *Server) routeDockerSweep(writer http.ResponseWriter, request *http.Request) error { + if _, err := requireAdmin(request); err != nil { + return err + } + + results, err := s.sweepDockerRepositories(true) + if err != nil { + return err + } + + writeJSON(writer, http.StatusOK, map[string]any{"repositories": results}) + return nil +} + +// routeDockerReindex rebuilds the parsed metadata of every docker repository. It +// repairs a migration that was interrupted, or one run before this server knew how to +// index, without recopying a byte. +func (s *Server) routeDockerReindex(writer http.ResponseWriter, request *http.Request) error { + if _, err := requireAdmin(request); err != nil { + return err + } + + results, err := s.reindexDockerRepositories() + if err != nil { + return err + } + + writeJSON(writer, http.StatusOK, map[string]any{"repositories": results}) + return nil +} + func (s *Server) routeCancelMigration(writer http.ResponseWriter, request *http.Request) error { if _, err := requireAdmin(request); err != nil { return err diff --git a/internal/server/api_repositories.go b/internal/server/api_repositories.go index 467b937..60087fe 100644 --- a/internal/server/api_repositories.go +++ b/internal/server/api_repositories.go @@ -21,7 +21,11 @@ var ( visibilities = []string{models.VisibilityPublic, models.VisibilityPrivate} permissions = []string{models.PermissionRead, models.PermissionWrite, models.PermissionAdmin} repositoryTypes = []string{models.TypeHosted, models.TypeProxy, models.TypeGroup} - formats = []string{format.Maven2, format.NPM, format.P2} + formats = []string{format.Maven2, format.NPM, format.P2, format.Docker} + // rawFormats are what a Nexus raw repository may be migrated into. Docker is + // absent on purpose: raw holds loose files, while docker content only means + // anything at the paths the registry API defines, so the mapping cannot exist. + rawFormats = []string{format.Maven2, format.NPM, format.P2} ) type repositoryResponse struct { diff --git a/internal/server/docker.go b/internal/server/docker.go new file mode 100644 index 0000000..7fc392f --- /dev/null +++ b/internal/server/docker.go @@ -0,0 +1,322 @@ +package server + +import ( + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/charmbracelet/log" + + "arca/internal/docker" + "arca/internal/format" + "arca/internal/store" + "arca/internal/store/models" +) + +const ( + maxManifestBytes = 4 << 20 + blobContentType = "application/octet-stream" +) + +// dockerBlobLimit translates the configured ceiling into what storeUpload expects, +// where a negative limit means unlimited and zero would fall back to the much +// smaller artifact limit. +func (s *Server) dockerBlobLimit() int64 { + if s.config.MaxBlobBytes > 0 { + return s.config.MaxBlobBytes + } + return -1 +} + +func dockerError(writer http.ResponseWriter, status int, code, message string) { + writeJSON(writer, status, map[string]any{ + "errors": []map[string]any{{"code": code, "message": message, "detail": nil}}, + }) +} + +func (s *Server) dockerChallenge(writer http.ResponseWriter, request *http.Request, message string) { + if currentUser(request) != nil { + dockerError(writer, http.StatusForbidden, docker.ErrorDenied, message) + return + } + writer.Header().Set("WWW-Authenticate", fmt.Sprintf("Basic realm=%q", s.store.InstanceName())) + dockerError(writer, http.StatusUnauthorized, docker.ErrorUnauthorized, message) +} + +// registryRequest is one resolved registry call. Route.Name is kept alongside the +// image because every Location header has to echo the name the client used, which +// carries the repository prefix that the image name on its own has lost. +type registryRequest struct { + repository *models.Repository + image string + route docker.Route +} + +func (r registryRequest) namespace() (string, string, bool) { return docker.SplitImage(r.image) } + +func (r registryRequest) uploadLocation(id string) string { + return "/v2/" + r.route.Name + "/blobs/uploads/" + id +} + +// handleDockerRegistry serves /v2 at the host root, because a Docker client +// derives the registry from the image reference and has no way to be pointed at a +// path prefix the way Maven and npm clients do. +func (s *Server) handleDockerRegistry(writer http.ResponseWriter, request *http.Request) { + writer.Header().Set(docker.APIVersionHeader, docker.APIVersion) + + route := docker.Resolve(request.Method, strings.TrimPrefix(request.URL.Path, "/v2")) + + switch route.Kind { + case docker.RouteBase: + s.serveDockerBase(writer, request) + return + case docker.RouteCatalog: + s.serveDockerCatalog(writer, request) + return + case docker.RouteUnknown: + dockerError(writer, http.StatusNotFound, docker.ErrorNameInvalid, "Not found") + return + } + + repository, image, ok := s.resolveDockerRepository(route.Name) + if !ok { + dockerError(writer, http.StatusNotFound, docker.ErrorNameUnknown, + fmt.Sprintf("%q does not name a repository and an image on this server", route.Name)) + return + } + + required := models.PermissionRead + switch request.Method { + case http.MethodPost, http.MethodPut, http.MethodPatch, http.MethodDelete: + required = models.PermissionWrite + } + if !s.authorizeDocker(writer, request, repository, required) { + return + } + if required != models.PermissionRead && (repository.IsProxy() || repository.IsGroup()) { + dockerError(writer, http.StatusMethodNotAllowed, docker.ErrorUnsupported, "This repository is read-only") + return + } + + registry := registryRequest{repository: repository, image: image, route: route} + + switch route.Kind { + case docker.RouteTags: + s.serveDockerTags(writer, request, registry) + case docker.RouteManifest: + s.serveDockerManifest(writer, request, registry) + case docker.RouteBlob: + s.serveDockerBlob(writer, request, registry) + case docker.RouteUploadStart: + s.serveDockerUploadStart(writer, request, registry) + case docker.RouteUpload: + s.serveDockerUploadSession(writer, request, registry) + case docker.RouteReferrers: + // Unimplemented on purpose. A 404 is the spec's own signal for that and + // it sends clients to the fallback tag scheme. + dockerError(writer, http.StatusNotFound, docker.ErrorUnsupported, "The referrers API is not implemented") + default: + dockerError(writer, http.StatusNotFound, docker.ErrorNameUnknown, "Not found") + } +} + +func (s *Server) authorizeDocker(writer http.ResponseWriter, request *http.Request, repository *models.Repository, required string) bool { + permission, err := s.store.EffectivePermission(repository, currentUser(request)) + if err != nil { + log.Errorf("resolving repository permission failed: %v", err) + dockerError(writer, http.StatusInternalServerError, docker.ErrorUnsupported, "Internal server error") + return false + } + if store.Satisfies(permission, required) { + return true + } + + s.dockerChallenge(writer, request, "You do not have permission to do that") + return false +} + +// resolveDockerRepository splits an image name into the repository serving it and +// the image inside it. The leading segment wins when it names a docker +// repository, so "docker-hosted/team/api" is the team/api image of docker-hosted. +// Otherwise the whole name resolves against the instance default, which is what +// makes a bare "docker pull host/nginx" work on a single-repository install. +func (s *Server) resolveDockerRepository(name string) (*models.Repository, string, bool) { + if prefix, rest, found := strings.Cut(name, "/"); found && rest != "" { + repository, err := s.store.RepositoryByNameAndFormat(prefix, format.Docker) + if err == nil && docker.IsValidName(rest) { + return repository, rest, true + } + } + + fallback, err := s.store.Setting(store.SettingDockerRepository) + if err != nil || fallback == "" { + return nil, "", false + } + + repository, err := s.store.RepositoryByNameAndFormat(fallback, format.Docker) + if err != nil { + return nil, "", false + } + return repository, name, true +} + +// serveDockerBase answers the version check. It is the one endpoint that names no +// repository, so it cannot consult a grant: it succeeds for anyone who +// authenticated, and for anonymous clients only when some docker repository is +// public. Answering 401 with a challenge is also what makes docker login work, +// since that command has nothing else to post to. +func (s *Server) serveDockerBase(writer http.ResponseWriter, request *http.Request) { + if request.Method != http.MethodGet && request.Method != http.MethodHead { + dockerError(writer, http.StatusMethodNotAllowed, docker.ErrorUnsupported, "Method not allowed") + return + } + + if currentUser(request) != nil { + writeJSON(writer, http.StatusOK, map[string]any{}) + return + } + + public, err := s.store.HasPublicRepositories(format.Docker) + if err != nil { + log.Errorf("checking for public docker repositories failed: %v", err) + dockerError(writer, http.StatusInternalServerError, docker.ErrorUnsupported, "Internal server error") + return + } + if public { + writeJSON(writer, http.StatusOK, map[string]any{}) + return + } + + s.dockerChallenge(writer, request, "Authentication required") +} + +// serveDockerCatalog lists every image the caller can read, each prefixed with +// the repository that serves it. The spec assumes one registry per host, so the +// prefix is an addition rather than an omission: without it a client could not +// turn a catalog entry back into something it can pull. +func (s *Server) serveDockerCatalog(writer http.ResponseWriter, request *http.Request) { + if request.Method != http.MethodGet { + dockerError(writer, http.StatusMethodNotAllowed, docker.ErrorUnsupported, "Method not allowed") + return + } + + repositories, err := s.store.VisibleRepositories(currentUser(request)) + if err != nil { + log.Errorf("listing repositories for the docker catalog failed: %v", err) + dockerError(writer, http.StatusInternalServerError, docker.ErrorUnsupported, "Internal server error") + return + } + + names := []string{} + for _, repository := range repositories { + if repository.Format != format.Docker { + continue + } + + images, err := s.store.DockerImages(repository.ID) + if err != nil { + log.Errorf("listing images of %s failed: %v", repository.Name, err) + continue + } + for _, image := range images { + names = append(names, repository.Name+"/"+image) + } + } + + writeJSON(writer, http.StatusOK, map[string]any{"repositories": names}) +} + +func (s *Server) serveDockerTags(writer http.ResponseWriter, request *http.Request, registry registryRequest) { + if request.Method != http.MethodGet && request.Method != http.MethodHead { + dockerError(writer, http.StatusMethodNotAllowed, docker.ErrorUnsupported, "Method not allowed") + return + } + + namespace, name, ok := registry.namespace() + if !ok { + dockerError(writer, http.StatusNotFound, docker.ErrorNameInvalid, "Not found") + return + } + + components, err := s.store.ComponentVersions(registry.repository.ID, namespace, name) + if err != nil { + log.Errorf("listing tags failed: %v", err) + dockerError(writer, http.StatusInternalServerError, docker.ErrorUnsupported, "Internal server error") + return + } + if len(components) == 0 { + dockerError(writer, http.StatusNotFound, docker.ErrorNameUnknown, "Not found") + return + } + + tags := make([]string, 0, len(components)) + for _, component := range components { + tags = append(tags, component.Version) + } + + writeJSON(writer, http.StatusOK, map[string]any{ + "name": registry.route.Name, + // Sorting through the format's own comparator is what makes the spec's + // "last" parameter stable across requests. + "tags": paginateTags(format.SortVersions(docker.Layout{}, tags), request), + }) +} + +// serveDockerBrowse answers /repository//. A docker repository is pulled +// from at /v2 instead, so this exists only so that following the URL the rest of +// the UI shows lands on the stored tree rather than on a Maven handler. +func (s *Server) serveDockerBrowse(writer http.ResponseWriter, request *http.Request, repository *models.Repository, path string) { + if request.Method != http.MethodGet && request.Method != http.MethodHead { + plain(writer, http.StatusMethodNotAllowed, "Push to /v2/"+repository.Name+"/ instead") + return + } + if !s.authorize(writer, request, repository, models.PermissionRead) { + return + } + + if path == "" || strings.HasSuffix(path, "/") { + exists, err := s.store.DirectoryExists(repository.ID, path) + if err != nil { + log.Errorf("directory lookup failed: %v", err) + plain(writer, http.StatusInternalServerError, "Internal Server Error") + return + } + if !exists { + plain(writer, http.StatusNotFound, "Not Found") + return + } + s.writeDirectoryIndex(writer, repository, path) + return + } + + asset, err := s.store.FindAsset(repository.ID, path) + if err != nil { + plain(writer, http.StatusNotFound, "Not Found") + return + } + if request.Method == http.MethodGet { + s.recordAssetTraffic(request, models.TrafficDownload, repository, asset) + } + if !s.writeAsset(writer, request, asset) { + plain(writer, http.StatusNotFound, "Not Found") + } +} + +func paginateTags(tags []string, request *http.Request) []string { + query := request.URL.Query() + + if last := query.Get("last"); last != "" { + for index, tag := range tags { + if tag == last { + tags = tags[index+1:] + break + } + } + } + + if limit, err := strconv.Atoi(query.Get("n")); err == nil && limit >= 0 && len(tags) > limit { + tags = tags[:limit] + } + return tags +} diff --git a/internal/server/docker_blob.go b/internal/server/docker_blob.go new file mode 100644 index 0000000..fd5724a --- /dev/null +++ b/internal/server/docker_blob.go @@ -0,0 +1,471 @@ +package server + +import ( + "errors" + "net/http" + "strconv" + "strings" + "time" + + "github.com/charmbracelet/log" + + "arca/internal/blob" + "arca/internal/docker" + "arca/internal/store" + "arca/internal/store/models" +) + +// staleUploadWindow is how long an upload session survives without a chunk. A +// push of a large layer over a slow link is one long PATCH rather than a pause, +// so a day is generous without letting an abandoned session linger. +const staleUploadWindow = 24 * time.Hour + +func (s *Server) serveDockerBlob(writer http.ResponseWriter, request *http.Request, registry registryRequest) { + switch request.Method { + case http.MethodGet, http.MethodHead: + s.serveDockerBlobRead(writer, request, registry) + case http.MethodDelete: + s.serveDockerBlobDelete(writer, registry) + default: + dockerError(writer, http.StatusMethodNotAllowed, docker.ErrorUnsupported, "Method not allowed") + } +} + +func (s *Server) serveDockerBlobRead(writer http.ResponseWriter, request *http.Request, registry registryRequest) { + if registry.repository.IsProxy() { + s.serveDockerProxyBlob(writer, request, registry) + return + } + + asset, err := s.store.FindAsset(registry.repository.ID, docker.BlobPath(registry.route.Digest)) + if err != nil { + dockerError(writer, http.StatusNotFound, docker.ErrorBlobUnknown, "Not found") + return + } + + writer.Header().Set(docker.ContentDigestHeader, registry.route.Digest.String()) + + // A blob request names the image but not the tag, so traffic is recorded + // against the image with no version rather than guessed at. + if request.Method == http.MethodGet { + s.recordDockerTraffic(request, registry, models.TrafficDownload, asset.Size) + } + if !s.writeAsset(writer, request, asset) { + dockerError(writer, http.StatusNotFound, docker.ErrorBlobUnknown, "Not found") + } +} + +func (s *Server) serveDockerBlobDelete(writer http.ResponseWriter, registry registryRequest) { + keys, err := s.store.DeleteAsset(registry.repository.ID, docker.BlobPath(registry.route.Digest)) + if err != nil { + log.Errorf("deleting a docker blob failed: %v", err) + dockerError(writer, http.StatusInternalServerError, docker.ErrorUnsupported, "Internal server error") + return + } + if len(keys) == 0 { + dockerError(writer, http.StatusNotFound, docker.ErrorBlobUnknown, "Not found") + return + } + if err := s.blobs.Delete(keys...); err != nil { + log.Errorf("removing a deleted docker blob failed: %v", err) + } + + writer.WriteHeader(http.StatusAccepted) +} + +// serveDockerUploadStart opens a push. The same endpoint covers three shapes: a +// mount of a blob this server already holds, a whole blob in the body, and the +// session a chunked push appends to. +func (s *Server) serveDockerUploadStart(writer http.ResponseWriter, request *http.Request, registry registryRequest) { + if request.Method != http.MethodPost { + dockerError(writer, http.StatusMethodNotAllowed, docker.ErrorUnsupported, "Method not allowed") + return + } + + query := request.URL.Query() + + if mount := query.Get("mount"); mount != "" { + if s.mountDockerBlob(writer, request, registry, mount, query.Get("from")) { + return + } + // A mount that cannot be served falls through to an ordinary session, + // which is the fallback the spec prescribes. + } + + if digest := query.Get("digest"); digest != "" { + s.completeDockerMonolith(writer, request, registry, digest) + return + } + + id := store.NewID() + if err := s.blobs.BeginUpload(id); err != nil { + log.Errorf("opening a docker upload session failed: %v", err) + dockerError(writer, http.StatusInternalServerError, docker.ErrorUnsupported, "Internal server error") + return + } + + var userID *string + if user := currentUser(request); user != nil { + userID = &user.ID + } + upload := &models.DockerUpload{ + ID: id, + RepositoryID: registry.repository.ID, + Image: registry.image, + UserID: userID, + } + if err := s.store.CreateDockerUpload(upload); err != nil { + _ = s.blobs.AbortUpload(id) + log.Errorf("recording a docker upload session failed: %v", err) + dockerError(writer, http.StatusInternalServerError, docker.ErrorUnsupported, "Internal server error") + return + } + + writeDockerUploadProgress(writer, registry, id, 0, http.StatusAccepted) +} + +// mountDockerBlob reports whether it answered. Blob keys are scoped per +// repository, so the bytes are copied rather than linked, which still saves the +// client the upload and keeps per-repository storage accounting truthful. +func (s *Server) mountDockerBlob(writer http.ResponseWriter, request *http.Request, registry registryRequest, mount, from string) bool { + digest, ok := docker.ParseDigest(mount) + if !ok { + return false + } + + source := registry.repository + if from != "" && from != registry.route.Name { + found, err := s.dockerMountSource(request, from) + if err != nil { + return false + } + source = found + } + + existing, err := s.store.FindAsset(source.ID, docker.BlobPath(digest)) + if err != nil { + return false + } + + file, _, err := s.blobs.Open(existing.StorageKey) + if err != nil { + return false + } + defer file.Close() + + var uploadedBy *string + if user := currentUser(request); user != nil { + uploadedBy = &user.ID + } + + stored, err := s.storeUpload(registry.repository, docker.BlobPath(digest), file, + uploadDetails{UploadedBy: uploadedBy, ContentType: existing.ContentType, Limit: s.dockerBlobLimit()}) + if err != nil { + log.Errorf("mounting a docker blob failed: %v", err) + return false + } + if stored.SHA256 != digest.Hex { + log.Warnf("a mounted blob of %s hashed to %s rather than %s", source.Name, stored.SHA256, digest.Hex) + if _, err := s.store.DeleteAsset(registry.repository.ID, stored.Path); err != nil { + log.Errorf("dropping a mismatched mounted blob failed: %v", err) + } + return false + } + + writeDockerBlobCreated(writer, registry, digest) + return true +} + +// dockerMountSource resolves the repository a cross-repository mount reads from, +// which the caller must be able to read in its own right. +func (s *Server) dockerMountSource(request *http.Request, from string) (*models.Repository, error) { + repository, _, ok := s.resolveDockerRepository(from) + if !ok { + return nil, errors.New("server: the mount source names no repository") + } + + permission, err := s.store.EffectivePermission(repository, currentUser(request)) + if err != nil { + return nil, err + } + if !store.Satisfies(permission, models.PermissionRead) { + return nil, errors.New("server: the mount source is not readable by this user") + } + return repository, nil +} + +// completeDockerMonolith stores a blob whose whole body arrived with the POST or +// PUT that named its digest. +func (s *Server) completeDockerMonolith(writer http.ResponseWriter, request *http.Request, registry registryRequest, raw string) { + digest, ok := docker.ParseDigest(raw) + if !ok { + dockerError(writer, http.StatusBadRequest, docker.ErrorDigestInvalid, raw+" is not a digest this server stores") + return + } + + var uploadedBy *string + if user := currentUser(request); user != nil { + uploadedBy = &user.ID + } + + path := docker.BlobPath(digest) + stored, err := s.storeUpload(registry.repository, path, request.Body, + uploadDetails{UploadedBy: uploadedBy, ContentType: blobContentType, Limit: s.dockerBlobLimit()}) + if errors.Is(err, blob.ErrTooLarge) { + dockerError(writer, http.StatusRequestEntityTooLarge, docker.ErrorSizeInvalid, "The blob exceeds the upload limit") + return + } + if err != nil { + log.Errorf("storing a docker blob failed: %v", err) + dockerError(writer, http.StatusInternalServerError, docker.ErrorUnsupported, "Internal server error") + return + } + + if stored.SHA256 != digest.Hex { + if _, deleteErr := s.store.DeleteAsset(registry.repository.ID, path); deleteErr != nil { + log.Errorf("dropping a mismatched docker blob failed: %v", deleteErr) + } + _ = s.blobs.Delete(stored.StorageKey) + dockerError(writer, http.StatusBadRequest, docker.ErrorDigestInvalid, + "The blob hashed to sha256:"+stored.SHA256+" rather than "+digest.String()) + return + } + + s.recordDockerTraffic(request, registry, models.TrafficUpload, stored.Size) + writeDockerBlobCreated(writer, registry, digest) +} + +func (s *Server) serveDockerUploadSession(writer http.ResponseWriter, request *http.Request, registry registryRequest) { + id := registry.route.Upload + + if _, err := s.store.DockerUpload(registry.repository.ID, id); err != nil { + dockerError(writer, http.StatusNotFound, docker.ErrorBlobUploadUnknown, "Not found") + return + } + + switch request.Method { + case http.MethodGet, http.MethodHead: + s.serveDockerUploadStatus(writer, registry, id) + case http.MethodPatch: + s.serveDockerUploadChunk(writer, request, registry, id) + case http.MethodPut: + s.serveDockerUploadFinish(writer, request, registry, id) + case http.MethodDelete: + s.serveDockerUploadAbort(writer, registry, id) + default: + dockerError(writer, http.StatusMethodNotAllowed, docker.ErrorUnsupported, "Method not allowed") + } +} + +func (s *Server) serveDockerUploadStatus(writer http.ResponseWriter, registry registryRequest, id string) { + size, err := s.blobs.UploadSize(id) + if err != nil { + dockerError(writer, http.StatusNotFound, docker.ErrorBlobUploadUnknown, "Not found") + return + } + writeDockerUploadProgress(writer, registry, id, size, http.StatusNoContent) +} + +func (s *Server) serveDockerUploadChunk(writer http.ResponseWriter, request *http.Request, registry registryRequest, id string) { + current, err := s.blobs.UploadSize(id) + if err != nil { + dockerError(writer, http.StatusNotFound, docker.ErrorBlobUploadUnknown, "Not found") + return + } + + // A chunk that does not start where the session ended would silently corrupt + // the blob, so it is refused with the offset the client should resume from. + if start, ok := parseContentRangeStart(request.Header.Get("Content-Range")); ok && start != current { + writer.Header().Set("Range", "0-"+strconv.FormatInt(current-1, 10)) + dockerError(writer, http.StatusRequestedRangeNotSatisfiable, docker.ErrorBlobUploadInvalid, + "The chunk does not continue from the end of the session") + return + } + + size, err := s.blobs.AppendUpload(id, request.Body, s.config.MaxBlobBytes) + if errors.Is(err, blob.ErrTooLarge) { + dockerError(writer, http.StatusRequestEntityTooLarge, docker.ErrorSizeInvalid, "The blob exceeds the upload limit") + return + } + if err != nil { + log.Errorf("appending to a docker upload session failed: %v", err) + dockerError(writer, http.StatusInternalServerError, docker.ErrorUnsupported, "Internal server error") + return + } + + if err := s.store.SetDockerUploadSize(id, size); err != nil { + log.Errorf("recording docker upload progress failed: %v", err) + } + + writeDockerUploadProgress(writer, registry, id, size, http.StatusAccepted) +} + +func (s *Server) serveDockerUploadFinish(writer http.ResponseWriter, request *http.Request, registry registryRequest, id string) { + digest, ok := docker.ParseDigest(request.URL.Query().Get("digest")) + if !ok { + dockerError(writer, http.StatusBadRequest, docker.ErrorDigestInvalid, "The upload was committed without a valid digest") + return + } + + // The final PUT may carry the last chunk, and for a small blob it carries the + // whole of it. + if _, err := s.blobs.AppendUpload(id, request.Body, s.config.MaxBlobBytes); errors.Is(err, blob.ErrTooLarge) { + dockerError(writer, http.StatusRequestEntityTooLarge, docker.ErrorSizeInvalid, "The blob exceeds the upload limit") + return + } else if err != nil { + log.Errorf("appending the final docker chunk failed: %v", err) + dockerError(writer, http.StatusInternalServerError, docker.ErrorUnsupported, "Internal server error") + return + } + + path := docker.BlobPath(digest) + key := registry.repository.ID + "/" + path + + size, digests, err := s.blobs.CompleteUpload(id, key, digest.Hex) + if errors.Is(err, blob.ErrDigestMismatch) { + dockerError(writer, http.StatusBadRequest, docker.ErrorDigestInvalid, + "The upload hashed to sha256:"+digests.SHA256+" rather than "+digest.String()) + return + } + if err != nil { + log.Errorf("committing a docker upload failed: %v", err) + dockerError(writer, http.StatusInternalServerError, docker.ErrorUnsupported, "Internal server error") + return + } + + var uploadedBy *string + if user := currentUser(request); user != nil { + uploadedBy = &user.ID + } + + // The bytes are already in place, so the asset row is written directly rather + // than through storeUpload, which would want to copy them a second time. + asset := &models.Asset{ + RepositoryID: registry.repository.ID, + Path: path, + StorageKey: key, + Size: size, + ContentType: blobContentType, + MD5: digests.MD5, + SHA1: digests.SHA1, + SHA256: digests.SHA256, + SHA512: digests.SHA512, + UploadedBy: uploadedBy, + } + if err := s.store.UpsertAsset(asset); err != nil { + log.Errorf("recording a docker blob failed: %v", err) + dockerError(writer, http.StatusInternalServerError, docker.ErrorUnsupported, "Internal server error") + return + } + if err := s.store.DeleteDockerUpload(id); err != nil { + log.Errorf("closing a docker upload session failed: %v", err) + } + + s.recordDockerTraffic(request, registry, models.TrafficUpload, size) + writeDockerBlobCreated(writer, registry, digest) +} + +func (s *Server) serveDockerUploadAbort(writer http.ResponseWriter, registry registryRequest, id string) { + if err := s.blobs.AbortUpload(id); err != nil { + log.Errorf("discarding a docker upload session failed: %v", err) + } + if err := s.store.DeleteDockerUpload(id); err != nil { + log.Errorf("closing a docker upload session failed: %v", err) + dockerError(writer, http.StatusInternalServerError, docker.ErrorUnsupported, "Internal server error") + return + } + writer.WriteHeader(http.StatusNoContent) +} + +func writeDockerUploadProgress(writer http.ResponseWriter, registry registryRequest, id string, size int64, status int) { + header := writer.Header() + header.Set("Location", registry.uploadLocation(id)) + header.Set("Docker-Upload-UUID", id) + // An empty session reports 0-0 rather than 0--1, which is what the spec's + // inclusive range would otherwise produce. + if size == 0 { + header.Set("Range", "0-0") + } else { + header.Set("Range", "0-"+strconv.FormatInt(size-1, 10)) + } + header.Set("Content-Length", "0") + writer.WriteHeader(status) +} + +func writeDockerBlobCreated(writer http.ResponseWriter, registry registryRequest, digest docker.Digest) { + header := writer.Header() + header.Set("Location", "/v2/"+registry.route.Name+"/blobs/"+digest.String()) + header.Set(docker.ContentDigestHeader, digest.String()) + header.Set("Content-Length", "0") + writer.WriteHeader(http.StatusCreated) +} + +// parseContentRangeStart reads the offset a chunk claims to begin at. The registry +// API uses a bare "start-end" here rather than the "bytes start-end/total" form +// that HTTP defines, so this is deliberately narrow. +func parseContentRangeStart(value string) (int64, bool) { + trimmed := strings.TrimSpace(value) + if trimmed == "" { + return 0, false + } + + start, _, found := strings.Cut(trimmed, "-") + if !found { + return 0, false + } + + offset, err := strconv.ParseInt(strings.TrimSpace(start), 10, 64) + if err != nil || offset < 0 { + return 0, false + } + return offset, true +} + +// recordDockerTraffic attributes a blob transfer to the image. A blob request +// names no tag, so the version is left empty rather than guessed at. +func (s *Server) recordDockerTraffic(request *http.Request, registry registryRequest, kind string, bytes int64) { + namespace, name, ok := registry.namespace() + if !ok { + return + } + + var userID *string + if user := currentUser(request); user != nil { + userID = &user.ID + } + + s.traffic.record(models.TrafficEvent{ + Kind: kind, + RepositoryID: registry.repository.ID, + Namespace: namespace, + Name: name, + Bytes: bytes, + UserID: userID, + }) +} + +// purgeStaleDockerUploads drops sessions a client walked away from. The file goes +// first: a row without a file reports a length nothing can satisfy, while a file +// without a row is unreachable and would never be swept again. +func (s *Server) purgeStaleDockerUploads(now time.Time) { + ids, err := s.store.StaleDockerUploads(now.Add(-staleUploadWindow).UnixMilli()) + if err != nil { + log.Errorf("listing stale docker uploads failed: %v", err) + return + } + if len(ids) == 0 { + return + } + + for _, id := range ids { + if err := s.blobs.AbortUpload(id); err != nil { + log.Errorf("discarding the stale docker upload %s failed: %v", id, err) + } + } + if err := s.store.DeleteDockerUploads(ids); err != nil { + log.Errorf("removing stale docker upload records failed: %v", err) + return + } + + log.Infof("discarded %d abandoned docker upload sessions", len(ids)) +} diff --git a/internal/server/docker_manifest.go b/internal/server/docker_manifest.go new file mode 100644 index 0000000..701e1c5 --- /dev/null +++ b/internal/server/docker_manifest.go @@ -0,0 +1,359 @@ +package server + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "io" + "net/http" + "strings" + + "github.com/charmbracelet/log" + + "arca/internal/docker" + "arca/internal/store/models" +) + +func (s *Server) serveDockerManifest(writer http.ResponseWriter, request *http.Request, registry registryRequest) { + switch request.Method { + case http.MethodGet, http.MethodHead: + s.serveDockerManifestRead(writer, request, registry) + case http.MethodPut: + s.serveDockerManifestPut(writer, request, registry) + case http.MethodDelete: + s.serveDockerManifestDelete(writer, registry) + default: + dockerError(writer, http.StatusMethodNotAllowed, docker.ErrorUnsupported, "Method not allowed") + } +} + +// manifestPathFor names where a reference is stored. A digest reads from the +// content-addressed store and a tag from its own directory, which holds the same +// bytes so that a tag pull needs no second lookup. +func manifestPathFor(registry registryRequest) string { + if registry.route.ByDigest { + return docker.ManifestPath(registry.image, registry.route.Digest) + } + return docker.TagPath(registry.image, registry.route.Tag) +} + +func (s *Server) serveDockerManifestRead(writer http.ResponseWriter, request *http.Request, registry registryRequest) { + if registry.repository.IsProxy() { + s.serveDockerProxyManifest(writer, request, registry) + return + } + + asset, err := s.store.FindAsset(registry.repository.ID, manifestPathFor(registry)) + if err != nil { + dockerError(writer, http.StatusNotFound, docker.ErrorManifestUnknown, "Not found") + return + } + + // A client that asked for manifest types and excluded the stored one is told + // the manifest is absent, which is the spec's answer and is what sends it on to + // another reference rather than leaving it to choke on a type it cannot read. + if accept := request.Header.Get("Accept"); !docker.Accepts(accept, asset.ContentType) { + log.Warnf("%s/%s is stored as %s, which %q does not accept", + registry.repository.Name, registry.image, asset.ContentType, accept) + dockerError(writer, http.StatusNotFound, docker.ErrorManifestUnknown, + "This manifest is stored as "+asset.ContentType+", which the request does not accept") + return + } + + writer.Header().Set(docker.ContentDigestHeader, docker.SHA256(asset.SHA256).String()) + + if request.Method == http.MethodGet { + s.recordAssetTraffic(request, models.TrafficDownload, registry.repository, asset) + } + if !s.writeAsset(writer, request, asset) { + dockerError(writer, http.StatusNotFound, docker.ErrorManifestUnknown, "Not found") + } +} + +func (s *Server) serveDockerManifestPut(writer http.ResponseWriter, request *http.Request, registry registryRequest) { + document, err := io.ReadAll(io.LimitReader(request.Body, maxManifestBytes+1)) + if err != nil { + dockerError(writer, http.StatusBadRequest, docker.ErrorManifestInvalid, "The manifest could not be read") + return + } + if len(document) > maxManifestBytes { + dockerError(writer, http.StatusRequestEntityTooLarge, docker.ErrorManifestInvalid, "The manifest is too large") + return + } + + manifest, err := docker.ParseManifest(document) + if err != nil { + dockerError(writer, http.StatusBadRequest, docker.ErrorManifestInvalid, err.Error()) + return + } + + // The digest is of the bytes exactly as they arrived, so it is computed here + // rather than taken from the storage layer, which would see them re-encoded. + sum := sha256.Sum256(document) + digest := docker.SHA256(hex.EncodeToString(sum[:])) + + // A client may PUT by digest, in which case the two have to agree. + if registry.route.ByDigest && registry.route.Digest.String() != digest.String() { + dockerError(writer, http.StatusBadRequest, docker.ErrorDigestInvalid, + "The manifest does not match the digest it was pushed under") + return + } + + if code, message, ok := s.checkDockerManifestBlobs(registry, manifest); !ok { + dockerError(writer, http.StatusNotFound, code, message) + return + } + if !registry.route.ByDigest && !s.checkDockerTagWritable(writer, registry) { + return + } + + mediaType := manifest.MediaType + if declared := request.Header.Get("Content-Type"); docker.IsManifestMediaType(declared) { + mediaType = declared + } + + var uploadedBy *string + if user := currentUser(request); user != nil { + uploadedBy = &user.ID + } + details := uploadDetails{UploadedBy: uploadedBy, ContentType: mediaType} + + stored, err := s.storeUpload(registry.repository, docker.ManifestPath(registry.image, digest), bytes.NewReader(document), details) + if err != nil { + log.Errorf("storing a docker manifest failed: %v", err) + dockerError(writer, http.StatusInternalServerError, docker.ErrorUnsupported, "Internal server error") + return + } + + // The tag copy is what carries the component, so it is written second: a + // failure here leaves the content addressable but untagged, which is a state + // the registry already has to handle. + if !registry.route.ByDigest { + tagged, err := s.storeUpload(registry.repository, docker.TagPath(registry.image, registry.route.Tag), bytes.NewReader(document), details) + if err != nil { + log.Errorf("tagging a docker manifest failed: %v", err) + dockerError(writer, http.StatusInternalServerError, docker.ErrorUnsupported, "Internal server error") + return + } + stored = tagged + } + + if err := s.indexDockerManifest(registry, digest, manifest, int64(len(document))); err != nil { + log.Errorf("indexing a docker manifest failed: %v", err) + } + + s.recordAssetTraffic(request, models.TrafficUpload, registry.repository, stored) + + writer.Header().Set(docker.ContentDigestHeader, digest.String()) + writer.Header().Set("Location", "/v2/"+registry.route.Name+"/manifests/"+registry.route.Reference) + writer.WriteHeader(http.StatusCreated) +} + +// checkDockerManifestBlobs enforces the spec's rule that everything a manifest +// references must already be pushed. Nondistributable layers are exempt: they name +// content a client is expected to fetch from its vendor, so they are never +// uploaded here. +func (s *Server) checkDockerManifestBlobs(registry registryRequest, manifest docker.Manifest) (string, string, bool) { + for _, reference := range manifest.References() { + if docker.IsNondistributable(reference.MediaType) { + continue + } + + digest, ok := docker.ParseDigest(reference.Digest) + if !ok { + return docker.ErrorDigestInvalid, reference.Digest + " is not a digest this server stores", false + } + + path := docker.BlobPath(digest) + code := docker.ErrorManifestBlobUnknown + if reference.Kind == docker.ReferenceManifest { + path = docker.ManifestPath(registry.image, digest) + code = docker.ErrorManifestUnknown + } + + if _, err := s.store.FindAsset(registry.repository.ID, path); err != nil { + return code, reference.Digest + " has not been pushed to this repository", false + } + } + return "", "", true +} + +// checkDockerTagWritable applies the repository's version policy and, for a tag +// that already exists, its redeploy setting. Moving a tag is ordinary Docker +// practice, so a docker repository allows it by default and only an explicitly +// locked one refuses. +func (s *Server) checkDockerTagWritable(writer http.ResponseWriter, registry registryRequest) bool { + tag := registry.route.Tag + + if !registry.repository.AcceptsPolicy(docker.Prerelease(tag)) { + dockerError(writer, http.StatusBadRequest, docker.ErrorTagInvalid, + "Repository policy '"+registry.repository.Policy+"' rejects the tag "+tag) + return false + } + + if registry.repository.AllowRedeploy { + return true + } + if _, err := s.store.FindAsset(registry.repository.ID, docker.TagPath(registry.image, tag)); err == nil { + dockerError(writer, http.StatusConflict, docker.ErrorDenied, + "The tag "+tag+" already exists and this repository does not allow tags to move") + return false + } + return true +} + +// indexDockerManifest records the parsed manifest and everything it references, so +// a version's real size and its layer list can be read without reopening any +// document. It runs after the bytes are stored, because the config blob it reads +// is looked up the same way a pull would. +func (s *Server) indexDockerManifest(registry registryRequest, digest docker.Digest, manifest docker.Manifest, size int64) error { + namespace, name, ok := registry.namespace() + if !ok { + return errors.New("server: the image name is not one that maps onto coordinates") + } + + record := &models.DockerManifest{ + RepositoryID: registry.repository.ID, + Digest: digest.String(), + MediaType: manifest.MediaType, + Size: size, + Namespace: namespace, + Name: name, + ConfigDigest: manifest.Config.Digest, + LayerCount: len(manifest.Layers), + Annotations: encodeJSONMap(manifest.Annotations), + } + if manifest.Subject != nil { + record.Subject = manifest.Subject.Digest + } + + references := make([]models.DockerReference, 0, len(manifest.Layers)+len(manifest.Manifests)+1) + for _, reference := range manifest.References() { + references = append(references, models.DockerReference{ + RepositoryID: registry.repository.ID, + ManifestDigest: digest.String(), + ChildDigest: reference.Digest, + Kind: reference.Kind, + MediaType: reference.MediaType, + Size: reference.Size, + Position: reference.Position, + Platform: reference.Platform, + URLs: strings.Join(reference.URLs, "\n"), + Annotations: encodeJSONMap(reference.Annotations), + }) + } + + if manifest.IsIndex() { + record.TotalSize, record.ImageCreated = s.describeDockerIndex(registry, manifest) + } else { + record.TotalSize = manifest.DeclaredSize() + s.describeDockerImage(registry, manifest, record) + } + + return s.store.SaveDockerManifest(record, references) +} + +// describeDockerIndex sums the children rather than the descriptors, because a +// descriptor's size is the child document's own and says nothing about its layers. A +// child that has not been indexed yet contributes what it declares. +// +// The build time comes from the children too: an index has no config of its own, and +// reporting nothing when every platform knows when it was built is needlessly bare. +func (s *Server) describeDockerIndex(registry registryRequest, manifest docker.Manifest) (int64, int64) { + var total, created int64 + + for _, child := range manifest.Manifests { + indexed, err := s.store.DockerManifest(registry.repository.ID, child.Digest) + if err != nil { + total += child.Size + continue + } + total += indexed.TotalSize + created = max(created, indexed.ImageCreated) + } + return total, created +} + +// describeDockerImage reads the platform and labels out of the config blob. A +// config that was never pushed is normal in a partially migrated repository, so +// its absence leaves the fields empty rather than failing the index. +func (s *Server) describeDockerImage(registry registryRequest, manifest docker.Manifest, record *models.DockerManifest) { + digest, ok := docker.ParseDigest(manifest.Config.Digest) + if !ok { + return + } + + asset, err := s.store.FindAsset(registry.repository.ID, docker.BlobPath(digest)) + if err != nil { + return + } + + file, _, err := s.blobs.Open(asset.StorageKey) + if err != nil { + return + } + defer file.Close() + + document, err := io.ReadAll(io.LimitReader(file, maxManifestBytes)) + if err != nil { + return + } + + config, err := docker.ParseConfig(document) + if err != nil { + log.Warnf("the config blob of %s/%s is unreadable: %v", registry.repository.Name, registry.image, err) + return + } + + record.Architecture = config.Architecture + record.OS = config.OS + record.Variant = config.Variant + record.ImageCreated = config.Created + record.Labels = encodeJSONMap(config.Labels) +} + +func encodeJSONMap(values map[string]string) string { + if len(values) == 0 { + return "" + } + encoded, err := json.Marshal(values) + if err != nil { + return "" + } + return string(encoded) +} + +// serveDockerManifestDelete removes a tag or an untagged manifest. Blobs are left +// alone either way: they are shared, so reclaiming them is the sweep's job and not +// something a single delete can reason about. +func (s *Server) serveDockerManifestDelete(writer http.ResponseWriter, registry registryRequest) { + path := manifestPathFor(registry) + + asset, err := s.store.FindAsset(registry.repository.ID, path) + if err != nil { + dockerError(writer, http.StatusNotFound, docker.ErrorManifestUnknown, "Not found") + return + } + + keys, err := s.store.DeleteAsset(registry.repository.ID, path) + if err != nil { + log.Errorf("deleting a docker manifest failed: %v", err) + dockerError(writer, http.StatusInternalServerError, docker.ErrorUnsupported, "Internal server error") + return + } + if err := s.blobs.Delete(keys...); err != nil { + log.Errorf("removing a deleted docker manifest failed: %v", err) + } + + // Only a digest delete retires the parsed record. Deleting a tag leaves the + // manifest reachable by digest, which is what the spec means by untagging. + if registry.route.ByDigest { + if err := s.store.DeleteDockerManifest(registry.repository.ID, docker.SHA256(asset.SHA256).String()); err != nil { + log.Errorf("deleting a docker manifest record failed: %v", err) + } + } + + writer.WriteHeader(http.StatusAccepted) +} diff --git a/internal/server/docker_proxy.go b/internal/server/docker_proxy.go new file mode 100644 index 0000000..29e8066 --- /dev/null +++ b/internal/server/docker_proxy.go @@ -0,0 +1,156 @@ +package server + +import ( + "io" + "net/http" + "net/url" + "strings" + + "github.com/charmbracelet/log" + + "arca/internal/docker" + "arca/internal/store/models" +) + +// dockerHubHosts are the registries that expect a single-segment image to be +// addressed as library/. Nothing else does, and guessing wrongly turns every +// pull of a private registry's top-level image into a 404, so the rule is keyed on +// the host rather than applied everywhere. +var dockerHubHosts = map[string]bool{ + "registry-1.docker.io": true, + "registry.hub.docker.com": true, + "index.docker.io": true, + "docker.io": true, +} + +// manifestAccept asks for every manifest encoding this server understands. Without +// it Docker Hub answers with a schema 1 manifest, which is deprecated and which the +// parser deliberately rejects. +var manifestAccept = strings.Join([]string{ + docker.MediaTypeOCIIndex, + docker.MediaTypeOCIManifest, + docker.MediaTypeDockerList, + docker.MediaTypeDockerManifest, +}, ", ") + +// remoteRoot is the registry API's own prefix. Every upstream serves under it, which +// is the same reason this server mounts /v2 at its host root: a Docker client builds +// the path itself and there is nowhere else to put it. +const remoteRoot = "v2/" + +// upstreamImage is the name to ask the remote for. It differs from the local name +// only on Docker Hub, whose official images live under an implicit library/ scope. +func upstreamImage(repository *models.Repository, image string) string { + if strings.Contains(image, "/") { + return image + } + + parsed, err := url.Parse(repository.RemoteURL) + if err != nil || !dockerHubHosts[parsed.Host] { + return image + } + return "library/" + image +} + +// dockerManifestFetch names both ends of a cached manifest. A tag is cached under its +// own directory so it carries coordinates and shows up as a version, while a digest +// is cached in the content-addressed store where it can never go stale. +func (s *Server) dockerManifestFetch(registry registryRequest) proxyFetch { + remote := upstreamImage(registry.repository, registry.image) + + fetch := proxyFetch{ + CachePath: manifestPathFor(registry), + RemotePath: remoteRoot + remote + "/manifests/" + registry.route.Reference, + Accept: manifestAccept, + } + fetch.Indexed = func(asset *models.Asset) error { + return s.indexCachedDockerManifest(registry, asset) + } + // A tag request names no digest, so the header can only be filled in once the + // cached row is known. The row's SHA256 is that digest by construction. + fetch.Headers = func(asset *models.Asset) map[string]string { + return map[string]string{docker.ContentDigestHeader: docker.SHA256(asset.SHA256).String()} + } + return fetch +} + +func (s *Server) dockerBlobFetch(registry registryRequest) proxyFetch { + remote := upstreamImage(registry.repository, registry.image) + + digest := registry.route.Digest + + return proxyFetch{ + CachePath: docker.BlobPath(digest), + RemotePath: remoteRoot + remote + "/blobs/" + digest.String(), + Limit: s.dockerBlobLimit(), + Headers: func(*models.Asset) map[string]string { + return map[string]string{docker.ContentDigestHeader: digest.String()} + }, + } +} + +// indexCachedDockerManifest builds the same parsed metadata a push does, so a proxied +// image is as legible in the UI as a hosted one. A manifest whose blobs have not been +// pulled yet still indexes: the layer list is what the manifest says, and the sizes +// come from the document rather than from the files. +func (s *Server) indexCachedDockerManifest(registry registryRequest, asset *models.Asset) error { + file, _, err := s.blobs.Open(asset.StorageKey) + if err != nil { + return err + } + defer file.Close() + + document, err := io.ReadAll(io.LimitReader(file, maxManifestBytes)) + if err != nil { + return err + } + + manifest, err := docker.ParseManifest(document) + if err != nil { + // An upstream that served something unparseable is worth a line in the log, + // but the bytes are cached and servable either way. + log.Warnf("the cached manifest %s/%s is not one this server can parse: %v", + registry.repository.Name, asset.Path, err) + return nil + } + + // The layout could only guess the media type from the filename, so the document's + // own is written over it now that it has been parsed. + if manifest.MediaType != asset.ContentType { + if err := s.store.SetAssetContentType(asset.ID, manifest.MediaType); err != nil { + return err + } + } + + return s.indexDockerManifest(registry, docker.SHA256(asset.SHA256), manifest, int64(len(document))) +} + +// serveDockerProxyManifest answers from the cache, refilling it from the remote when +// the copy is cold or a tag has passed its TTL. Only a tag expires: a digest names +// exactly one document for all time. +func (s *Server) serveDockerProxyManifest(writer http.ResponseWriter, request *http.Request, registry registryRequest) { + if !s.dockerProxyReadable(writer, registry) { + return + } + + s.serveProxyFetch(writer, request, registry.repository, s.dockerManifestFetch(registry)) +} + +func (s *Server) serveDockerProxyBlob(writer http.ResponseWriter, request *http.Request, registry registryRequest) { + if !s.dockerProxyReadable(writer, registry) { + return + } + s.serveProxyFetch(writer, request, registry.repository, s.dockerBlobFetch(registry)) +} + +// dockerProxyReadable refuses a request a proxy cannot serve. A tag list needs the +// upstream's own catalogue, which this server does not mirror, so it answers from +// what has been pulled instead and says so rather than pretending to be complete. +func (s *Server) dockerProxyReadable(writer http.ResponseWriter, registry registryRequest) bool { + if registry.repository.RemoteURL == "" { + dockerError(writer, http.StatusNotFound, docker.ErrorNameUnknown, + "This proxy repository has no remote to fetch from") + return false + } + return true +} diff --git a/internal/server/docker_proxy_test.go b/internal/server/docker_proxy_test.go new file mode 100644 index 0000000..272a93d --- /dev/null +++ b/internal/server/docker_proxy_test.go @@ -0,0 +1,318 @@ +package server + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "arca/internal/docker" + "arca/internal/store/models" +) + +// fakeRegistry stands in for Docker Hub: it refuses an unauthenticated request with a +// challenge to a separate token service and only serves content to a bearer token, +// which is the protocol the proxy client has to implement rather than an option. +type fakeRegistry struct { + server *httptest.Server + tokens *httptest.Server + manifest []byte + config []byte + layer []byte + + issued atomic.Int32 + unauthed atomic.Int32 + manifests atomic.Int32 + requested []string +} + +const fakeToken = "issued-bearer-token" + +func newFakeRegistry(t *testing.T) *fakeRegistry { + t.Helper() + + registry := &fakeRegistry{ + config: []byte(testImageConfig), + layer: bytes.Repeat([]byte("upstream-layer"), 32), + } + registry.manifest = imageManifestFor( + digestOf(registry.config), len(registry.config), + map[string]int{digestOf(registry.layer): len(registry.layer)}, + ) + + registry.tokens = httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + // The scope names the repository the token is for, which is what makes it + // worth caching per scope rather than once per remote. + if request.URL.Query().Get("scope") == "" { + http.Error(writer, "no scope", http.StatusBadRequest) + return + } + registry.issued.Add(1) + writeJSON(writer, http.StatusOK, map[string]any{"token": fakeToken, "expires_in": 300}) + })) + t.Cleanup(registry.tokens.Close) + + registry.server = httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + registry.requested = append(registry.requested, request.URL.Path) + + if request.Header.Get("Authorization") != "Bearer "+fakeToken { + registry.unauthed.Add(1) + writer.Header().Set("WWW-Authenticate", fmt.Sprintf( + `Bearer realm="%s/token",service="fake.registry",scope="repository:library/nginx:pull"`, + registry.tokens.URL, + )) + http.Error(writer, "unauthorized", http.StatusUnauthorized) + return + } + + // Matched in full rather than by suffix. A lenient fake here hid a missing + // /v2/ prefix that the real Docker Hub answers with a bare 404. + switch request.URL.Path { + case "/v2/nginx/manifests/1.25", "/v2/nginx/manifests/" + digestOf(registry.manifest): + registry.manifests.Add(1) + writer.Header().Set("Content-Type", docker.MediaTypeOCIManifest) + writer.Write(registry.manifest) + + case "/v2/nginx/blobs/" + digestOf(registry.config): + writer.Write(registry.config) + + case "/v2/nginx/blobs/" + digestOf(registry.layer): + writer.Write(registry.layer) + + default: + http.Error(writer, "not found", http.StatusNotFound) + } + })) + t.Cleanup(registry.server.Close) + + return registry +} + +func (i *testInstance) proxyOf(name, remote string) *registryClient { + i.t.Helper() + + expectStatus(i.t, i.api(http.MethodPost, "/api/repositories", + `{"name":"`+name+`","format":"docker","type":"proxy","policy":"mixed","remoteUrl":"`+remote+`"}`), + http.StatusCreated) + + return ®istryClient{instance: i, repository: name} +} + +func TestDockerProxyPullThrough(t *testing.T) { + upstream := newFakeRegistry(t) + + instance := newTestInstance(t) + instance.setup() + proxy := instance.proxyOf("hub", upstream.server.URL) + + t.Run("a manifest is fetched and cached", func(t *testing.T) { + response := proxy.do(http.MethodGet, "/v2/hub/nginx/manifests/1.25", nil, "") + body := expectStatus(t, response, http.StatusOK) + + if digestOf([]byte(body)) != digestOf(upstream.manifest) { + t.Fatal("the served manifest is not the one upstream holds") + } + if got := response.Header.Get(docker.ContentDigestHeader); got != digestOf(upstream.manifest) { + t.Fatalf("%s = %q, want %q", docker.ContentDigestHeader, got, digestOf(upstream.manifest)) + } + }) + + t.Run("the token was obtained rather than the credentials being sent", func(t *testing.T) { + if upstream.issued.Load() == 0 { + t.Fatal("no token was ever requested from the token service") + } + if upstream.unauthed.Load() == 0 { + t.Fatal("the first request already carried a token, so no challenge was answered") + } + }) + + t.Run("the layer is fetched and cached", func(t *testing.T) { + layer := digestOf(upstream.layer) + + body := expectStatus(t, proxy.do(http.MethodGet, "/v2/hub/nginx/blobs/"+layer, nil, ""), http.StatusOK) + if digestOf([]byte(body)) != layer { + t.Fatal("the served layer does not match its digest") + } + }) + + t.Run("the token is reused rather than refetched per request", func(t *testing.T) { + issued := upstream.issued.Load() + + expectStatus(t, proxy.do(http.MethodGet, "/v2/hub/nginx/blobs/"+digestOf(upstream.config), nil, ""), http.StatusOK) + + if upstream.issued.Load() != issued { + t.Fatalf("a cached token was not reused: issued went from %d to %d", issued, upstream.issued.Load()) + } + }) + + t.Run("the cached image appears in the UI", func(t *testing.T) { + body := expectStatus(t, instance.api(http.MethodGet, + "/api/repositories/hub/artifacts?name=nginx", ""), http.StatusOK) + + if !strings.Contains(body, `"version":"1.25"`) { + t.Fatalf("the cached tag is not a version: %s", body) + } + }) + + t.Run("the cached manifest was indexed, so its layers are browsable", func(t *testing.T) { + body := expectStatus(t, instance.api(http.MethodGet, + "/api/repositories/hub/docker/manifests?name=nginx&reference=1.25", ""), http.StatusOK) + + var payload struct { + LayerCount int `json:"layerCount"` + Layers []struct { + Digest string `json:"digest"` + Stored bool `json:"stored"` + } `json:"layers"` + } + if err := json.Unmarshal([]byte(body), &payload); err != nil { + t.Fatalf("decoding the manifest: %v", err) + } + if payload.LayerCount != 1 || len(payload.Layers) != 1 { + t.Fatalf("layers = %v, want one", payload.Layers) + } + if payload.Layers[0].Digest != digestOf(upstream.layer) { + t.Fatalf("layer digest = %q, want %q", payload.Layers[0].Digest, digestOf(upstream.layer)) + } + // It was pulled in an earlier subtest, so it is present locally too. + if !payload.Layers[0].Stored { + t.Fatal("the pulled layer is not reported as stored") + } + }) +} + +// A blob is immutable by construction, so a proxy must never recheck one. A tag must, +// because upstream can move it. +func TestDockerProxyCachesContentForever(t *testing.T) { + upstream := newFakeRegistry(t) + + instance := newTestInstance(t) + instance.setup() + proxy := instance.proxyOf("hub", upstream.server.URL) + + layer := digestOf(upstream.layer) + expectStatus(t, proxy.do(http.MethodGet, "/v2/hub/nginx/blobs/"+layer, nil, ""), http.StatusOK) + + before := len(upstream.requested) + for range 3 { + expectStatus(t, proxy.do(http.MethodGet, "/v2/hub/nginx/blobs/"+layer, nil, ""), http.StatusOK) + } + + if len(upstream.requested) != before { + t.Fatalf("the upstream was asked again for an immutable blob: %v", upstream.requested[before:]) + } +} + +func TestDockerProxyIsReadOnly(t *testing.T) { + upstream := newFakeRegistry(t) + + instance := newTestInstance(t) + instance.setup() + proxy := instance.proxyOf("hub", upstream.server.URL) + + cases := []struct { + name string + method string + path string + }{ + {"opening an upload", http.MethodPost, "/v2/hub/nginx/blobs/uploads/"}, + {"pushing a manifest", http.MethodPut, "/v2/hub/nginx/manifests/1.0"}, + {"deleting a manifest", http.MethodDelete, "/v2/hub/nginx/manifests/1.0"}, + {"deleting a blob", http.MethodDelete, "/v2/hub/nginx/blobs/" + digestOf(upstream.layer)}, + } + + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + expectStatus(t, proxy.do(testCase.method, testCase.path, nil, ""), http.StatusMethodNotAllowed) + }) + } +} + +func TestDockerProxyReportsAnUpstreamMiss(t *testing.T) { + upstream := newFakeRegistry(t) + + instance := newTestInstance(t) + instance.setup() + proxy := instance.proxyOf("hub", upstream.server.URL) + + expectStatus(t, proxy.do(http.MethodGet, "/v2/hub/nginx/manifests/9.9", nil, ""), http.StatusNotFound) +} + +// Docker Hub keeps its official images under an implicit library/ scope, which a +// single-segment name has to be expanded into. No other registry does, and applying it +// everywhere would break every private registry's top-level image. +func TestUpstreamImageNaming(t *testing.T) { + cases := []struct { + name string + remote string + image string + want string + }{ + {"a bare name on Docker Hub", "https://registry-1.docker.io", "nginx", "library/nginx"}, + {"a bare name on index.docker.io", "https://index.docker.io", "nginx", "library/nginx"}, + {"a scoped name on Docker Hub", "https://registry-1.docker.io", "bitnami/nginx", "bitnami/nginx"}, + {"a bare name elsewhere", "https://ghcr.io", "nginx", "nginx"}, + {"a bare name on a private registry", "https://registry.example.com", "internal", "internal"}, + {"a deep name elsewhere", "https://quay.io", "a/b/c", "a/b/c"}, + } + + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + repository := &models.Repository{RemoteURL: testCase.remote} + if got := upstreamImage(repository, testCase.image); got != testCase.want { + t.Fatalf("upstreamImage(%q, %q) = %q, want %q", testCase.remote, testCase.image, got, testCase.want) + } + }) + } +} + +// A proxy always fetches a manifest before the config blob it points at, so indexing +// had nothing to read the platform from. It has to be repaired once the blob arrives, +// or every proxied image reports an unknown architecture for ever. +func TestDockerProxyBackfillsThePlatform(t *testing.T) { + upstream := newFakeRegistry(t) + + instance := newTestInstance(t) + instance.setup() + proxy := instance.proxyOf("hub", upstream.server.URL) + + expectStatus(t, proxy.do(http.MethodGet, "/v2/hub/nginx/manifests/1.25", nil, ""), http.StatusOK) + + read := func() (os, architecture string, created int64) { + body := expectStatus(t, instance.api(http.MethodGet, + "/api/repositories/hub/docker/manifests?name=nginx&reference=1.25", ""), http.StatusOK) + + var payload struct { + OS string `json:"os"` + Architecture string `json:"architecture"` + ImageCreated int64 `json:"imageCreated"` + } + if err := json.Unmarshal([]byte(body), &payload); err != nil { + t.Fatalf("decoding the manifest: %v", err) + } + return payload.OS, payload.Architecture, payload.ImageCreated + } + + t.Run("the platform is unknown while only the manifest is cached", func(t *testing.T) { + os, architecture, _ := read() + if os != "" || architecture != "" { + t.Fatalf("platform = %s/%s, want empty before the config is pulled", os, architecture) + } + }) + + t.Run("pulling the config fills it in", func(t *testing.T) { + expectStatus(t, proxy.do(http.MethodGet, "/v2/hub/nginx/blobs/"+digestOf(upstream.config), nil, ""), http.StatusOK) + + os, architecture, created := read() + if os != "linux" || architecture != "amd64" { + t.Fatalf("platform = %s/%s, want linux/amd64", os, architecture) + } + if created == 0 { + t.Fatal("the build time was not read from the config") + } + }) +} diff --git a/internal/server/docker_reindex.go b/internal/server/docker_reindex.go new file mode 100644 index 0000000..4a8f794 --- /dev/null +++ b/internal/server/docker_reindex.go @@ -0,0 +1,169 @@ +package server + +import ( + "io" + "sort" + + "github.com/charmbracelet/log" + + "arca/internal/docker" + "arca/internal/format" + "arca/internal/store/models" +) + +// Indexing a manifest reads the config blob it points at, so it can only be complete +// once that blob is local. Two paths cannot guarantee that at the time the manifest +// arrives: a proxy fetches the manifest first by definition, and a migration copies in +// whatever order the source pages. Both are repaired by walking what is stored and +// indexing it afresh, which is also the only way to fix a repository whose copy was +// interrupted halfway. + +type reindexResult struct { + Repository string `json:"repository"` + Manifests int `json:"manifests"` + Failed int `json:"failed"` +} + +// storedManifest is one manifest found on disk, with the image it belongs to, which the +// path carries and the document does not. +type storedManifest struct { + image string + tag string + asset *models.Asset + parsed docker.Manifest + size int64 + isIndex bool +} + +func (s *Server) reindexDockerRepository(repository *models.Repository) (reindexResult, error) { + result := reindexResult{Repository: repository.Name} + + files, err := s.store.DockerStoredFiles(repository.ID, "") + if err != nil { + return result, err + } + + manifests := make([]storedManifest, 0, len(files)) + for _, file := range files { + found, ok, err := s.readStoredManifest(repository, file.Path) + if err != nil { + result.Failed++ + log.Warnf("reading %s/%s failed: %v", repository.Name, file.Path, err) + continue + } + if ok { + manifests = append(manifests, found) + } + } + + // Children before parents: an index's total size is the sum of its children's, so + // indexing it first would record only what its descriptors declare. Nested indexes + // are legal and vanishingly rare, and would need a second pass they do not get. + sort.SliceStable(manifests, func(i, j int) bool { + return !manifests[i].isIndex && manifests[j].isIndex + }) + + for _, manifest := range manifests { + if err := s.indexStoredManifest(repository, manifest); err != nil { + result.Failed++ + log.Warnf("indexing %s/%s failed: %v", repository.Name, manifest.asset.Path, err) + continue + } + result.Manifests++ + } + + if result.Manifests > 0 || result.Failed > 0 { + log.Infof("indexed %d manifests of %s, %d failed", result.Manifests, repository.Name, result.Failed) + } + return result, nil +} + +// readStoredManifest recognises a manifest by its path and parses it. Anything else in +// the repository is a blob, which reports false rather than an error. +func (s *Server) readStoredManifest(repository *models.Repository, path string) (storedManifest, bool, error) { + found := storedManifest{} + + switch image, tag, ok := docker.ParseTagPath(path); { + case ok: + found.image, found.tag = image, tag + case docker.IsDigestManifestPath(path): + found.image = docker.ImageOfManifestPath(path) + default: + return found, false, nil + } + + asset, err := s.store.FindAsset(repository.ID, path) + if err != nil { + return found, false, err + } + found.asset = asset + + file, _, err := s.blobs.Open(asset.StorageKey) + if err != nil { + return found, false, err + } + defer file.Close() + + document, err := io.ReadAll(io.LimitReader(file, maxManifestBytes)) + if err != nil { + return found, false, err + } + + found.parsed, err = docker.ParseManifest(document) + if err != nil { + // A stored document that is not a manifest is worth reporting once and then + // leaving alone: it is servable either way, just not describable. + log.Warnf("%s/%s is not a manifest this server can parse: %v", repository.Name, path, err) + return found, false, nil + } + + found.size = int64(len(document)) + found.isIndex = found.parsed.IsIndex() + return found, true, nil +} + +func (s *Server) indexStoredManifest(repository *models.Repository, manifest storedManifest) error { + route := docker.Route{Reference: manifest.tag, Tag: manifest.tag} + if manifest.tag == "" { + digest := docker.SHA256(manifest.asset.SHA256) + route = docker.Route{Reference: digest.String(), Digest: digest, ByDigest: true} + } + + registry := registryRequest{repository: repository, image: manifest.image, route: route} + + // The layout could only guess the media type from the filename, which cannot tell + // an index from an image manifest. The document knows. + if manifest.parsed.MediaType != manifest.asset.ContentType { + if err := s.store.SetAssetContentType(manifest.asset.ID, manifest.parsed.MediaType); err != nil { + return err + } + } + + return s.indexDockerManifest(registry, docker.SHA256(manifest.asset.SHA256), manifest.parsed, manifest.size) +} + +// reindexDockerRepositories rebuilds the metadata of every docker repository. It is +// offered as a maintenance action so a migration that was interrupted, or one run +// before this server knew how to index, can be repaired without recopying anything. +func (s *Server) reindexDockerRepositories() ([]reindexResult, error) { + repositories, err := s.store.RepositoriesOfFormat(format.Docker) + if err != nil { + return nil, err + } + + results := make([]reindexResult, 0, len(repositories)) + for index := range repositories { + repository := &repositories[index] + if repository.IsGroup() { + continue + } + + result, err := s.reindexDockerRepository(repository) + if err != nil { + log.Errorf("reindexing %s failed: %v", repository.Name, err) + continue + } + results = append(results, result) + } + return results, nil +} diff --git a/internal/server/docker_sweep.go b/internal/server/docker_sweep.go new file mode 100644 index 0000000..7219622 --- /dev/null +++ b/internal/server/docker_sweep.go @@ -0,0 +1,210 @@ +package server + +import ( + "time" + + "github.com/charmbracelet/log" + + "arca/internal/docker" + "arca/internal/format" + "arca/internal/store" + "arca/internal/store/models" +) + +// A docker repository is the one format here that cannot reclaim space when +// something is deleted. Removing a tag leaves its layers behind, correctly, because +// they are shared: any other tag may still need them. Nothing else ever revisits +// that decision, so without a sweep a repository grows for as long as it is used. +// +// Reachability starts at the tags and follows the reference graph. An untagged +// manifest is unreachable by definition, which is what makes an index's children +// survive while a retired tag's exclusive layers do not. +// +// Only hosted repositories are swept. A proxy's content is refetchable, and a client +// that pulled an image by digest cached no tag to be reachable from, so reachability +// would delete exactly what it is actively using. Proxies age out by last access +// instead, which purgeIdleCaches already does for every format. + +type sweepResult struct { + Repository string `json:"repository"` + Blobs int `json:"blobs"` + Manifests int `json:"manifests"` + Bytes int64 `json:"bytes"` +} + +type sweepPlan struct { + repository *models.Repository + // paths are the assets to remove, blobs and manifests together, since both are + // deleted the same way. + paths []string + // digests are the manifest records whose parsed form goes with them. + digests []string + result sweepResult +} + +// planDockerSweep decides what is unreachable without deleting anything, so the same +// walk backs both the preview and the sweep. +func (s *Server) planDockerSweep(repository *models.Repository) (sweepPlan, error) { + plan := sweepPlan{repository: repository, result: sweepResult{Repository: repository.Name}} + + roots, err := s.store.DockerTagRoots(repository.ID) + if err != nil { + return plan, err + } + + reachableManifests := map[string]bool{} + reachableBlobs := map[string]bool{} + + frontier := make([]string, 0, len(roots)) + for _, root := range roots { + digest := docker.SHA256(root.SHA256).String() + if reachableManifests[digest] { + continue + } + reachableManifests[digest] = true + frontier = append(frontier, digest) + } + + // Breadth-first through the index children. The visited set is what stops a + // manifest that somehow references itself from looping forever. + for len(frontier) > 0 { + children, blobs, err := s.store.DockerChildrenOf(repository.ID, frontier) + if err != nil { + return plan, err + } + + for _, blob := range blobs { + reachableBlobs[blob] = true + } + + frontier = frontier[:0] + for _, child := range children { + if reachableManifests[child] { + continue + } + reachableManifests[child] = true + frontier = append(frontier, child) + } + } + + blobs, err := s.store.DockerStoredFiles(repository.ID, docker.BlobsDirectory+"/") + if err != nil { + return plan, err + } + for _, blob := range blobs { + if reachableBlobs[docker.SHA256(blob.SHA256).String()] { + continue + } + plan.paths = append(plan.paths, blob.Path) + plan.result.Blobs++ + } + + manifests, err := s.unreachableDockerManifests(repository, reachableManifests) + if err != nil { + return plan, err + } + for _, manifest := range manifests { + plan.paths = append(plan.paths, manifest.Path) + plan.digests = append(plan.digests, docker.SHA256(manifest.SHA256).String()) + plan.result.Manifests++ + } + + return plan, nil +} + +// unreachableDockerManifests finds digest-addressed manifests no tag can reach. They +// live under a directory per image rather than one shared store, so they are found by +// walking every asset and recognising the shape rather than by one prefix. +func (s *Server) unreachableDockerManifests(repository *models.Repository, reachable map[string]bool) ([]store.StoredFile, error) { + files, err := s.store.DockerStoredFiles(repository.ID, "") + if err != nil { + return nil, err + } + + unreachable := []store.StoredFile{} + for _, file := range files { + if !docker.IsDigestManifestPath(file.Path) { + continue + } + if reachable[docker.SHA256(file.SHA256).String()] { + continue + } + unreachable = append(unreachable, file) + } + return unreachable, nil +} + +// applyDockerSweep deletes what the plan named. Rows go before files: a row without +// a file answers every request with a 404 it can never recover from, while a file +// without a row is merely unreferenced and the next sweep collects it. +func (s *Server) applyDockerSweep(plan sweepPlan) (sweepResult, error) { + if len(plan.paths) == 0 { + return plan.result, nil + } + + keys, err := s.store.DeleteAssetsAt(plan.repository.ID, plan.paths) + if err != nil { + return plan.result, err + } + if err := s.store.DeleteDockerManifests(plan.repository.ID, plan.digests); err != nil { + return plan.result, err + } + + usage, err := s.blobs.SizeOf(keys) + if err != nil { + log.Warnf("measuring swept files of %s failed: %v", plan.repository.Name, err) + } + plan.result.Bytes = usage + + if err := s.blobs.Delete(keys...); err != nil { + return plan.result, err + } + + log.Infof("swept %d unreferenced blobs and %d untagged manifests from %s", + plan.result.Blobs, plan.result.Manifests, plan.repository.Name) + + return plan.result, nil +} + +// sweepDockerRepositories runs the sweep over every docker repository. A failure on +// one is logged and the rest carry on, because a repository whose graph cannot be +// read should not stop the others reclaiming their space. +func (s *Server) sweepDockerRepositories(apply bool) ([]sweepResult, error) { + repositories, err := s.store.RepositoriesOfFormat(format.Docker) + if err != nil { + return nil, err + } + + results := make([]sweepResult, 0, len(repositories)) + for index := range repositories { + repository := &repositories[index] + if repository.IsProxy() || repository.IsGroup() { + continue + } + + plan, err := s.planDockerSweep(repository) + if err != nil { + log.Errorf("planning the %s sweep failed: %v", repository.Name, err) + continue + } + if !apply { + results = append(results, plan.result) + continue + } + + result, err := s.applyDockerSweep(plan) + if err != nil { + log.Errorf("sweeping %s failed: %v", repository.Name, err) + } + results = append(results, result) + } + return results, nil +} + +// purgeUnreachableDockerContent is the periodic half. It runs alongside the proxy +// cache eviction, which is the same job for a different reason. +func (s *Server) purgeUnreachableDockerContent(time.Time) { + if _, err := s.sweepDockerRepositories(true); err != nil { + log.Errorf("sweeping docker repositories failed: %v", err) + } +} diff --git a/internal/server/docker_sweep_test.go b/internal/server/docker_sweep_test.go new file mode 100644 index 0000000..9bd0f87 --- /dev/null +++ b/internal/server/docker_sweep_test.go @@ -0,0 +1,368 @@ +package server + +import ( + "bytes" + "encoding/json" + "net/http" + "testing" + + "arca/internal/docker" +) + +type sweepReport struct { + Repositories []struct { + Repository string `json:"repository"` + Blobs int `json:"blobs"` + Manifests int `json:"manifests"` + Bytes int64 `json:"bytes"` + } `json:"repositories"` +} + +func (i *testInstance) sweep(method string) sweepReport { + i.t.Helper() + + body := expectStatus(i.t, i.api(method, "/api/admin/docker/sweep", ""), http.StatusOK) + + var report sweepReport + if err := json.Unmarshal([]byte(body), &report); err != nil { + i.t.Fatalf("decoding the sweep report: %v", err) + } + return report +} + +func (i *testInstance) sweepOf(report sweepReport, repository string) (blobs, manifests int) { + i.t.Helper() + + for _, entry := range report.Repositories { + if entry.Repository == repository { + return entry.Blobs, entry.Manifests + } + } + i.t.Fatalf("%q is missing from the sweep report", repository) + return 0, 0 +} + +// A tagged image must survive a sweep untouched. This is the case that would be +// catastrophic to get wrong, so it is asserted before anything about reclaiming. +func TestDockerSweepKeepsTaggedImages(t *testing.T) { + instance := newTestInstance(t) + instance.setup() + registry := instance.registry("images") + + manifestDigest, layerDigest := registry.pushImage("team/api", "1.0") + + blobs, manifests := instance.sweepOf(instance.sweep(http.MethodPost), "images") + if blobs != 0 || manifests != 0 { + t.Fatalf("a sweep removed %d blobs and %d manifests from a fully tagged repository", blobs, manifests) + } + + cases := []struct { + name string + path string + }{ + {"the tag", "/v2/images/team/api/manifests/1.0"}, + {"the manifest by digest", "/v2/images/team/api/manifests/" + manifestDigest}, + {"the layer", "/v2/images/team/api/blobs/" + layerDigest}, + } + + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + expectStatus(t, registry.do(http.MethodGet, testCase.path, nil, ""), http.StatusOK) + }) + } +} + +func TestDockerSweepReclaimsAnUntaggedImage(t *testing.T) { + instance := newTestInstance(t) + instance.setup() + registry := instance.registry("images") + + // Two tags with layers of their own, so removing one leaves content that only it + // referenced and content the survivor still needs. + _, keptLayer := registry.pushImage("team/api", "1.0") + + config := []byte(testImageConfig) + configDigest := digestOf(config) + retiredLayer := registry.pushBlob("team/api", bytes.Repeat([]byte("retired"), 32)) + retired := imageManifestFor(configDigest, len(config), map[string]int{retiredLayer: 224}) + expectStatus(t, registry.pushManifest("team/api", "0.9", retired), http.StatusCreated) + + // Deleting the tag leaves the manifest and its exclusive layer behind, which is + // correct on its own and is exactly what the sweep exists to finish. + expectStatus(t, registry.do(http.MethodDelete, "/v2/images/team/api/manifests/0.9", nil, ""), http.StatusAccepted) + expectStatus(t, registry.do(http.MethodGet, "/v2/images/team/api/blobs/"+retiredLayer, nil, ""), http.StatusOK) + + t.Run("the preview reports what would go without removing it", func(t *testing.T) { + blobs, manifests := instance.sweepOf(instance.sweep(http.MethodGet), "images") + if blobs != 1 || manifests != 1 { + t.Fatalf("preview reported %d blobs and %d manifests, want 1 and 1", blobs, manifests) + } + expectStatus(t, registry.do(http.MethodGet, "/v2/images/team/api/blobs/"+retiredLayer, nil, ""), http.StatusOK) + }) + + t.Run("the sweep removes them", func(t *testing.T) { + blobs, manifests := instance.sweepOf(instance.sweep(http.MethodPost), "images") + if blobs != 1 || manifests != 1 { + t.Fatalf("the sweep removed %d blobs and %d manifests, want 1 and 1", blobs, manifests) + } + expectStatus(t, registry.do(http.MethodGet, "/v2/images/team/api/blobs/"+retiredLayer, nil, ""), http.StatusNotFound) + }) + + t.Run("the surviving tag is intact", func(t *testing.T) { + expectStatus(t, registry.do(http.MethodGet, "/v2/images/team/api/manifests/1.0", nil, ""), http.StatusOK) + expectStatus(t, registry.do(http.MethodGet, "/v2/images/team/api/blobs/"+keptLayer, nil, ""), http.StatusOK) + }) + + t.Run("a second sweep finds nothing left", func(t *testing.T) { + blobs, manifests := instance.sweepOf(instance.sweep(http.MethodPost), "images") + if blobs != 0 || manifests != 0 { + t.Fatalf("a repeat sweep removed %d blobs and %d manifests, want none", blobs, manifests) + } + }) +} + +// A layer shared by a surviving tag must not be reclaimed when the tag that also used +// it goes. This is the whole reason the reference table exists. +func TestDockerSweepKeepsSharedLayers(t *testing.T) { + instance := newTestInstance(t) + instance.setup() + registry := instance.registry("images") + + _, layer := registry.pushImage("shared/app", "1.0") + + config := []byte(testImageConfig) + second := withAnnotation( + imageManifestFor(digestOf(config), len(config), map[string]int{layer: 704}), + "org.opencontainers.image.revision", "second", + ) + expectStatus(t, registry.pushManifest("shared/app", "1.1", second), http.StatusCreated) + + expectStatus(t, registry.do(http.MethodDelete, "/v2/images/shared/app/manifests/1.0", nil, ""), http.StatusAccepted) + + blobs, manifests := instance.sweepOf(instance.sweep(http.MethodPost), "images") + + // The retired manifest goes, but its layer stays because 1.1 still needs it. + if manifests != 1 { + t.Fatalf("the sweep removed %d manifests, want the one that lost its tag", manifests) + } + if blobs != 0 { + t.Fatalf("the sweep removed %d blobs, want none since the survivor shares them", blobs) + } + + expectStatus(t, registry.do(http.MethodGet, "/v2/images/shared/app/blobs/"+layer, nil, ""), http.StatusOK) + expectStatus(t, registry.do(http.MethodGet, "/v2/images/shared/app/manifests/1.1", nil, ""), http.StatusOK) +} + +// An index's children carry no tag of their own, so a sweep that only looked at tags +// would delete every platform of every multi-arch image. +func TestDockerSweepKeepsIndexChildren(t *testing.T) { + instance := newTestInstance(t) + instance.setup() + registry := instance.registry("images") + + config := []byte(testImageConfig) + configDigest := registry.pushBlob("multi/app", config) + + children := []map[string]any{} + layers := []string{} + for index := range 2 { + layer := bytes.Repeat([]byte{byte('a' + index)}, 128) + layerDigest := registry.pushBlob("multi/app", layer) + layers = append(layers, layerDigest) + + child := imageManifestFor(configDigest, len(config), map[string]int{layerDigest: len(layer)}) + expectStatus(t, registry.pushManifest("multi/app", digestOf(child), child), http.StatusCreated) + + children = append(children, map[string]any{ + "mediaType": docker.MediaTypeOCIManifest, + "digest": digestOf(child), + "size": len(child), + "platform": map[string]any{"os": "linux", "architecture": []string{"amd64", "arm64"}[index]}, + }) + } + + index, err := json.Marshal(map[string]any{ + "schemaVersion": 2, + "mediaType": docker.MediaTypeOCIIndex, + "manifests": children, + }) + if err != nil { + t.Fatalf("building the index: %v", err) + } + expectStatus(t, registry.do(http.MethodPut, "/v2/images/multi/app/manifests/1.0", index, docker.MediaTypeOCIIndex), http.StatusCreated) + + blobs, manifests := instance.sweepOf(instance.sweep(http.MethodPost), "images") + if blobs != 0 || manifests != 0 { + t.Fatalf("the sweep removed %d blobs and %d manifests reachable only through an index", blobs, manifests) + } + + for _, layer := range layers { + t.Run("layer "+layer[7:19], func(t *testing.T) { + expectStatus(t, registry.do(http.MethodGet, "/v2/images/multi/app/blobs/"+layer, nil, ""), http.StatusOK) + }) + } + for _, child := range children { + digest := child["digest"].(string) + t.Run("child "+digest[7:19], func(t *testing.T) { + expectStatus(t, registry.do(http.MethodGet, "/v2/images/multi/app/manifests/"+digest, nil, ""), http.StatusOK) + }) + } + + t.Run("removing the index tag retires the whole tree", func(t *testing.T) { + expectStatus(t, registry.do(http.MethodDelete, "/v2/images/multi/app/manifests/1.0", nil, ""), http.StatusAccepted) + + blobs, manifests := instance.sweepOf(instance.sweep(http.MethodPost), "images") + // Two children plus the index itself, and the config plus both layers. + if manifests != 3 { + t.Fatalf("the sweep removed %d manifests, want 3", manifests) + } + if blobs != 3 { + t.Fatalf("the sweep removed %d blobs, want 3", blobs) + } + }) +} + +// An upload session leaves a file that is not an asset. The sweep walks assets, so it +// must not confuse itself over one, and the session sweep is what collects them. +func TestDockerSweepIgnoresOpenUploads(t *testing.T) { + instance := newTestInstance(t) + instance.setup() + registry := instance.registry("images") + registry.pushImage("app", "1.0") + + response := registry.do(http.MethodPost, "/v2/images/app/blobs/uploads/", nil, "") + expectStatus(t, response, http.StatusAccepted) + location := response.Header.Get("Location") + expectStatus(t, registry.do(http.MethodPatch, location, []byte("half a layer"), blobContentType), http.StatusAccepted) + + blobs, manifests := instance.sweepOf(instance.sweep(http.MethodPost), "images") + if blobs != 0 || manifests != 0 { + t.Fatalf("the sweep removed %d blobs and %d manifests with an upload open", blobs, manifests) + } + + // The session is still usable, which is the point: a sweep is not allowed to + // interfere with a push in flight. + expectStatus(t, registry.do(http.MethodGet, location, nil, ""), http.StatusNoContent) +} + +func TestDockerSweepRequiresAnAdministrator(t *testing.T) { + instance := newTestInstance(t) + instance.setup() + + for _, method := range []string{http.MethodGet, http.MethodPost} { + t.Run(method, func(t *testing.T) { + expectStatus(t, instance.doAnonymously(instance.request(method, "/api/admin/docker/sweep", "")), http.StatusUnauthorized) + }) + } +} + +// A proxy's content is refetchable, and a pull by digest caches no tag for +// reachability to start from, so a reachability sweep would delete exactly what is in +// use. Proxies age out by last access instead. +func TestDockerSweepSkipsProxies(t *testing.T) { + upstream := newFakeRegistry(t) + + instance := newTestInstance(t) + instance.setup() + proxy := instance.proxyOf("hub", upstream.server.URL) + + // Pulled by digest only, so nothing here is reachable from a tag. + layer := digestOf(upstream.layer) + expectStatus(t, proxy.do(http.MethodGet, "/v2/hub/nginx/blobs/"+layer, nil, ""), http.StatusOK) + + report := instance.sweep(http.MethodPost) + for _, entry := range report.Repositories { + if entry.Repository == "hub" { + t.Fatalf("the sweep considered a proxy repository: %+v", entry) + } + } + + expectStatus(t, proxy.do(http.MethodGet, "/v2/hub/nginx/blobs/"+layer, nil, ""), http.StatusOK) +} + +// Rebuilding reads every stored manifest afresh. It is how a repository copied before +// this server knew how to index, or one whose migration stopped halfway, is repaired +// without recopying a byte. +func TestDockerReindexRebuildsMetadata(t *testing.T) { + instance := newTestInstance(t) + instance.setup() + registry := instance.registry("images") + + manifestDigest, _ := registry.pushImage("team/api", "1.0") + + // Wiping the parsed form leaves the bytes in place, which is the state a copy that + // never settled leaves behind. + if err := instance.app.store.DeleteDockerManifest(dockerRepositoryID(t, instance, "images"), manifestDigest); err != nil { + t.Fatalf("clearing the index: %v", err) + } + expectStatus(t, instance.api(http.MethodGet, + "/api/repositories/images/docker/manifests?namespace=team&name=api&reference=1.0", ""), http.StatusNotFound) + + body := expectStatus(t, instance.api(http.MethodPost, "/api/admin/docker/reindex", ""), http.StatusOK) + + var report struct { + Repositories []struct { + Repository string `json:"repository"` + Manifests int `json:"manifests"` + Failed int `json:"failed"` + } `json:"repositories"` + } + if err := json.Unmarshal([]byte(body), &report); err != nil { + t.Fatalf("decoding the report: %v", err) + } + + found := false + for _, entry := range report.Repositories { + if entry.Repository != "images" { + continue + } + found = true + // The tag copy and the digest copy are both manifests on disk. + if entry.Manifests != 2 || entry.Failed != 0 { + t.Fatalf("reindexed %d manifests, %d failed", entry.Manifests, entry.Failed) + } + } + if !found { + t.Fatalf("images is missing from the report: %s", body) + } + + t.Run("the platform is back", func(t *testing.T) { + body := expectStatus(t, instance.api(http.MethodGet, + "/api/repositories/images/docker/manifests?namespace=team&name=api&reference=1.0", ""), http.StatusOK) + + var payload struct { + Architecture string `json:"architecture"` + LayerCount int `json:"layerCount"` + } + if err := json.Unmarshal([]byte(body), &payload); err != nil { + t.Fatalf("decoding the manifest: %v", err) + } + if payload.Architecture != "amd64" || payload.LayerCount != 1 { + t.Fatalf("architecture = %q, layers = %d", payload.Architecture, payload.LayerCount) + } + }) + + t.Run("a sweep after rebuilding still keeps the tagged image", func(t *testing.T) { + blobs, manifests := instance.sweepOf(instance.sweep(http.MethodPost), "images") + if blobs != 0 || manifests != 0 { + t.Fatalf("the sweep removed %d blobs and %d manifests after a rebuild", blobs, manifests) + } + }) +} + +func dockerRepositoryID(t *testing.T, instance *testInstance, name string) string { + t.Helper() + + repository, err := instance.app.store.RepositoryByName(name) + if err != nil { + t.Fatalf("looking up %s: %v", name, err) + } + return repository.ID +} + +func TestDockerReindexRequiresAnAdministrator(t *testing.T) { + instance := newTestInstance(t) + instance.setup() + + expectStatus(t, instance.doAnonymously(instance.request(http.MethodPost, "/api/admin/docker/reindex", "")), http.StatusUnauthorized) +} diff --git a/internal/server/docker_test.go b/internal/server/docker_test.go new file mode 100644 index 0000000..250ec45 --- /dev/null +++ b/internal/server/docker_test.go @@ -0,0 +1,824 @@ +package server + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "testing" + + "arca/internal/docker" +) + +// registryClient drives the registry API the way a Docker client does, so a test +// exercises the wire protocol rather than the handlers directly. +type registryClient struct { + instance *testInstance + repository string +} + +func (i *testInstance) registry(repository string) *registryClient { + i.t.Helper() + + expectStatus(i.t, i.api(http.MethodPost, "/api/repositories", + `{"name":"`+repository+`","format":"docker","policy":"mixed","allowRedeploy":true}`), http.StatusCreated) + + return ®istryClient{instance: i, repository: repository} +} + +func (c *registryClient) do(method, path string, body []byte, contentType string) *http.Response { + c.instance.t.Helper() + + var reader io.Reader + if body != nil { + reader = bytes.NewReader(body) + } + + request, err := http.NewRequest(method, c.instance.server.URL+path, reader) + if err != nil { + c.instance.t.Fatalf("building a registry request: %v", err) + } + if contentType != "" { + request.Header.Set("Content-Type", contentType) + } + request.SetBasicAuth(administratorEmail, administratorPassword) + + return c.instance.do(request) +} + +func (c *registryClient) name(image string) string { return c.repository + "/" + image } + +func digestOf(payload []byte) string { + sum := sha256.Sum256(payload) + return "sha256:" + hex.EncodeToString(sum[:]) +} + +// pushBlob runs the three-request chunked push a Docker client uses, splitting the +// payload so the PATCH path is exercised rather than only the monolithic one. +func (c *registryClient) pushBlob(image string, payload []byte) string { + c.instance.t.Helper() + t := c.instance.t + + response := c.do(http.MethodPost, "/v2/"+c.name(image)+"/blobs/uploads/", nil, "") + expectStatus(t, response, http.StatusAccepted) + + location := response.Header.Get("Location") + if location == "" { + t.Fatal("the upload did not report a Location") + } + if got := response.Header.Get("Range"); got != "0-0" { + t.Fatalf("a fresh session reported Range %q, want 0-0", got) + } + + split := len(payload) / 2 + response = c.do(http.MethodPatch, location, payload[:split], blobContentType) + expectStatus(t, response, http.StatusAccepted) + + if got, want := response.Header.Get("Range"), fmt.Sprintf("0-%d", split-1); got != want { + t.Fatalf("after one chunk Range = %q, want %q", got, want) + } + + digest := digestOf(payload) + response = c.do(http.MethodPut, location+"?digest="+digest, payload[split:], blobContentType) + expectStatus(t, response, http.StatusCreated) + + if got := response.Header.Get(docker.ContentDigestHeader); got != digest { + t.Fatalf("commit reported digest %q, want %q", got, digest) + } + return digest +} + +func (c *registryClient) pushManifest(image, reference string, manifest []byte) *http.Response { + return c.do(http.MethodPut, "/v2/"+c.name(image)+"/manifests/"+reference, manifest, docker.MediaTypeOCIManifest) +} + +func imageManifestFor(configDigest string, configSize int, layers map[string]int) []byte { + descriptors := make([]map[string]any, 0, len(layers)) + for digest, size := range layers { + descriptors = append(descriptors, map[string]any{ + "mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", + "digest": digest, + "size": size, + }) + } + + document, err := json.Marshal(map[string]any{ + "schemaVersion": 2, + "mediaType": docker.MediaTypeOCIManifest, + "config": map[string]any{ + "mediaType": docker.MediaTypeOCIConfig, + "digest": configDigest, + "size": configSize, + }, + "layers": descriptors, + }) + if err != nil { + panic(err) + } + return document +} + +// withAnnotation makes a manifest distinct without changing what it references, +// which is how a test builds two tags that genuinely share one layer. +func withAnnotation(document []byte, key, value string) []byte { + var manifest map[string]any + if err := json.Unmarshal(document, &manifest); err != nil { + panic(err) + } + manifest["annotations"] = map[string]string{key: value} + + annotated, err := json.Marshal(manifest) + if err != nil { + panic(err) + } + return annotated +} + +const testImageConfig = `{"architecture":"amd64","os":"linux","created":"2026-07-30T10:11:12Z",` + + `"config":{"Entrypoint":["/bin/app"],"Labels":{"owner":"platform"}},` + + `"rootfs":{"type":"layers","diff_ids":[]}}` + +// pushImage stores a config, a layer and a manifest, which is the whole of what +// "docker push" does for a single-platform image. +func (c *registryClient) pushImage(image, tag string) (manifestDigest string, layerDigest string) { + c.instance.t.Helper() + + config := []byte(testImageConfig) + layer := bytes.Repeat([]byte("layer-bytes"), 64) + + configDigest := c.pushBlob(image, config) + layerDigest = c.pushBlob(image, layer) + + manifest := imageManifestFor(configDigest, len(config), map[string]int{layerDigest: len(layer)}) + response := c.pushManifest(image, tag, manifest) + expectStatus(c.instance.t, response, http.StatusCreated) + + return digestOf(manifest), layerDigest +} + +func TestDockerVersionCheck(t *testing.T) { + instance := newTestInstance(t) + instance.setup() + + t.Run("an anonymous client is challenged when nothing is public", func(t *testing.T) { + response := instance.doAnonymously(instance.request(http.MethodGet, "/v2/", "")) + expectStatus(t, response, http.StatusUnauthorized) + + if challenge := response.Header.Get("WWW-Authenticate"); !strings.HasPrefix(challenge, "Basic ") { + t.Fatalf("WWW-Authenticate = %q, want a Basic challenge so docker login works", challenge) + } + }) + + t.Run("an authenticated client is accepted", func(t *testing.T) { + request := instance.request(http.MethodGet, "/v2/", "") + request.SetBasicAuth(administratorEmail, administratorPassword) + + response := instance.do(request) + expectStatus(t, response, http.StatusOK) + + if got := response.Header.Get(docker.APIVersionHeader); got != docker.APIVersion { + t.Fatalf("%s = %q, want %q", docker.APIVersionHeader, got, docker.APIVersion) + } + }) + + t.Run("an anonymous client is accepted once a docker repository is public", func(t *testing.T) { + expectStatus(t, instance.api(http.MethodPost, "/api/repositories", + `{"name":"public-images","format":"docker","visibility":"public"}`), http.StatusCreated) + + expectStatus(t, instance.doAnonymously(instance.request(http.MethodGet, "/v2/", "")), http.StatusOK) + }) +} + +func TestDockerPushAndPull(t *testing.T) { + instance := newTestInstance(t) + instance.setup() + registry := instance.registry("images") + + manifestDigest, layerDigest := registry.pushImage("team/api", "1.4.0") + + t.Run("the manifest is pullable by tag", func(t *testing.T) { + response := registry.do(http.MethodGet, "/v2/images/team/api/manifests/1.4.0", nil, "") + body := expectStatus(t, response, http.StatusOK) + + if got := response.Header.Get(docker.ContentDigestHeader); got != manifestDigest { + t.Fatalf("%s = %q, want %q", docker.ContentDigestHeader, got, manifestDigest) + } + if got := response.Header.Get("Content-Type"); got != docker.MediaTypeOCIManifest { + t.Fatalf("Content-Type = %q, want %q", got, docker.MediaTypeOCIManifest) + } + if !strings.Contains(body, layerDigest) { + t.Fatalf("the manifest does not mention its layer: %s", body) + } + }) + + t.Run("the manifest is pullable by digest", func(t *testing.T) { + response := registry.do(http.MethodGet, "/v2/images/team/api/manifests/"+manifestDigest, nil, "") + expectStatus(t, response, http.StatusOK) + }) + + t.Run("a HEAD reports the digest without a body", func(t *testing.T) { + response := registry.do(http.MethodHead, "/v2/images/team/api/manifests/1.4.0", nil, "") + body := expectStatus(t, response, http.StatusOK) + + if body != "" { + t.Fatalf("HEAD returned a body: %q", body) + } + if got := response.Header.Get(docker.ContentDigestHeader); got != manifestDigest { + t.Fatalf("%s = %q, want %q", docker.ContentDigestHeader, got, manifestDigest) + } + }) + + t.Run("the layer is pullable", func(t *testing.T) { + response := registry.do(http.MethodGet, "/v2/images/team/api/blobs/"+layerDigest, nil, "") + body := expectStatus(t, response, http.StatusOK) + + if digestOf([]byte(body)) != layerDigest { + t.Fatalf("the served layer hashed to %s, want %s", digestOf([]byte(body)), layerDigest) + } + }) + + t.Run("the tag is listed", func(t *testing.T) { + body := expectStatus(t, registry.do(http.MethodGet, "/v2/images/team/api/tags/list", nil, ""), http.StatusOK) + + var payload struct { + Name string `json:"name"` + Tags []string `json:"tags"` + } + if err := json.Unmarshal([]byte(body), &payload); err != nil { + t.Fatalf("decoding the tag list: %v", err) + } + if payload.Name != "images/team/api" { + t.Fatalf("name = %q, want %q", payload.Name, "images/team/api") + } + if len(payload.Tags) != 1 || payload.Tags[0] != "1.4.0" { + t.Fatalf("tags = %v, want [1.4.0]", payload.Tags) + } + }) + + t.Run("the tag became a component the rest of the UI can see", func(t *testing.T) { + body := expectStatus(t, instance.api(http.MethodGet, + "/api/repositories/images/artifacts?namespace=team&name=api", ""), http.StatusOK) + + if !strings.Contains(body, `"version":"1.4.0"`) { + t.Fatalf("the tag is not a version: %s", body) + } + if !strings.Contains(body, `"format":"docker"`) { + t.Fatalf("the format was not recorded: %s", body) + } + }) +} + +func TestDockerManifestRequiresItsBlobs(t *testing.T) { + instance := newTestInstance(t) + instance.setup() + registry := instance.registry("images") + + config := []byte(testImageConfig) + configDigest := registry.pushBlob("solo", config) + + missing := digestOf([]byte("never pushed")) + manifest := imageManifestFor(configDigest, len(config), map[string]int{missing: 11}) + + body := expectStatus(t, registry.pushManifest("solo", "1.0", manifest), http.StatusNotFound) + if !strings.Contains(body, docker.ErrorManifestBlobUnknown) { + t.Fatalf("expected %s, got %s", docker.ErrorManifestBlobUnknown, body) + } +} + +func TestDockerBlobUploadRejections(t *testing.T) { + instance := newTestInstance(t) + instance.setup() + registry := instance.registry("images") + + payload := []byte("some layer content") + + t.Run("a commit whose digest does not match the bytes is refused", func(t *testing.T) { + response := registry.do(http.MethodPost, "/v2/images/app/blobs/uploads/", nil, "") + expectStatus(t, response, http.StatusAccepted) + location := response.Header.Get("Location") + + wrong := digestOf([]byte("different content")) + body := expectStatus(t, registry.do(http.MethodPut, location+"?digest="+wrong, payload, blobContentType), http.StatusBadRequest) + + if !strings.Contains(body, docker.ErrorDigestInvalid) { + t.Fatalf("expected %s, got %s", docker.ErrorDigestInvalid, body) + } + }) + + t.Run("a chunk that does not continue the session is refused", func(t *testing.T) { + response := registry.do(http.MethodPost, "/v2/images/app/blobs/uploads/", nil, "") + expectStatus(t, response, http.StatusAccepted) + location := response.Header.Get("Location") + + request, err := http.NewRequest(http.MethodPatch, instance.server.URL+location, bytes.NewReader(payload)) + if err != nil { + t.Fatalf("building a PATCH: %v", err) + } + request.Header.Set("Content-Range", "500-600") + request.SetBasicAuth(administratorEmail, administratorPassword) + + expectStatus(t, instance.do(request), http.StatusRequestedRangeNotSatisfiable) + }) + + t.Run("an unknown session is not found", func(t *testing.T) { + expectStatus(t, registry.do(http.MethodPatch, "/v2/images/app/blobs/uploads/nosuchsession", payload, blobContentType), http.StatusNotFound) + }) + + t.Run("a monolithic push in one POST is accepted", func(t *testing.T) { + digest := digestOf(payload) + response := registry.do(http.MethodPost, "/v2/images/app/blobs/uploads/?digest="+digest, payload, blobContentType) + expectStatus(t, response, http.StatusCreated) + + if got := response.Header.Get(docker.ContentDigestHeader); got != digest { + t.Fatalf("%s = %q, want %q", docker.ContentDigestHeader, got, digest) + } + expectStatus(t, registry.do(http.MethodGet, "/v2/images/app/blobs/"+digest, nil, ""), http.StatusOK) + }) +} + +func TestDockerLayersAreSharedBetweenTags(t *testing.T) { + instance := newTestInstance(t) + instance.setup() + registry := instance.registry("images") + + first, layer := registry.pushImage("shared/app", "1.0") + + // The same layer is referenced by a second tag through an otherwise identical + // manifest, which is what makes a per-version size need the reference table + // rather than the assets under the tag. + config := []byte(testImageConfig) + configDigest := digestOf(config) + second := withAnnotation( + imageManifestFor(configDigest, len(config), map[string]int{layer: 704}), + "org.opencontainers.image.revision", "second", + ) + expectStatus(t, registry.pushManifest("shared/app", "1.1", second), http.StatusCreated) + + if first == digestOf(second) { + t.Fatal("the two manifests are identical, so this proves nothing about sharing") + } + + body := expectStatus(t, registry.do(http.MethodGet, "/v2/images/shared/app/tags/list", nil, ""), http.StatusOK) + for _, tag := range []string{"1.0", "1.1"} { + if !strings.Contains(body, `"`+tag+`"`) { + t.Fatalf("tag %s is missing from %s", tag, body) + } + } + + t.Run("deleting one tag leaves the layer for the other", func(t *testing.T) { + expectStatus(t, registry.do(http.MethodDelete, "/v2/images/shared/app/manifests/1.0", nil, ""), http.StatusAccepted) + + expectStatus(t, registry.do(http.MethodGet, "/v2/images/shared/app/manifests/1.0", nil, ""), http.StatusNotFound) + expectStatus(t, registry.do(http.MethodGet, "/v2/images/shared/app/blobs/"+layer, nil, ""), http.StatusOK) + expectStatus(t, registry.do(http.MethodGet, "/v2/images/shared/app/manifests/1.1", nil, ""), http.StatusOK) + }) +} + +func TestDockerRepositoryResolution(t *testing.T) { + instance := newTestInstance(t) + instance.setup() + registry := instance.registry("images") + registry.pushImage("nginx", "1.25") + + cases := []struct { + name string + path string + status int + }{ + {"the leading segment names the repository", "/v2/images/nginx/manifests/1.25", http.StatusOK}, + {"an unknown repository is not found", "/v2/nowhere/nginx/manifests/1.25", http.StatusNotFound}, + {"an unknown image in a known repository is not found", "/v2/images/absent/manifests/1.25", http.StatusNotFound}, + {"an unknown tag is not found", "/v2/images/nginx/manifests/9.9", http.StatusNotFound}, + } + + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + expectStatus(t, registry.do(http.MethodGet, testCase.path, nil, ""), testCase.status) + }) + } + + t.Run("a bare image resolves through the default repository", func(t *testing.T) { + if err := instance.app.store.SetSetting("docker_repository", "images"); err != nil { + t.Fatalf("setting the default repository: %v", err) + } + expectStatus(t, registry.do(http.MethodGet, "/v2/nginx/manifests/1.25", nil, ""), http.StatusOK) + }) +} + +func TestDockerCatalogListsPrefixedImages(t *testing.T) { + instance := newTestInstance(t) + instance.setup() + + registry := instance.registry("images") + registry.pushImage("team/api", "1.0") + registry.pushImage("nginx", "1.25") + + body := expectStatus(t, registry.do(http.MethodGet, "/v2/_catalog", nil, ""), http.StatusOK) + + var payload struct { + Repositories []string `json:"repositories"` + } + if err := json.Unmarshal([]byte(body), &payload); err != nil { + t.Fatalf("decoding the catalog: %v", err) + } + + // Every entry carries the repository prefix, because without it a client + // could not turn a catalog entry back into something it can pull. + want := map[string]bool{"images/team/api": false, "images/nginx": false} + for _, name := range payload.Repositories { + if _, ok := want[name]; ok { + want[name] = true + } + } + for name, found := range want { + if !found { + t.Fatalf("%q is missing from the catalog: %v", name, payload.Repositories) + } + } +} + +func TestDockerPermissions(t *testing.T) { + instance := newTestInstance(t) + instance.setup() + registry := instance.registry("private-images") + registry.pushImage("app", "1.0") + + t.Run("an anonymous pull from a private repository is challenged", func(t *testing.T) { + response := instance.doAnonymously(instance.request(http.MethodGet, "/v2/private-images/app/manifests/1.0", "")) + expectStatus(t, response, http.StatusUnauthorized) + + if challenge := response.Header.Get("WWW-Authenticate"); !strings.HasPrefix(challenge, "Basic ") { + t.Fatalf("WWW-Authenticate = %q, want a Basic challenge", challenge) + } + }) + + t.Run("a proxy repository refuses writes", func(t *testing.T) { + expectStatus(t, instance.api(http.MethodPost, "/api/repositories", + `{"name":"hub","format":"docker","type":"proxy","remoteUrl":"https://registry-1.docker.io"}`), http.StatusCreated) + + proxy := ®istryClient{instance: instance, repository: "hub"} + expectStatus(t, proxy.do(http.MethodPost, "/v2/hub/nginx/blobs/uploads/", nil, ""), http.StatusMethodNotAllowed) + }) +} + +func TestDockerTagPolicyAndImmutability(t *testing.T) { + instance := newTestInstance(t) + instance.setup() + + expectStatus(t, instance.api(http.MethodPost, "/api/repositories", + `{"name":"releases","format":"docker","policy":"release","allowRedeploy":false}`), http.StatusCreated) + + registry := ®istryClient{instance: instance, repository: "releases"} + + config := []byte(testImageConfig) + configDigest := registry.pushBlob("app", config) + manifest := imageManifestFor(configDigest, len(config), nil) + + t.Run("a release tag is accepted", func(t *testing.T) { + expectStatus(t, registry.pushManifest("app", "1.0.0", manifest), http.StatusCreated) + }) + + t.Run("a prerelease tag is rejected by the release policy", func(t *testing.T) { + body := expectStatus(t, registry.pushManifest("app", "1.1.0-rc1", manifest), http.StatusBadRequest) + if !strings.Contains(body, docker.ErrorTagInvalid) { + t.Fatalf("expected %s, got %s", docker.ErrorTagInvalid, body) + } + }) + + t.Run("a locked repository refuses to move an existing tag", func(t *testing.T) { + body := expectStatus(t, registry.pushManifest("app", "1.0.0", manifest), http.StatusConflict) + if !strings.Contains(body, docker.ErrorDenied) { + t.Fatalf("expected %s, got %s", docker.ErrorDenied, body) + } + }) + + t.Run("a variant tag is not a prerelease", func(t *testing.T) { + expectStatus(t, registry.pushManifest("app", "1.0.0-alpine", manifest), http.StatusCreated) + }) +} + +func TestDockerManifestMetadata(t *testing.T) { + instance := newTestInstance(t) + instance.setup() + registry := instance.registry("images") + + manifestDigest, layerDigest := registry.pushImage("team/api", "1.4.0") + + body := expectStatus(t, instance.api(http.MethodGet, + "/api/repositories/images/docker/manifests?namespace=team&name=api&reference=1.4.0", ""), http.StatusOK) + + var payload struct { + Digest string `json:"digest"` + MediaType string `json:"mediaType"` + TotalSize int64 `json:"totalSize"` + LayerCount int `json:"layerCount"` + IsIndex bool `json:"isIndex"` + OS string `json:"os"` + Architecture string `json:"architecture"` + ImageCreated int64 `json:"imageCreated"` + Layers []struct { + Digest string `json:"digest"` + Size int64 `json:"size"` + SharedWith int `json:"sharedWith"` + Stored bool `json:"stored"` + Foreign bool `json:"foreign"` + } `json:"layers"` + Config *struct { + Entrypoint []string `json:"entrypoint"` + History []struct { + CreatedBy string `json:"createdBy"` + EmptyLayer bool `json:"emptyLayer"` + } `json:"history"` + } `json:"config"` + PullBy map[string]string `json:"pullBy"` + } + if err := json.Unmarshal([]byte(body), &payload); err != nil { + t.Fatalf("decoding the manifest: %v", err) + } + + t.Run("the summary describes the image", func(t *testing.T) { + if payload.Digest != manifestDigest { + t.Fatalf("digest = %q, want %q", payload.Digest, manifestDigest) + } + if payload.IsIndex { + t.Fatal("a single-platform image reported itself as an index") + } + if payload.OS != "linux" || payload.Architecture != "amd64" { + t.Fatalf("platform = %s/%s, want linux/amd64", payload.OS, payload.Architecture) + } + if payload.ImageCreated == 0 { + t.Fatal("the config timestamp was not read") + } + }) + + t.Run("the total size is the layers rather than the manifest", func(t *testing.T) { + // A manifest is a couple of hundred bytes; its config and layer are more. + if payload.TotalSize != int64(len(testImageConfig))+704 { + t.Fatalf("totalSize = %d, want %d", payload.TotalSize, len(testImageConfig)+704) + } + }) + + t.Run("the layer is listed as stored and unshared", func(t *testing.T) { + if payload.LayerCount != 1 || len(payload.Layers) != 1 { + t.Fatalf("layers = %v, want one", payload.Layers) + } + layer := payload.Layers[0] + if layer.Digest != layerDigest { + t.Fatalf("layer digest = %q, want %q", layer.Digest, layerDigest) + } + if !layer.Stored { + t.Fatal("the layer was pushed but is not reported as stored") + } + if layer.Foreign { + t.Fatal("an ordinary layer was reported as foreign") + } + if layer.SharedWith != 0 { + t.Fatalf("sharedWith = %d, want 0 for the only image using it", layer.SharedWith) + } + }) + + t.Run("the config is read from its blob", func(t *testing.T) { + if payload.Config == nil { + t.Fatal("the config was not read") + } + if len(payload.Config.Entrypoint) != 1 || payload.Config.Entrypoint[0] != "/bin/app" { + t.Fatalf("entrypoint = %v, want [/bin/app]", payload.Config.Entrypoint) + } + if len(payload.Config.History) != 0 { + t.Fatalf("history = %v, want none for a config without one", payload.Config.History) + } + }) + + t.Run("the pull references carry the repository prefix", func(t *testing.T) { + if payload.PullBy["tag"] != "images/team/api:1.4.0" { + t.Fatalf("pullBy.tag = %q", payload.PullBy["tag"]) + } + if payload.PullBy["digest"] != "images/team/api@"+manifestDigest { + t.Fatalf("pullBy.digest = %q", payload.PullBy["digest"]) + } + }) + + t.Run("a second image sharing the layer reports it as shared", func(t *testing.T) { + config := []byte(testImageConfig) + second := withAnnotation( + imageManifestFor(digestOf(config), len(config), map[string]int{layerDigest: 704}), + "org.opencontainers.image.revision", "second", + ) + expectStatus(t, registry.pushManifest("team/api", "1.5.0", second), http.StatusCreated) + + body := expectStatus(t, instance.api(http.MethodGet, + "/api/repositories/images/docker/manifests?namespace=team&name=api&reference=1.4.0", ""), http.StatusOK) + + var reread struct { + Layers []struct { + SharedWith int `json:"sharedWith"` + } `json:"layers"` + } + if err := json.Unmarshal([]byte(body), &reread); err != nil { + t.Fatalf("decoding the manifest: %v", err) + } + if reread.Layers[0].SharedWith != 1 { + t.Fatalf("sharedWith = %d, want 1 now that a second image needs it", reread.Layers[0].SharedWith) + } + }) +} + +func TestDockerManifestMetadataForAnIndex(t *testing.T) { + instance := newTestInstance(t) + instance.setup() + registry := instance.registry("images") + + // Two single-platform images become the children of one index, which is what a + // buildx push of a multi-arch image produces. + config := []byte(testImageConfig) + configDigest := registry.pushBlob("multi/app", config) + + children := []map[string]any{} + for index, platform := range []map[string]any{ + {"os": "linux", "architecture": "amd64"}, + {"os": "linux", "architecture": "arm64", "variant": "v8"}, + } { + layer := bytes.Repeat([]byte{byte('a' + index)}, 128) + layerDigest := registry.pushBlob("multi/app", layer) + + child := imageManifestFor(configDigest, len(config), map[string]int{layerDigest: len(layer)}) + expectStatus(t, registry.pushManifest("multi/app", digestOf(child), child), http.StatusCreated) + + children = append(children, map[string]any{ + "mediaType": docker.MediaTypeOCIManifest, + "digest": digestOf(child), + "size": len(child), + "platform": platform, + }) + } + + index, err := json.Marshal(map[string]any{ + "schemaVersion": 2, + "mediaType": docker.MediaTypeOCIIndex, + "manifests": children, + }) + if err != nil { + t.Fatalf("building the index: %v", err) + } + + response := registry.do(http.MethodPut, "/v2/images/multi/app/manifests/1.0", index, docker.MediaTypeOCIIndex) + expectStatus(t, response, http.StatusCreated) + + body := expectStatus(t, instance.api(http.MethodGet, + "/api/repositories/images/docker/manifests?namespace=multi&name=app&reference=1.0", ""), http.StatusOK) + + var payload struct { + IsIndex bool `json:"isIndex"` + TotalSize int64 `json:"totalSize"` + Children []struct { + Platform string `json:"platform"` + TotalSize int64 `json:"totalSize"` + LayerCount int `json:"layerCount"` + Indexed bool `json:"indexed"` + } `json:"children"` + Layers []any `json:"layers"` + } + if err := json.Unmarshal([]byte(body), &payload); err != nil { + t.Fatalf("decoding the index: %v", err) + } + + if !payload.IsIndex { + t.Fatal("IsIndex = false, want true") + } + if len(payload.Layers) != 0 { + t.Fatalf("an index reported %d layers of its own, want none", len(payload.Layers)) + } + if len(payload.Children) != 2 { + t.Fatalf("children = %v, want two platforms", payload.Children) + } + + platforms := []string{"linux/amd64", "linux/arm64/v8"} + for position, child := range payload.Children { + if child.Platform != platforms[position] { + t.Fatalf("child %d platform = %q, want %q", position, child.Platform, platforms[position]) + } + // The children were pushed first, so the index knows their real weight + // rather than only the size of their manifest documents. + if !child.Indexed || child.LayerCount != 1 { + t.Fatalf("child %d was not indexed: %+v", position, child) + } + } + + var childTotal int64 + for _, child := range payload.Children { + childTotal += child.TotalSize + } + if payload.TotalSize != childTotal { + t.Fatalf("totalSize = %d, want the sum of its children %d", payload.TotalSize, childTotal) + } +} + +// Every buildkit multi-platform build attaches an attestation child, which declares +// a platform of unknown/unknown. Without the descriptor annotation it would read as a +// broken architecture rather than as provenance. +func TestDockerIndexAttestationChild(t *testing.T) { + instance := newTestInstance(t) + instance.setup() + registry := instance.registry("images") + + config := []byte(testImageConfig) + configDigest := registry.pushBlob("multi/app", config) + + platform := imageManifestFor(configDigest, len(config), nil) + expectStatus(t, registry.pushManifest("multi/app", digestOf(platform), platform), http.StatusCreated) + + attestation := withAnnotation( + imageManifestFor(configDigest, len(config), nil), + "vnd.docker.reference.type", "attestation-manifest", + ) + expectStatus(t, registry.pushManifest("multi/app", digestOf(attestation), attestation), http.StatusCreated) + + index, err := json.Marshal(map[string]any{ + "schemaVersion": 2, + "mediaType": docker.MediaTypeOCIIndex, + "manifests": []map[string]any{ + { + "mediaType": docker.MediaTypeOCIManifest, + "digest": digestOf(platform), + "size": len(platform), + "platform": map[string]any{"os": "linux", "architecture": "amd64"}, + }, + { + "mediaType": docker.MediaTypeOCIManifest, + "digest": digestOf(attestation), + "size": len(attestation), + // buildkit really does declare unknown/unknown here. + "platform": map[string]any{"os": "unknown", "architecture": "unknown"}, + "annotations": map[string]string{"vnd.docker.reference.type": "attestation-manifest"}, + }, + }, + }) + if err != nil { + t.Fatalf("building the index: %v", err) + } + + expectStatus(t, registry.do(http.MethodPut, "/v2/images/multi/app/manifests/1.0", index, docker.MediaTypeOCIIndex), http.StatusCreated) + + body := expectStatus(t, instance.api(http.MethodGet, + "/api/repositories/images/docker/manifests?namespace=multi&name=app&reference=1.0", ""), http.StatusOK) + + var payload struct { + ImageCreated int64 `json:"imageCreated"` + Children []struct { + Platform string `json:"platform"` + ReferenceType string `json:"referenceType"` + } `json:"children"` + } + if err := json.Unmarshal([]byte(body), &payload); err != nil { + t.Fatalf("decoding the index: %v", err) + } + if len(payload.Children) != 2 { + t.Fatalf("children = %v, want two", payload.Children) + } + + if payload.Children[0].ReferenceType != "" { + t.Fatalf("the real platform carries a reference type: %q", payload.Children[0].ReferenceType) + } + if payload.Children[1].ReferenceType != "attestation-manifest" { + t.Fatalf("the attestation reference type = %q, want attestation-manifest", payload.Children[1].ReferenceType) + } + + // An index has no config of its own, so its build time comes from its children + // rather than being reported as unknown. + if payload.ImageCreated == 0 { + t.Fatal("the index reported no build time, want the newest child's") + } +} + +func TestDockerManifestMetadataRejections(t *testing.T) { + instance := newTestInstance(t) + instance.setup() + registry := instance.registry("images") + registry.pushImage("app", "1.0") + + expectStatus(t, instance.api(http.MethodPost, "/api/repositories", `{"name":"jars","format":"maven2"}`), http.StatusCreated) + + cases := []struct { + name string + path string + status int + }{ + {"an unknown tag", "/api/repositories/images/docker/manifests?name=app&reference=9.9", http.StatusNotFound}, + {"an unknown image", "/api/repositories/images/docker/manifests?name=absent&reference=1.0", http.StatusNotFound}, + {"a missing reference", "/api/repositories/images/docker/manifests?name=app", http.StatusBadRequest}, + {"a missing name", "/api/repositories/images/docker/manifests?reference=1.0", http.StatusBadRequest}, + {"a bad reference", "/api/repositories/images/docker/manifests?name=app&reference=.hidden", http.StatusBadRequest}, + {"a repository of another format", "/api/repositories/jars/docker/manifests?name=app&reference=1.0", http.StatusBadRequest}, + {"an unknown repository", "/api/repositories/nowhere/docker/manifests?name=app&reference=1.0", http.StatusNotFound}, + } + + for _, testCase := range cases { + t.Run(testCase.name, func(t *testing.T) { + expectStatus(t, instance.api(http.MethodGet, testCase.path, ""), testCase.status) + }) + } +} diff --git a/internal/server/formats.go b/internal/server/formats.go index 18a312a..8579e3c 100644 --- a/internal/server/formats.go +++ b/internal/server/formats.go @@ -1,6 +1,7 @@ package server import ( + "arca/internal/docker" "arca/internal/format" "arca/internal/maven" "arca/internal/npm" @@ -20,6 +21,8 @@ func layoutForFormat(name string) format.Layout { return npm.Layout{} case format.P2: return p2.Layout{} + case format.Docker: + return docker.Layout{} default: return maven.Layout{} } diff --git a/internal/server/http.go b/internal/server/http.go index a00f31b..00f50bf 100644 --- a/internal/server/http.go +++ b/internal/server/http.go @@ -50,6 +50,7 @@ func (s *Server) routes() { api.Handle("/repositories/{name}/artifacts", handler(s.routeDeleteArtifact)).Methods(http.MethodDelete) api.Handle("/repositories/{name}/artifacts/files", handler(s.routeArtifactFiles)).Methods(http.MethodGet) api.Handle("/repositories/{name}/recent", handler(s.routeRecent)).Methods(http.MethodGet) + api.Handle("/repositories/{name}/docker/manifests", handler(s.routeDockerManifest)).Methods(http.MethodGet) api.Handle("/search", handler(s.routeSearch)).Methods(http.MethodGet) api.Handle("/admin/health", handler(s.routeHealth)).Methods(http.MethodGet) @@ -66,6 +67,10 @@ func (s *Server) routes() { api.Handle("/admin/migration", handler(s.routeCancelMigration)).Methods(http.MethodDelete) api.Handle("/admin/migration/preview", handler(s.routeMigrationPreview)).Methods(http.MethodPost) + api.Handle("/admin/docker/sweep", handler(s.routeDockerSweepPreview)).Methods(http.MethodGet) + api.Handle("/admin/docker/sweep", handler(s.routeDockerSweep)).Methods(http.MethodPost) + api.Handle("/admin/docker/reindex", handler(s.routeDockerReindex)).Methods(http.MethodPost) + api.PathPrefix("/").HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { writeJSON(writer, http.StatusNotFound, map[string]string{"error": "Not found"}) }) @@ -74,6 +79,16 @@ func (s *Server) routes() { s.router.Handle("/repository/{repository}", http.HandlerFunc(s.handleRepository)).Methods(repositoryMethods...) s.router.PathPrefix("/repository/{repository}/").HandlerFunc(s.handleRepository).Methods(repositoryMethods...) + // The registry API lives at the host root because a Docker client builds its + // URLs from the image reference and cannot be pointed at a path prefix. PATCH + // appears here and nowhere else: it is how a chunked blob push appends. + registryMethods := []string{ + http.MethodGet, http.MethodHead, http.MethodPost, + http.MethodPut, http.MethodPatch, http.MethodDelete, + } + s.router.Handle("/v2", http.HandlerFunc(s.handleDockerRegistry)).Methods(registryMethods...) + s.router.PathPrefix("/v2/").HandlerFunc(s.handleDockerRegistry).Methods(registryMethods...) + s.mountFrontend() } diff --git a/internal/server/maintenance_test.go b/internal/server/maintenance_test.go index c0fb39c..c3c4514 100644 --- a/internal/server/maintenance_test.go +++ b/internal/server/maintenance_test.go @@ -68,7 +68,7 @@ func TestMigrationPreviewWritesNothing(t *testing.T) { `"format":"p2"`, `"action":"create"`, `"action":"skip"`, - "arca has no docker format", + "arca has no rubygems format", } { if !strings.Contains(body, expected) { t.Errorf("the preview is missing %s: %s", expected, body) @@ -125,8 +125,8 @@ func TestMigrationRunsInTheBackground(t *testing.T) { if entry := byName["maven-public"]; !strings.Contains(entry.Reason, "group") { t.Fatalf("maven-public = %+v", entry) } - if entry := byName["docker-hosted"]; entry.State != repositorySkipped { - t.Fatalf("docker-hosted = %+v", entry) + if entry := byName["gems"]; entry.State != repositorySkipped { + t.Fatalf("gems = %+v", entry) } }) diff --git a/internal/server/migrate_test.go b/internal/server/migrate_test.go index 71686a4..05c23be 100644 --- a/internal/server/migrate_test.go +++ b/internal/server/migrate_test.go @@ -23,13 +23,34 @@ type fakeNexus struct { mutex sync.Mutex methods map[string]int assets map[string]map[string]string - repos string + // components maps a repository to the versions it holds and the asset paths of + // each. A real Nexus lists a docker tag here and nowhere else. + components map[string][]fakeComponent + // componentOnly are paths the asset endpoint withholds while still serving their + // content. That is exactly how a real Nexus behaves for a docker tag, and it is the + // whole reason a migration has to walk components as well as assets. + componentOnly map[string]bool + repos string +} + +// fakeComponent mirrors what /service/rest/v1/components returns: a name, a version and +// the assets nested under it. +type fakeComponent struct { + name string + version string + paths []string } func newFakeNexus(t *testing.T, repos string, assets map[string]map[string]string) *fakeNexus { t.Helper() - remote := &fakeNexus{methods: map[string]int{}, assets: assets, repos: repos} + remote := &fakeNexus{ + methods: map[string]int{}, + assets: assets, + components: map[string][]fakeComponent{}, + componentOnly: map[string]bool{}, + repos: repos, + } remote.server = httptest.NewServer(http.HandlerFunc(remote.serve)) t.Cleanup(remote.server.Close) @@ -48,6 +69,9 @@ func (n *fakeNexus) serve(writer http.ResponseWriter, request *http.Request) { case request.URL.Path == "/service/rest/v1/assets": n.serveAssets(writer, request.URL.Query().Get("repository")) + case request.URL.Path == "/service/rest/v1/components": + n.serveComponents(writer, request.URL.Query().Get("repository")) + case strings.HasPrefix(request.URL.Path, "/repository/"): n.serveContent(writer, strings.TrimPrefix(request.URL.Path, "/repository/")) @@ -59,6 +83,9 @@ func (n *fakeNexus) serve(writer http.ResponseWriter, request *http.Request) { func (n *fakeNexus) serveAssets(writer http.ResponseWriter, repository string) { items := []map[string]any{} for path, content := range n.assets[repository] { + if n.componentOnly[repository+"/"+path] { + continue + } items = append(items, map[string]any{ "path": path, "downloadUrl": n.server.URL + "/repository/" + repository + "/" + path, @@ -68,6 +95,26 @@ func (n *fakeNexus) serveAssets(writer http.ResponseWriter, repository string) { json.NewEncoder(writer).Encode(map[string]any{"items": items, "continuationToken": ""}) } +func (n *fakeNexus) serveComponents(writer http.ResponseWriter, repository string) { + items := []map[string]any{} + for _, component := range n.components[repository] { + assets := []map[string]any{} + for _, path := range component.paths { + assets = append(assets, map[string]any{ + "path": path, + "downloadUrl": n.server.URL + "/repository/" + repository + "/" + path, + "fileSize": len(n.assets[repository][path]), + }) + } + items = append(items, map[string]any{ + "name": component.name, + "version": component.version, + "assets": assets, + }) + } + json.NewEncoder(writer).Encode(map[string]any{"items": items, "continuationToken": ""}) +} + func (n *fakeNexus) serveContent(writer http.ResponseWriter, target string) { repository, path, _ := strings.Cut(target, "/") content, ok := n.assets[repository][path] @@ -99,15 +146,61 @@ const fakeRepositories = `[ {"name":"maven-snapshots","format":"maven2","type":"hosted","attributes":{"maven":{"versionPolicy":"SNAPSHOT"}}}, {"name":"maven-central","format":"maven2","type":"proxy","attributes":{"proxy":{"remoteUrl":"https://repo1.maven.org/maven2/"}}}, {"name":"maven-public","format":"maven2","type":"group","attributes":{"group":{"memberNames":["maven-central","maven-snapshots"]}}}, - {"name":"docker-hosted","format":"docker","type":"hosted"} + {"name":"docker-hosted","format":"docker","type":"hosted"}, + {"name":"gems","format":"rubygems","type":"hosted"} ]` +// dockerSource is a Nexus docker repository as one really looks: blobs on the shared +// path, manifests addressed by digest in the asset listing, and the tag reachable only +// through the component listing. The digests are real so the reindex can read the config +// blob the manifest points at. +type dockerSource struct { + assets map[string]string + components []fakeComponent + manifest string + config string + layer string +} + +func newDockerSource() dockerSource { + config := testImageConfig + layer := "a-compressed-layer" + + manifest := string(imageManifestFor( + digestOf([]byte(config)), len(config), + map[string]int{digestOf([]byte(layer)): len(layer)}, + )) + + source := dockerSource{ + manifest: manifest, + config: config, + layer: layer, + components: []fakeComponent{ + {name: "team/api", version: "1.0", paths: []string{"v2/team/api/manifests/1.0"}}, + }, + } + source.assets = map[string]string{ + "v2/-/blobs/" + digestOf([]byte(config)): config, + "v2/-/blobs/" + digestOf([]byte(layer)): layer, + "v2/team/api/manifests/" + digestOf([]byte(manifest)): manifest, + // Downloadable, but withheld from the asset listing below, which is what a real + // Nexus does with a docker tag. + "v2/team/api/manifests/1.0": manifest, + // Nexus exposes paths that mean nothing to this server. They have to be counted + // rather than dropped in silence, or a copy claims to be complete when it is not. + "v2/team/api/tags/list": "{}", + } + return source +} + func migrationFixture(t *testing.T) (*testInstance, *fakeNexus, *migrate.Runner, []migrate.Decision) { t.Helper() instance := newTestInstance(t) instance.setup() + images := newDockerSource() + remote := newFakeNexus(t, fakeRepositories, map[string]map[string]string{ "p2-releases": { "logging/2.0/artifacts.jar": "artifacts-bytes", @@ -120,7 +213,10 @@ func migrationFixture(t *testing.T) (*testInstance, *fakeNexus, *migrate.Runner, "maven-snapshots": { "com/example/seeded/1.0.0-SNAPSHOT/seeded-1.0.0-SNAPSHOT.jar": "seeded-bytes", }, + "docker-hosted": images.assets, }) + remote.components["docker-hosted"] = images.components + remote.componentOnly["docker-hosted/v2/team/api/manifests/1.0"] = true source, err := nexus.New(remote.server.URL, "reader", "secret", 30*time.Second) if err != nil { @@ -223,7 +319,7 @@ func TestMigrationReproducesNexusRepositories(t *testing.T) { }) t.Run("unsupported repositories are skipped, not invented", func(t *testing.T) { - for _, name := range []string{"maven-public", "docker-hosted"} { + for _, name := range []string{"maven-public", "gems"} { if byName[name].Created { t.Fatalf("%s should not have been created", name) } @@ -277,3 +373,146 @@ func TestMigrationIsResumable(t *testing.T) { } } } + +// A docker migration is the case that cannot work off the asset listing alone: Nexus +// reports manifests there by digest only, so a copy that ignored the component listing +// would transfer every byte and leave nothing pullable. +func TestMigrationReproducesDockerImages(t *testing.T) { + instance, remote, runner, plan := migrationFixture(t) + images := newDockerSource() + + results := runner.Apply(context.Background(), plan) + + var docker migrate.Result + for _, result := range results { + if result.Decision.Name() == "docker-hosted" { + docker = result + } + } + + t.Run("the repository was created with a mixed policy", func(t *testing.T) { + if !docker.Created { + t.Fatalf("docker-hosted was not created: %+v", docker) + } + body := expectStatus(t, instance.api(http.MethodGet, "/api/repositories/docker-hosted", ""), http.StatusOK) + if !strings.Contains(body, `"policy":"mixed"`) { + t.Fatalf("policy is not mixed: %s", body) + } + }) + + t.Run("the paths this server has no place for are counted, not dropped in silence", func(t *testing.T) { + if docker.Untranslatable != 1 { + t.Fatalf("untranslatable = %d, want the one tags/list path", docker.Untranslatable) + } + if docker.Failed != 0 { + t.Fatalf("failed = %d: %v", docker.Failed, docker.Errors) + } + }) + + registry := ®istryClient{instance: instance, repository: "docker-hosted"} + + t.Run("the tag is pullable, which only the component walk makes possible", func(t *testing.T) { + response := registry.do(http.MethodGet, "/v2/docker-hosted/team/api/manifests/1.0", nil, "") + body := expectStatus(t, response, http.StatusOK) + + if body != images.manifest { + t.Fatalf("the migrated manifest differs from the source") + } + if got := response.Header.Get("Docker-Content-Digest"); got != digestOf([]byte(images.manifest)) { + t.Fatalf("digest = %q, want %q", got, digestOf([]byte(images.manifest))) + } + }) + + t.Run("the manifest is also pullable by digest", func(t *testing.T) { + expectStatus(t, registry.do(http.MethodGet, + "/v2/docker-hosted/team/api/manifests/"+digestOf([]byte(images.manifest)), nil, ""), http.StatusOK) + }) + + t.Run("the shared blob path was translated to this layout", func(t *testing.T) { + for _, content := range []string{images.config, images.layer} { + body := expectStatus(t, registry.do(http.MethodGet, + "/v2/docker-hosted/team/api/blobs/"+digestOf([]byte(content)), nil, ""), http.StatusOK) + if body != content { + t.Fatalf("a migrated blob differs from the source") + } + } + }) + + t.Run("the tag became a version the UI can list", func(t *testing.T) { + body := expectStatus(t, registry.do(http.MethodGet, "/v2/docker-hosted/team/api/tags/list", nil, ""), http.StatusOK) + if !strings.Contains(body, `"1.0"`) { + t.Fatalf("tags = %s", body) + } + }) + + t.Run("settling indexed the manifest, platform and all", func(t *testing.T) { + body := expectStatus(t, instance.api(http.MethodGet, + "/api/repositories/docker-hosted/docker/manifests?namespace=team&name=api&reference=1.0", ""), http.StatusOK) + + var payload struct { + Architecture string `json:"architecture"` + OS string `json:"os"` + LayerCount int `json:"layerCount"` + Layers []struct { + Stored bool `json:"stored"` + } `json:"layers"` + Config *struct { + Entrypoint []string `json:"entrypoint"` + } `json:"config"` + } + if err := json.Unmarshal([]byte(body), &payload); err != nil { + t.Fatalf("decoding the manifest: %v", err) + } + + // The platform comes from the config blob, which is what makes this prove the + // copy finished before the metadata was built. + if payload.OS != "linux" || payload.Architecture != "amd64" { + t.Fatalf("platform = %s/%s, want linux/amd64", payload.OS, payload.Architecture) + } + if payload.LayerCount != 1 || len(payload.Layers) != 1 || !payload.Layers[0].Stored { + t.Fatalf("layers = %+v", payload.Layers) + } + if payload.Config == nil || len(payload.Config.Entrypoint) != 1 { + t.Fatalf("config = %+v", payload.Config) + } + }) + + t.Run("the migration only ever read from Nexus", func(t *testing.T) { + if writes := remote.writeRequests(); writes != 0 { + t.Fatalf("the migration made %d write requests to Nexus", writes) + } + }) +} + +// Rerunning must not re-transfer anything, which for docker means the translated path +// is what gets checked rather than the source path. +func TestMigrationOfDockerIsResumable(t *testing.T) { + _, _, runner, plan := migrationFixture(t) + + first := runner.Apply(context.Background(), plan) + second := runner.Apply(context.Background(), plan) + + find := func(results []migrate.Result) migrate.Result { + for _, result := range results { + if result.Decision.Name() == "docker-hosted" { + return result + } + } + return migrate.Result{} + } + + if find(first).Copied == 0 { + t.Fatalf("the first run copied nothing: %+v", find(first)) + } + if copied := find(second).Copied; copied != 0 { + t.Fatalf("the second run copied %d assets again", copied) + } + if find(second).Skipped != find(first).Copied { + t.Fatalf("the second run skipped %d, want the %d the first copied", + find(second).Skipped, find(first).Copied) + } + if find(second).Untranslatable != find(first).Untranslatable { + t.Fatalf("the untranslatable count changed between runs: %d then %d", + find(first).Untranslatable, find(second).Untranslatable) + } +} diff --git a/internal/server/migration.go b/internal/server/migration.go index b0b2d92..55a9877 100644 --- a/internal/server/migration.go +++ b/internal/server/migration.go @@ -30,18 +30,19 @@ const ( ) type migrationRepository struct { - Name string `json:"name"` - Source string `json:"source"` - Action string `json:"action"` - Format string `json:"format"` - Type string `json:"type"` - Reason string `json:"reason"` - State string `json:"state"` - Existed bool `json:"existed"` - Copied int `json:"copied"` - Present int `json:"present"` - Failed int `json:"failed"` - CopyingAssets bool `json:"copyingAssets"` + Name string `json:"name"` + Source string `json:"source"` + Action string `json:"action"` + Format string `json:"format"` + Type string `json:"type"` + Reason string `json:"reason"` + State string `json:"state"` + Existed bool `json:"existed"` + Copied int `json:"copied"` + Present int `json:"present"` + Failed int `json:"failed"` + Untranslatable int `json:"untranslatable"` + CopyingAssets bool `json:"copyingAssets"` } type migrationStatus struct { @@ -195,6 +196,7 @@ func (m *migrationRun) record(results []migrate.Result) { m.update(result.Decision.Name(), func(entry *migrationRepository) { entry.Existed = result.Existed entry.Copied, entry.Present, entry.Failed = result.Copied, result.Skipped, result.Failed + entry.Untranslatable = result.Untranslatable entry.State = repositoryDone if result.Failed > 0 { entry.State = repositoryFailed @@ -278,4 +280,27 @@ func (d *localDestination) PutAsset(_ context.Context, repository, path string, return err } +// Settled builds the metadata a format cannot index as its files arrive. Only docker +// needs it, and only after every blob is present: a manifest's platform and labels come +// from a config blob the copy order gives no guarantee of having seen yet. +func (d *localDestination) Settled(_ context.Context, decision migrate.Decision) error { + if decision.Format != format.Docker { + return nil + } + + stored, err := d.repository(decision.Name()) + if err != nil { + return err + } + + result, err := d.server.reindexDockerRepository(stored) + if err != nil { + return err + } + if result.Failed > 0 { + return fmt.Errorf("%d of %d manifests could not be indexed", result.Failed, result.Failed+result.Manifests) + } + return nil +} + func sanitizeSource(url string) string { return strings.TrimSpace(url) } diff --git a/internal/server/proxy.go b/internal/server/proxy.go index b27eca7..8146ff5 100644 --- a/internal/server/proxy.go +++ b/internal/server/proxy.go @@ -12,6 +12,7 @@ import ( "github.com/charmbracelet/log" "arca/internal/blob" + "arca/internal/docker" "arca/internal/format" "arca/internal/maven" "arca/internal/proxy" @@ -53,6 +54,15 @@ type proxyFetch struct { // packument rewrite this one does not depend on the request, so it runs once // on the way in and the stored digests describe what is actually served. Rewrite func(document []byte) ([]byte, error) + // Indexed runs after a fetched asset is stored, for a format that keeps parsed + // metadata beside the bytes and wants a cache fill to build it too. + Indexed func(asset *models.Asset) error + // Headers adds response headers derived from the cached row. Docker needs the + // manifest digest, which is the asset's own SHA256 and so is not known until the + // row is resolved, whether that was from the cache or from the remote. + Headers func(asset *models.Asset) map[string]string + // Limit overrides the upload ceiling, as a container layer needs. + Limit int64 } func pathFetch(path string) proxyFetch { @@ -79,7 +89,7 @@ func (s *Server) serveProxyFetch(writer http.ResponseWriter, request *http.Reque return } - if s.serveCachedAsset(writer, request, repository, asset) { + if s.serveCachedAsset(writer, request, repository, fetch, asset) { return } } @@ -126,9 +136,15 @@ func (s *Server) proxyAsset(repository *models.Repository, fetch proxyFetch) (*m // file has gone missing is dropped rather than left to answer every future // request with a 404: eviction removes rows before files, so a fetch that lands // in that window can outlive its own blob. -func (s *Server) serveCachedAsset(writer http.ResponseWriter, request *http.Request, repository *models.Repository, asset *models.Asset) bool { +func (s *Server) serveCachedAsset(writer http.ResponseWriter, request *http.Request, repository *models.Repository, fetch proxyFetch, asset *models.Asset) bool { s.recordCacheHit(request, repository, asset) + if fetch.Headers != nil { + for name, value := range fetch.Headers(asset) { + writer.Header().Set(name, value) + } + } + if s.writeAsset(writer, request, asset) { return true } @@ -162,6 +178,11 @@ func (s *Server) dropDanglingAsset(asset *models.Asset) { // artifact is already cached is answered from that copy, which keeps clients // that insist on one working against upstreams that publish only some of them. func (s *Server) serveProxyMiss(writer http.ResponseWriter, repository *models.Repository, path string) { + if repository.Format == format.Docker { + dockerError(writer, http.StatusNotFound, docker.ErrorManifestUnknown, "The remote does not have this") + return + } + base, checksum := maven.SplitChecksum(path) if checksum != "" { if parent, err := s.store.FindAsset(repository.ID, base); err == nil { @@ -244,7 +265,8 @@ func (s *Server) storeRemoteAsset(repository *models.Repository, fetch proxyFetc body = rewritten } - _, err := s.storeUpload(repository, fetch.CachePath, body, uploadDetails{ + asset, err := s.storeUpload(repository, fetch.CachePath, body, uploadDetails{ + Limit: fetch.Limit, FetchedAt: now, ExpiresAt: cacheExpiry(layoutFor(repository), repository, fetch.CachePath), LastAccessedAt: now, @@ -258,6 +280,14 @@ func (s *Server) storeRemoteAsset(repository *models.Repository, fetch proxyFetc return err } + // Indexing failure is logged rather than failing the fetch: the bytes are cached + // and servable, and a missing index only costs the UI its detail view. + if fetch.Indexed != nil { + if err := fetch.Indexed(asset); err != nil { + log.Errorf("indexing the cached %s/%s failed: %v", repository.Name, fetch.CachePath, err) + } + } + return s.store.ClearRemoteMiss(repository.ID, fetch.CachePath) } diff --git a/internal/server/repository.go b/internal/server/repository.go index 2fcbb75..2603067 100644 --- a/internal/server/repository.go +++ b/internal/server/repository.go @@ -44,6 +44,8 @@ func (s *Server) handleRepository(writer http.ResponseWriter, request *http.Requ s.serveNPM(writer, request, repository, path) case format.P2: s.serveP2(writer, request, repository, path) + case format.Docker: + s.serveDockerBrowse(writer, request, repository, path) default: s.serveMaven(writer, request, repository, path) } @@ -89,7 +91,15 @@ func (s *Server) writeAsset(writer http.ResponseWriter, request *http.Request, a } type uploadDetails struct { - UploadedBy *string + UploadedBy *string + // ContentType wins over what the layout derives from the filename. A docker + // manifest is the case that needs it: the same filename holds an OCI + // manifest, a Docker one or an index, and only the push says which. + ContentType string + // Limit overrides the configured upload ceiling: zero takes MaxUploadBytes and + // a negative value lifts the limit entirely, which is what a container layer + // needs since it is routinely larger than any sane ceiling for an artifact. + Limit int64 FetchedAt int64 ExpiresAt int64 LastAccessedAt int64 @@ -103,7 +113,15 @@ func (s *Server) storeUpload(repository *models.Repository, path string, body io layout := layoutFor(repository) key := repository.ID + "/" + path - size, digests, err := s.blobs.Put(key, body, s.config.MaxUploadBytes) + limit := s.config.MaxUploadBytes + switch { + case details.Limit < 0: + limit = 0 + case details.Limit > 0: + limit = details.Limit + } + + size, digests, err := s.blobs.Put(key, body, limit) if err != nil { return nil, err } @@ -127,13 +145,18 @@ func (s *Server) storeUpload(repository *models.Repository, path string, body io segments := strings.Split(path, "/") + contentType := details.ContentType + if contentType == "" { + contentType = layout.ContentType(segments[len(segments)-1]) + } + asset := &models.Asset{ RepositoryID: repository.ID, ComponentID: componentID, Path: path, StorageKey: key, Size: size, - ContentType: layout.ContentType(segments[len(segments)-1]), + ContentType: contentType, MD5: digests.MD5, SHA1: digests.SHA1, SHA256: digests.SHA256, diff --git a/internal/server/server.go b/internal/server/server.go index 36a64c5..d5f6700 100644 --- a/internal/server/server.go +++ b/internal/server/server.go @@ -133,6 +133,8 @@ func (s *Server) runMaintenance() { log.Infof("removed %d expired upstream misses", removed) } + s.purgeStaleDockerUploads(time.Now()) + s.purgeUnreachableDockerContent(time.Now()) s.purgeIdleCaches(time.Now()) } } diff --git a/internal/store/assets.go b/internal/store/assets.go index d7fcd8e..3758d8b 100644 --- a/internal/store/assets.go +++ b/internal/store/assets.go @@ -47,6 +47,14 @@ func (s *Store) MarkAssetRefreshed(id string, expiresAt int64, etag, lastModifie }).Error } +// SetAssetContentType corrects what a cache fill recorded. A docker manifest's media +// type is content rather than presentation, since a client negotiates on it, and the +// filename it is stored under cannot tell an image manifest from an index. +func (s *Store) SetAssetContentType(id, contentType string) error { + return s.db.Model(&models.Asset{}).Where("id = ?", id). + UpdateColumn("content_type", contentType).Error +} + func (s *Store) MarkAssetAccessed(id string) error { return s.db.Model(&models.Asset{}).Where("id = ?", id). UpdateColumn("last_accessed_at", NowMillis()).Error @@ -69,6 +77,12 @@ func underPrefix(prefix string) assetSelection { } } +func atPaths(paths []string) assetSelection { + return func(query *gorm.DB) *gorm.DB { + return query.Where("path IN ?", paths) + } +} + func ofComponents(componentIDs []string) assetSelection { return func(query *gorm.DB) *gorm.DB { return query.Where("component_id IN ?", componentIDs) @@ -95,6 +109,15 @@ func (s *Store) DeleteAssetTree(repositoryID, prefix string) ([]string, error) { return s.deleteAssets(repositoryID, underPrefix(prefix)) } +// DeleteAssetsAt removes a named set of paths, which is what a sweep produces: the +// files it decided are unreachable rather than everything under one prefix. +func (s *Store) DeleteAssetsAt(repositoryID string, paths []string) ([]string, error) { + if len(paths) == 0 { + return nil, nil + } + return s.deleteAssets(repositoryID, atPaths(paths)) +} + func (s *Store) deleteAssets(repositoryID string, selection assetSelection) ([]string, error) { var keys []string diff --git a/internal/store/docker.go b/internal/store/docker.go new file mode 100644 index 0000000..0b3075f --- /dev/null +++ b/internal/store/docker.go @@ -0,0 +1,318 @@ +package store + +import ( + "gorm.io/gorm/clause" + + "arca/internal/store/models" +) + +func (s *Store) CreateDockerUpload(upload *models.DockerUpload) error { + now := NowMillis() + upload.CreatedAt = now + upload.UpdatedAt = now + return s.db.Create(upload).Error +} + +func (s *Store) DockerUpload(repositoryID, id string) (*models.DockerUpload, error) { + var upload models.DockerUpload + err := s.db.First(&upload, "id = ? AND repository_id = ?", id, repositoryID).Error + if err != nil { + return nil, err + } + return &upload, nil +} + +func (s *Store) SetDockerUploadSize(id string, size int64) error { + return s.db.Model(&models.DockerUpload{}).Where("id = ?", id). + UpdateColumns(map[string]any{"size": size, "updated_at": NowMillis()}).Error +} + +func (s *Store) DeleteDockerUpload(id string) error { + return s.db.Delete(&models.DockerUpload{}, "id = ?", id).Error +} + +// StaleDockerUploads names the sessions a client walked away from, so the sweep +// can drop their files before removing the rows. +func (s *Store) StaleDockerUploads(cutoff int64) ([]string, error) { + var ids []string + err := s.db.Model(&models.DockerUpload{}). + Where("updated_at < ?", cutoff). + Pluck("id", &ids).Error + return ids, err +} + +func (s *Store) DeleteDockerUploads(ids []string) error { + if len(ids) == 0 { + return nil + } + return s.db.Delete(&models.DockerUpload{}, "id IN ?", ids).Error +} + +// SaveDockerManifest records a parsed manifest together with everything it +// references. The references are replaced rather than merged so a re-pushed +// digest cannot accumulate edges from an earlier parse. +func (s *Store) SaveDockerManifest(manifest *models.DockerManifest, references []models.DockerReference) error { + now := NowMillis() + + return s.Transaction(func(tx *Store) error { + manifest.ID = NewID() + manifest.CreatedAt = now + manifest.UpdatedAt = now + + err := tx.db.Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "repository_id"}, {Name: "digest"}}, + DoUpdates: clause.AssignmentColumns([]string{ + "media_type", "size", "namespace", "name", "config_digest", + "architecture", "os", "variant", "image_created", "layer_count", + "total_size", "labels", "annotations", "subject", "updated_at", + }), + }).Create(manifest).Error + if err != nil { + return err + } + + err = tx.query().Where("repository_id = ? AND manifest_digest = ?", manifest.RepositoryID, manifest.Digest). + Delete(&models.DockerReference{}).Error + if err != nil { + return err + } + if len(references) == 0 { + return nil + } + return tx.db.Create(&references).Error + }) +} + +func (s *Store) DockerManifest(repositoryID, digest string) (*models.DockerManifest, error) { + var manifest models.DockerManifest + err := s.db.First(&manifest, "repository_id = ? AND digest = ?", repositoryID, digest).Error + if err != nil { + return nil, err + } + return &manifest, nil +} + +// UpdateDockerManifestPlatform writes back the fields that could only be read once the +// config blob was local, which on a proxy is always after the manifest itself. +func (s *Store) UpdateDockerManifestPlatform(manifest *models.DockerManifest) error { + return s.db.Model(&models.DockerManifest{}). + Where("repository_id = ? AND digest = ?", manifest.RepositoryID, manifest.Digest). + UpdateColumns(map[string]any{ + "architecture": manifest.Architecture, + "os": manifest.OS, + "variant": manifest.Variant, + "image_created": manifest.ImageCreated, + "labels": manifest.Labels, + }).Error +} + +func (s *Store) DockerReferences(repositoryID, digest string) ([]models.DockerReference, error) { + var references []models.DockerReference + err := s.db. + Where("repository_id = ? AND manifest_digest = ?", repositoryID, digest). + Order("kind, position"). + Find(&references).Error + return references, err +} + +// DockerBlobSharing counts how many manifests in a repository still reference each +// digest. It is what turns "this layer is 400 MB" into "this layer is 400 MB shared +// with four other tags", which is the honest answer to why deleting a tag reclaims +// so little. +func (s *Store) DockerBlobSharing(repositoryID string, digests []string) (map[string]int, error) { + sharing := map[string]int{} + if len(digests) == 0 { + return sharing, nil + } + + var rows []struct { + ChildDigest string + Manifests int + } + err := s.db.Model(&models.DockerReference{}). + Select("child_digest, COUNT(DISTINCT manifest_digest) AS manifests"). + Where("repository_id = ? AND child_digest IN ?", repositoryID, digests). + Group("child_digest"). + Scan(&rows).Error + if err != nil { + return nil, err + } + + for _, row := range rows { + sharing[row.ChildDigest] = row.Manifests + } + return sharing, nil +} + +// DockerManifestsByDigest reads several parsed manifests at once, which is how the +// children of a multi-platform index are described without a query each. +func (s *Store) DockerManifestsByDigest(repositoryID string, digests []string) (map[string]models.DockerManifest, error) { + found := map[string]models.DockerManifest{} + if len(digests) == 0 { + return found, nil + } + + var manifests []models.DockerManifest + err := s.db.Where("repository_id = ? AND digest IN ?", repositoryID, digests).Find(&manifests).Error + if err != nil { + return nil, err + } + + for _, manifest := range manifests { + found[manifest.Digest] = manifest + } + return found, nil +} + +func (s *Store) DeleteDockerManifest(repositoryID, digest string) error { + return s.Transaction(func(tx *Store) error { + err := tx.query().Where("repository_id = ? AND manifest_digest = ?", repositoryID, digest). + Delete(&models.DockerReference{}).Error + if err != nil { + return err + } + return tx.query().Where("repository_id = ? AND digest = ?", repositoryID, digest). + Delete(&models.DockerManifest{}).Error + }) +} + +// DockerImages lists the images a repository holds, rebuilt from the coordinates +// of its tags, so the catalog needs no table of its own. +func (s *Store) DockerImages(repositoryID string) ([]string, error) { + var rows []struct { + Namespace string + Name string + } + err := s.db.Model(&models.Component{}). + Select("DISTINCT namespace, name"). + Where("repository_id = ?", repositoryID). + Order("namespace, name"). + Scan(&rows).Error + if err != nil { + return nil, err + } + + images := make([]string, 0, len(rows)) + for _, row := range rows { + if row.Namespace == "" { + images = append(images, row.Name) + continue + } + images = append(images, row.Namespace+"/"+row.Name) + } + return images, nil +} + +// StoredFile is one asset a sweep has to decide about: its path to delete by, and +// its digest to match against what is still reachable. +type StoredFile struct { + Path string + SHA256 string +} + +// DockerTagRoots names every manifest a tag points at. A tag manifest is the only +// docker asset linked to a component, and its own SHA256 is the digest it resolves +// to, so the roots of the reachability graph need no table of their own. +func (s *Store) DockerTagRoots(repositoryID string) ([]StoredFile, error) { + files := []StoredFile{} + err := s.db.Model(&models.Asset{}). + Select("path, sha256"). + Where("repository_id = ? AND component_id IS NOT NULL", repositoryID). + Scan(&files).Error + return files, err +} + +// DockerStoredFiles lists the assets under a path prefix with their digests, which +// is how the sweep enumerates the blob and digest-manifest stores. +func (s *Store) DockerStoredFiles(repositoryID, prefix string) ([]StoredFile, error) { + files := []StoredFile{} + + bounds, bounded := RangeForPrefix(prefix) + query := s.db.Model(&models.Asset{}). + Select("path, sha256"). + Where("repository_id = ?", repositoryID) + if bounded { + query = query.Where("path >= ? AND path < ?", bounds.Lower, bounds.Upper) + } + + return files, query.Scan(&files).Error +} + +// DockerChildrenOf names what the given manifests reference, split by whether the +// child is another manifest or a blob. The sweep walks the first to find everything +// reachable and keeps the second. +func (s *Store) DockerChildrenOf(repositoryID string, manifests []string) (children []string, blobs []string, err error) { + if len(manifests) == 0 { + return nil, nil, nil + } + + var rows []struct { + ChildDigest string + Kind string + } + err = s.db.Model(&models.DockerReference{}). + Select("child_digest, kind"). + Where("repository_id = ? AND manifest_digest IN ?", repositoryID, manifests). + Scan(&rows).Error + if err != nil { + return nil, nil, err + } + + for _, row := range rows { + if row.Kind == models.DockerReferenceManifest { + children = append(children, row.ChildDigest) + continue + } + blobs = append(blobs, row.ChildDigest) + } + return children, blobs, nil +} + +// DeleteDockerManifests drops the parsed records and edges of manifests a sweep has +// decided are unreachable. The asset holding the bytes is removed separately. +func (s *Store) DeleteDockerManifests(repositoryID string, digests []string) error { + if len(digests) == 0 { + return nil + } + + return s.Transaction(func(tx *Store) error { + err := tx.query().Where("repository_id = ? AND manifest_digest IN ?", repositoryID, digests). + Delete(&models.DockerReference{}).Error + if err != nil { + return err + } + return tx.query().Where("repository_id = ? AND digest IN ?", repositoryID, digests). + Delete(&models.DockerManifest{}).Error + }) +} + +// RepositoriesOfFormat is what the periodic docker sweep walks. The format is passed +// in rather than named here, so this package stays free of format constants. +func (s *Store) RepositoriesOfFormat(repositoryFormat string) ([]models.Repository, error) { + var repositories []models.Repository + err := s.db.Where("format = ?", repositoryFormat).Order("name").Find(&repositories).Error + return repositories, err +} + +// HasPublicRepositories reports whether anyone can read anything of a format +// without signing in. The registry version check is repository-agnostic, so it +// is the one endpoint that has to answer before a repository is even named. +func (s *Store) HasPublicRepositories(repositoryFormat string) (bool, error) { + var count int64 + err := s.db.Model(&models.Repository{}). + Where("format = ? AND visibility = ?", repositoryFormat, models.VisibilityPublic). + Limit(1). + Count(&count).Error + return count > 0, err +} + +// RepositoryByNameAndFormat keeps a docker request from resolving onto a Maven +// repository that happens to share the leading path segment of an image name. +func (s *Store) RepositoryByNameAndFormat(name, repositoryFormat string) (*models.Repository, error) { + var repository models.Repository + err := s.db.First(&repository, "name = ? AND format = ?", name, repositoryFormat).Error + if err != nil { + return nil, err + } + return &repository, nil +} diff --git a/internal/store/models/docker.go b/internal/store/models/docker.go new file mode 100644 index 0000000..f87288a --- /dev/null +++ b/internal/store/models/docker.go @@ -0,0 +1,83 @@ +package models + +const ( + DockerReferenceConfig = "config" + DockerReferenceLayer = "layer" + DockerReferenceManifest = "manifest" +) + +// DockerUpload is one in-flight blob push. It lives in the database rather than +// in memory so a GET on the upload location reports a truthful offset and a +// restart strands no half-written file that nothing will ever clean up. +type DockerUpload struct { + ID string `gorm:"primaryKey"` + RepositoryID string `gorm:"index;not null"` + Image string `gorm:"not null"` + Size int64 `gorm:"not null;default:0"` + UserID *string + CreatedAt int64 `gorm:"autoCreateTime:milli"` + UpdatedAt int64 `gorm:"autoUpdateTime:milli;index"` +} + +func (DockerUpload) TableName() string { return "docker_uploads" } + +// DockerManifest is the parsed form of a manifest document, keyed by the digest +// of the bytes it was parsed from. The bytes themselves are an asset; this is +// everything about them worth querying without reopening the file. +type DockerManifest struct { + ID string `gorm:"primaryKey"` + RepositoryID string `gorm:"not null;uniqueIndex:docker_manifests_digest,priority:1;index:docker_manifests_image,priority:1"` + Digest string `gorm:"not null;uniqueIndex:docker_manifests_digest,priority:2"` + MediaType string `gorm:"not null;default:''"` + Size int64 `gorm:"not null;default:0"` + Namespace string `gorm:"not null;default:'';index:docker_manifests_image,priority:2"` + Name string `gorm:"not null;default:'';index:docker_manifests_image,priority:3"` + ConfigDigest string `gorm:"not null;default:''"` + Architecture string `gorm:"not null;default:''"` + OS string `gorm:"column:os;not null;default:''"` + Variant string `gorm:"not null;default:''"` + // ImageCreated is when the image was built, read from its config blob, as + // opposed to CreatedAt which is when this server first saw it. + ImageCreated int64 `gorm:"not null;default:0"` + LayerCount int `gorm:"not null;default:0"` + TotalSize int64 `gorm:"not null;default:0"` + Labels string `gorm:"not null;default:''"` + Annotations string `gorm:"not null;default:''"` + Subject string `gorm:"not null;default:''"` + CreatedAt int64 `gorm:"autoCreateTime:milli"` + UpdatedAt int64 `gorm:"autoUpdateTime:milli"` +} + +func (DockerManifest) TableName() string { return "docker_manifests" } + +func (m DockerManifest) IsIndex() bool { return m.ConfigDigest == "" } + +// DockerReference is one edge from a manifest to a blob or to a child manifest. +// It exists because an asset row can belong to one component, and a layer is +// shared by every tag that references it, so per-version size and safe deletion +// cannot be read off the assets alone. +type DockerReference struct { + RepositoryID string `gorm:"primaryKey;index:docker_references_child,priority:1"` + ManifestDigest string `gorm:"primaryKey"` + // The reverse index answers "which manifests still need this blob", which is + // the question both the shared-layer marker and the eventual sweep ask. + ChildDigest string `gorm:"primaryKey;index:docker_references_child,priority:2"` + Kind string `gorm:"not null"` + MediaType string `gorm:"not null;default:''"` + Size int64 `gorm:"not null;default:0"` + Position int64 `gorm:"not null;default:0"` + Platform string `gorm:"not null;default:''"` + // URLs holds the newline-separated sources of a nondistributable layer, whose + // bytes are never pushed here. Recording them keeps the layer accountable in + // the UI instead of showing as a blob that is inexplicably missing. + URLs string `gorm:"column:urls;not null;default:''"` + // Annotations are the descriptor's own, encoded as JSON. They are what tells an + // attestation child of an index apart from a real platform. + Annotations string `gorm:"not null;default:''"` +} + +// IsForeign reports a layer whose bytes live outside this registry, which is why +// it has no asset behind it and must not be counted as stored. +func (r DockerReference) IsForeign() bool { return r.URLs != "" } + +func (DockerReference) TableName() string { return "docker_references" } diff --git a/internal/store/settings.go b/internal/store/settings.go index d0477ba..d470632 100644 --- a/internal/store/settings.go +++ b/internal/store/settings.go @@ -15,6 +15,10 @@ const ( SettingLogoType = "logo_type" SettingLogoUpdatedAt = "logo_updated_at" SettingInitialized = "initialized" + // SettingDockerRepository names the repository a bare image reference resolves + // to, so "docker pull host/nginx" works without the repository prefix that + // name-based routing otherwise requires. + SettingDockerRepository = "docker_repository" DefaultInstanceName = "arca" DefaultThemeColor = "#4f46e5" diff --git a/internal/store/store.go b/internal/store/store.go index cca1bfe..a75cf91 100644 --- a/internal/store/store.go +++ b/internal/store/store.go @@ -98,6 +98,9 @@ func (s *Store) migrate() error { &models.Asset{}, &models.RemoteMiss{}, &models.TrafficEvent{}, + &models.DockerUpload{}, + &models.DockerManifest{}, + &models.DockerReference{}, ) }