From 673ec380c6b7b50861a91e28830167e96c9ad19f Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 22:18:34 +0000 Subject: [PATCH 1/3] test(api,cmd): close the test and wording points that the reviews left open MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - TestResolveServeConfigEmptySyncAllowPrivate: serve reads an OPENSTORAGE_SYNC_ALLOW_PRIVATE that is present but empty as unset, the one layer that can tell it from an absent one (review of #897). - The docs and comments call OPENSTORAGE_SYNC_PUSH_ALLOW_PRIVATE the deprecated push-only variable, not a spelling: configuration.md also calls it a separate variable, not a legacy alias (review of #897). - TestReadValueListsPastGitArgBudget runs each missing-repository case as a subtest named after its route, and its comment says that only the v1 routes check the list first (review of #898). - TestSyncValuesTheRunRefusesAtRegistration gives each POST case a repository name of its own, and runs the tracked and repeat creates as a subtest, so one failure no longer hides the others (review of #900). - The stored-text row of wire-compatibility.md scopes the line-break token rule to a sync create (review of #900). Signed-off-by: Thomas Kosiewski --- _Generated with [`xum`](https://github.com/coder/xum) • Model: `anthropic:claude-opus-5-5` • Thinking: `xhigh`_ Change-Id: Ifbee7306f2c6802dfae82bfab4ea5e110cfa4291 Signed-off-by: Thomas Kosiewski --- cmd/openstorage/README.md | 2 +- cmd/openstorage/config.go | 10 ++-- cmd/openstorage/config_test.go | 44 +++++++++++++++++- cmd/openstorage/wire.go | 2 +- ...y-is-unified-behind-one-dial-time-guard.md | 6 +-- docs/explanation/wire-compatibility.md | 2 +- docs/reference/configuration.md | 4 +- internal/api/long_input_test.go | 23 ++++++---- internal/api/sync_values_test.go | 46 +++++++++++-------- 9 files changed, 94 insertions(+), 45 deletions(-) diff --git a/cmd/openstorage/README.md b/cmd/openstorage/README.md index 17797f9d4..a7a598993 100644 --- a/cmd/openstorage/README.md +++ b/cmd/openstorage/README.md @@ -96,7 +96,7 @@ The `hook` subprocess also reads per-push variables that the Git server sets (na - `[config.go:resolveSweepSettings]` `OPENSTORAGE_REPO_RETENTION`, when non-zero, must be `>= jobs.MinRepoPurgeAge` and needs an active sweep. - `[config.go:resolveLFSGCSettings]` Grace and object budget must be `> 0`. Delete mode needs a non-zero interval. - `[config.go:resolveRepackThreshold]` The threshold must be `0` or `>= 2`. -- `[config.go:resolveSyncEgress]` `OPENSTORAGE_SYNC_ALLOW_PRIVATE`, when set to a non-empty value, governs both push and pull. A malformed value in either spelling is an error. +- `[config.go:resolveSyncEgress]` `OPENSTORAGE_SYNC_ALLOW_PRIVATE`, when set to a non-empty value, governs both push and pull. A malformed value in either variable is an error. - `[config.go:resolveSecretsCodec]` `decrypt-only` requires a configured key. - `[config.go:normalizeBaseURL]` The base URL must be `http` or `https` with a host and no query or fragment. - `[config.go:assertf]` Impossible states panic with `openstorage assertion failed:`. Input errors return errors. diff --git a/cmd/openstorage/config.go b/cmd/openstorage/config.go index 8ff904be7..a951374d0 100644 --- a/cmd/openstorage/config.go +++ b/cmd/openstorage/config.go @@ -195,13 +195,13 @@ const ( // link-local/ULA upstreams (metadata/special-purpose stay denied); // "false" denies the private classes in both directions. Unset is the // transitional compatibility default: pushes stay strict (the - // deprecated per-direction spelling below may relax them) while pulls + // deprecated push-only variable below may relax them) while pulls // ALLOW private upstreams with a boot warning — internal-forge // mirrors are a mainstream released 0.6.x deployment shape, and a // future release flips the unset pull default to deny. envSyncAllowPrivate = "OPENSTORAGE_SYNC_ALLOW_PRIVATE" // envSyncPushAllowPrivate relaxes the outbound push egress guard's - // private receiver classes (issue #90). Deprecated spelling: it stays + // private receiver classes (issue #90). Deprecated variable: it stays // push-only (its historical meaning), and envSyncAllowPrivate wins // for both directions when set. envSyncPushAllowPrivate = "OPENSTORAGE_SYNC_PUSH_ALLOW_PRIVATE" @@ -1181,17 +1181,17 @@ type syncEgressSettings struct { // is unset, so pulls allow private upstreams (released 0.6.x behavior) // and boot warns about the future default flip to deny. pullTransitional bool - // legacyPushFlagSet records that the deprecated push-only spelling was + // legacyPushFlagSet records that the deprecated push-only variable was // present, for the boot-time deprecation notice (when the unified knob // is also set the legacy value is ignored entirely). legacyPushFlagSet bool } // resolveSyncEgress resolves the unified sync egress policy (issue #139). -// Both spellings parse fail-fast even when losing (a malformed boolean is +// Both variables parse fail-fast even when losing (a malformed boolean is // a boot error, never a silent fallback); the unified knob wins for both // directions. Unset keeps the transitional compatibility default: strict -// pushes (unless the deprecated push-only spelling relaxes them) and +// pushes (unless the deprecated push-only variable relaxes them) and // allow-private pulls, with wire.go warning about the future flip. func resolveSyncEgress(getenv func(string) string) (syncEgressSettings, error) { assertf(getenv != nil, "resolveSyncEgress: nil getenv") diff --git a/cmd/openstorage/config_test.go b/cmd/openstorage/config_test.go index 877751c3f..5fcf0683a 100644 --- a/cmd/openstorage/config_test.go +++ b/cmd/openstorage/config_test.go @@ -1344,8 +1344,8 @@ func TestResolveWebhookAllowPrivate(t *testing.T) { // TestResolveSyncEgress covers the unified sync egress policy resolution // (issue #139): the unified knob governs both directions when set; unset // keeps the transitional compatibility default (strict pushes unless the -// deprecated push-only spelling relaxes them, allow-private pulls with the -// boot warning); malformed values are boot errors for BOTH spellings. +// deprecated push-only variable relaxes them, allow-private pulls with the +// boot warning); malformed values are boot errors for BOTH variables. func TestResolveSyncEgress(t *testing.T) { for name, tc := range map[string]struct { env map[string]string @@ -1406,6 +1406,46 @@ func TestResolveSyncEgress(t *testing.T) { } } +// TestResolveServeConfigEmptySyncAllowPrivate: serve reads an +// OPENSTORAGE_SYNC_ALLOW_PRIVATE that is present but empty as unset, so the +// deprecated push-only variable still decides pushes and pulls keep the +// transitional default. resolveSyncEgress cannot tell an empty value from +// an absent one; this is the layer where the environment can. +func TestResolveServeConfigEmptySyncAllowPrivate(t *testing.T) { + for name, tc := range map[string]struct { + env map[string]string + want syncEgressSettings + }{ + "empty unified, legacy true": { + env: map[string]string{envSyncAllowPrivate: "", envSyncPushAllowPrivate: "true"}, + want: syncEgressSettings{pushAllowPrivate: true, pullAllowPrivate: true, pullTransitional: true, legacyPushFlagSet: true}, + }, + "empty unified, no legacy": { + env: map[string]string{envSyncAllowPrivate: ""}, + want: syncEgressSettings{pullAllowPrivate: true, pullTransitional: true}, + }, + "unified false wins over legacy true": { + env: map[string]string{envSyncAllowPrivate: "false", envSyncPushAllowPrivate: "true"}, + want: syncEgressSettings{legacyPushFlagSet: true}, + }, + } { + t.Run(name, func(t *testing.T) { + env := map[string]string{ + "OPENSTORAGE_ORG": "org1", + "OPENSTORAGE_ORG_PUBKEYS": "/tmp/org.pub.pem", + } + maps.Copy(env, tc.env) + cfg, err := resolveServeConfig(mapEnv(env), "", "127.0.0.1:0", "/tmp/cache", t.TempDir()) + if err != nil { + t.Fatal(err) + } + if cfg.syncEgress != tc.want { + t.Fatalf("syncEgress = %+v, want %+v", cfg.syncEgress, tc.want) + } + }) + } +} + func TestResolveSecretsCodec(t *testing.T) { // 32 zero bytes, base64 — a syntactically valid AES-256 key. validKey := base64.StdEncoding.EncodeToString(make([]byte, 32)) diff --git a/cmd/openstorage/wire.go b/cmd/openstorage/wire.go index 0792e8918..b66153ddc 100644 --- a/cmd/openstorage/wire.go +++ b/cmd/openstorage/wire.go @@ -38,7 +38,7 @@ import ( // logSyncEgressWarnings emits the boot-time sync egress policy notices // (issue #139) beside the webhook hatch warning: a relaxed guard must never // be silent, the transitional pull default announces its future flip to -// deny, and the deprecated push-only spelling names its replacement. +// deny, and the deprecated push-only variable names its replacement. func logSyncEgressWarnings(logger *slog.Logger, s syncEgressSettings) { assertf(logger != nil, "logSyncEgressWarnings: nil logger") switch { diff --git a/docs/adr/045-sync-egress-policy-is-unified-behind-one-dial-time-guard.md b/docs/adr/045-sync-egress-policy-is-unified-behind-one-dial-time-guard.md index 32ab700ae..50c001a5b 100644 --- a/docs/adr/045-sync-egress-policy-is-unified-behind-one-dial-time-guard.md +++ b/docs/adr/045-sync-egress-policy-is-unified-behind-one-dial-time-guard.md @@ -29,14 +29,14 @@ boot warning announcing the future flip of that default to deny. Syncing from internal RFC 1918 gitea/forgejo/GitLab forges is a mainstream released 0.6.x deployment shape; default-denying it is a breaking change that gets a deprecation window, not a surprise. `OPENSTORAGE_SYNC_PUSH_ALLOW_PRIVATE` -becomes the deprecated push-only spelling: still honored (its historical +becomes the deprecated push-only variable: still honored (its historical meaning, push only) when the unified knob is unset, ignored with a boot notice when it is set. One knob was chosen over per-direction flags because the threat and the deployment shape are symmetric — an internal-forge deployment wants both directions private, a hardened deployment wants neither — and a second per-direction flag would double the migration story for no expressible policy anyone asked for. Malformed booleans in either -spelling are boot errors. +variable are boot errors. **Ambient proxies no longer apply to sync transfers.** The pull fetch previously inherited `HTTP_PROXY`/`HTTPS_PROXY` from the environment; the @@ -61,5 +61,5 @@ gain the dial-time guard with private targets still allowed by default; `serve` warns at boot until `OPENSTORAGE_SYNC_ALLOW_PRIVATE` is set explicitly. Set `true` to keep internal-forge sync across the future flip, `false` to adopt strict mode early. A future minor release flips the unset -pull default to deny; the deprecated push-only spelling is removed no +pull default to deny; the deprecated push-only variable is removed no earlier than that flip. diff --git a/docs/explanation/wire-compatibility.md b/docs/explanation/wire-compatibility.md index e030cbff1..faf60a73b 100644 --- a/docs/explanation/wire-compatibility.md +++ b/docs/explanation/wire-compatibility.md @@ -239,7 +239,7 @@ the [archived phase notes](../archive/phases/)): | A token that fails verification (malformed, bad signature, expired, unknown issuer): upstream's live docs and servers answer `403` `forbidden` (`Invalid or expired token`) on the REST API, and `403` on the Git and LFS transports too (checked 2026-11-04). The vendored spec documents no status for it | The REST API answers `403` `forbidden` (`Invalid or expired token`), as upstream does ([#664](https://github.com/coder/openstorage/issues/664)). The Git and LFS transports answer `401` with `WWW-Authenticate: Basic`, as for a missing token ([authentication](../reference/authentication.md#failure-mapping)) | Git asks for credentials only after a `401`, and only a `401` makes it erase a stored credential. After a `403`, git fails and keeps the expired token in the credential helper, so every later command fails the same way until the user removes the token | | A `crit` member in the JWT header: the vendored docs and spec say nothing about it. They list `alg` and `typ` as the header ([authentication guide](../upstream/getting-started/authentication.md)) | A token whose header has a `crit` member of any value fails verification, even when the signature is valid. The status is the one for any token that fails verification (the row above) | RFC 7515 §4.1.11: a recipient that does not understand a listed extension must refuse the token. open.storage understands none, so it refuses any `crit` | | Every path field and `path` parameter is a plain `type: string`. The spec says nothing about tree entry names whose bytes are not valid UTF-8 | A response path that is not valid UTF-8 still lists with one U+FFFD per invalid byte, and gets an extra sibling `_base64` with the raw bytes in standard base64 (for example `path_base64`, `paths_base64`, `old_path_base64`, `original_path_base64`). The sibling is absent for valid UTF-8 paths, so those responses are unchanged. This covers `files`, `files/metadata`, the diff routes, `blame` and `grep`. The merge conflict paths do not have it yet, and JSON request bodies cannot name such a file yet. The `path` query parameter takes the raw bytes percent-encoded ([path encoding](../reference/http-api.md#conventions)) | JSON strings hold Unicode text, so such a byte has no exact JSON form. A sibling field (like GitHub's `pathRaw`) is additive: no SDK reads it, and the SDKs ignore response keys they do not know. If upstream documents another form, we carry both ([#587](https://github.com/coder/openstorage/issues/587)) | -| Text the request asks the server to store (a webhook `secret`, a git credential `username` and `password`, the `owner`, `name` and `auth.token` of a sync link, a `pull-upstream` `ref`) is plain `type: string`. The spec documents no `400` for its content | Such a value that holds a NUL byte answers `400` `bad_request` ("invalid secret (want UTF-8 text without NUL)", with `invalid_params` naming the field where the route has them), on every backend. A `GET /repos` `cursor` whose repo ID holds a NUL byte or invalid UTF-8 answers the `400` of an undecodable cursor. A path or body ID with such text (a webhook or git credential `id`) answers the `404` of an unknown ID. A git credential `username` or `password`, or a `base_repo.auth.token`, with a carriage return or line feed answers `400` too, naming the field: the sync run hands the pair to git through a credential helper that reads one value per line, and failed such a credential on every run as `repository configuration error` | PostgreSQL refuses a NUL byte or invalid UTF-8 in a text parameter (SQLSTATE 22021) and answered `500` for most of these values, while SQLite and S3-WAL stored them. With `OPENSTORAGE_SECRETS_KEY` set in the default `encrypt` mode, the `secret`, `password` and `auth.token` reach PostgreSQL encrypted, so PostgreSQL stored them too, and such a value now answers `400` there as well. A `ref` is stored only by PostgreSQL's job queue, as `jsonb`, which cannot hold a NUL (SQLSTATE 22P05). Every backend now gives the same answer, as for `q` and `prefix` below. The PostgreSQL store answers `ErrNotFound` for such a key before it runs the query | +| Text the request asks the server to store (a webhook `secret`, a git credential `username` and `password`, the `owner`, `name` and `auth.token` of a sync link, a `pull-upstream` `ref`) is plain `type: string`. The spec documents no `400` for its content | Such a value that holds a NUL byte answers `400` `bad_request` ("invalid secret (want UTF-8 text without NUL)", with `invalid_params` naming the field where the route has them), on every backend. A `GET /repos` `cursor` whose repo ID holds a NUL byte or invalid UTF-8 answers the `400` of an undecodable cursor. A path or body ID with such text (a webhook or git credential `id`) answers the `404` of an unknown ID. A git credential `username` or `password`, or the `base_repo.auth.token` of a sync create, with a carriage return or line feed answers `400` too, naming the field: the sync run hands the pair to git through a credential helper that reads one value per line, and failed such a credential on every run as `repository configuration error` | PostgreSQL refuses a NUL byte or invalid UTF-8 in a text parameter (SQLSTATE 22021) and answered `500` for most of these values, while SQLite and S3-WAL stored them. With `OPENSTORAGE_SECRETS_KEY` set in the default `encrypt` mode, the `secret`, `password` and `auth.token` reach PostgreSQL encrypted, so PostgreSQL stored them too, and such a value now answers `400` there as well. A `ref` is stored only by PostgreSQL's job queue, as `jsonb`, which cannot hold a NUL (SQLSTATE 22P05). Every backend now gives the same answer, as for `q` and `prefix` below. The PostgreSQL store answers `ErrNotFound` for such a key before it runs the query | | The `owner` and `name` of a sync link (`upstream`, `base_repo`), its host (`upstream.host`, `base_repo.upstream_host`) and `base_repo.auth.token`, and a git credential `username` and `password` are plain `type: string` with no `maxLength`. The spec documents no `400` for their length | A sync link whose upstream URL (`https://{host}/{owner}/{name}.git`, or the host as given when it has a scheme, without trailing `/`s) is over 131,070 bytes, and a token, `username` or `password` over 131,042 bytes, answer `400` before anything is stored, with `invalid_params` naming each field ("invalid upstream URL (want at most 131070 bytes)": the `owner`, the `name`, and the host when the body sent one) ([HTTP API](../reference/http-api.md#repository-lifecycle-repowrite--orgread)) | The sync run gives git the URL as one argument and each credential value as one environment entry, and Linux refuses one of 128 KiB or more, counting its terminating NUL and the `/` that git adds to the URL or the name of the variable. Such a link failed every run as `repository configuration error`, and the client learned it only from the failed run. A link or credential stored without the check (before it, or by an older server during an upgrade) still fails its runs that way | | `commit_message` and the `name` and `email` of an `author` or `committer` (`commit-pack`, `diff-commit`, `restore-commit` and `POST /repos/merge`), the `author` of `POST /repos/notes` (also the deprecated body of `DELETE /repos/notes`), and the `query.pattern` of `POST /repos/grep` are plain `type: string`. The spec documents no `400` for their content | Such a value that holds a NUL byte answers `400` ("invalid commit_message (want UTF-8 text without NUL)"). `merge`, `notes` and `grep` name the field in `invalid_params`. `commit-pack` and `diff-commit` answer the result envelope with `status` `invalid`, and `restore-commit` the plain problem body. A merge request refuses the NUL even when no commit is made, as it refuses a newline in an ident | git refuses a NUL in a message, takes a pattern as a process argument and an ident as an environment value, and none of them can hold a NUL. These requests answered `500` | | Every `path` parameter, grep `paths` entry and `query.pattern`, blame `range`, and the `name` and `email` of an `author` or `committer` (commit writes, `merge`, `notes`) is a plain `type: string` with no `maxLength`. The spec documents no `400` for their length | A `path` or `paths` entry over 131,060 bytes, a `range` or `query.pattern` over 131,071 bytes, and a `name` or `email` over 131,051 bytes answer `400` before git starts, with the field and the limit in the message ("invalid path (want at most 131060 bytes)"). `merge`, `notes` and `grep` name the field in `invalid_params`, and `commit-pack` and `diff-commit` answer the result envelope with `status` `invalid`. A `commit_message` has no such limit: git reads it on standard input ([HTTP API](../reference/http-api.md#conventions)) | git takes each of these values as one argument or one environment entry, and Linux refuses one of 128 KiB or more, counting its terminating NUL. These requests answered `500` from 128 KiB on. The limit holds on every platform, so a client gets the same answer from every server | diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 452aa520c..e89f6f704 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -48,7 +48,7 @@ Each setting takes its value from the first source that is set: | [`OPENSTORAGE_TRUSTED_PROXIES`](#openstorage_trusted_proxies) | unset | CIDR allowlist of reverse proxies for rate-limit client identity | | [`OPENSTORAGE_WEBHOOK_ALLOW_PRIVATE`](#openstorage_webhook_allow_private) | `false` | Permit private-network webhook receivers (metadata and redirects stay blocked) | | [`OPENSTORAGE_SYNC_ALLOW_PRIVATE`](#openstorage_sync_allow_private) | unset (transitional) | Unified sync egress policy: permit private-network upstreams for pull and push (two metadata addresses stay blocked) | -| [`OPENSTORAGE_SYNC_PUSH_ALLOW_PRIVATE`](#openstorage_sync_push_allow_private) | `false` | **Deprecated** push-only spelling of the sync egress policy | +| [`OPENSTORAGE_SYNC_PUSH_ALLOW_PRIVATE`](#openstorage_sync_push_allow_private) | `false` | **Deprecated** push-only variable of the sync egress policy | | [`OPENSTORAGE_SECRETS_KEY`](#openstorage_secrets_key) | unset (plaintext) | AES-256 key list for at-rest encryption of stored webhook secrets and upstream credentials | | [`OPENSTORAGE_SECRETS_MODE`](#openstorage_secrets_mode) | `encrypt` | `decrypt-only` stages the key fleet-wide before encrypted writes begin | | [`OPENSTORAGE_PACK_QUOTA_BYTES`](#openstorage_pack_quota_bytes) | 10 GiB | Per-repo storage quota, packs + LFS (`0` disables) | @@ -375,7 +375,7 @@ through an external proxy. ### `OPENSTORAGE_SYNC_PUSH_ALLOW_PRIVATE` -**Deprecated** push-only spelling of the sync egress policy, kept for 0.6.x deployments. +**Deprecated** push-only variable of the sync egress policy, kept for 0.6.x deployments. - **Default:** `false`. - If [`OPENSTORAGE_SYNC_ALLOW_PRIVATE`](#openstorage_sync_allow_private) is unset or diff --git a/internal/api/long_input_test.go b/internal/api/long_input_test.go index 70ef1442e..d45231a7c 100644 --- a/internal/api/long_input_test.go +++ b/internal/api/long_input_test.go @@ -606,19 +606,22 @@ func TestReadValueListsPastGitArgBudget(t *testing.T) { } } - // The list is checked before the repository is looked up and - // before git starts: a list over the budget answers its 400 for - // a repository that does not exist, not 404. + // On the v1 routes the list is checked before the repository is + // looked up and before git starts: a list over the budget answers + // its 400 for a repository that does not exist, not 404. (The + // path-style alias looks the repository up first and answers 404.) org := e.newOrg(t) missing := e.mint(t, tokenSpec{iss: org, repo: "https://example.com/" + org + "/missing", scopes: []string{auth.ScopeGitRead}}) for _, tc := range cases { - suffix, body := tc.req(list(tc.count, gitlimit.MaxArgListBytes+1, tc.value)) - method := http.MethodGet - if body != nil { - method = http.MethodPost - } - resp, out := e.doReq(t, method, "/api/v1/repos"+suffix, missing, body, "") - requireListTooLong(t, resp, out, tc.list, body != nil) + t.Run("missing repository/"+tc.name, func(t *testing.T) { + suffix, body := tc.req(list(tc.count, gitlimit.MaxArgListBytes+1, tc.value)) + method := http.MethodGet + if body != nil { + method = http.MethodPost + } + resp, out := e.doReq(t, method, "/api/v1/repos"+suffix, missing, body, "") + requireListTooLong(t, resp, out, tc.list, body != nil) + }) } // 200,000 entries of one byte fit a 1 MiB body. git got them as diff --git a/internal/api/sync_values_test.go b/internal/api/sync_values_test.go index 8a2142391..fc8037723 100644 --- a/internal/api/sync_values_test.go +++ b/internal/api/sync_values_test.go @@ -40,8 +40,11 @@ func TestSyncValuesTheRunRefusesAtRegistration(t *testing.T) { runBackends(t, func(t *testing.T, e *testEnv) { ctx := context.Background() org := e.newOrg(t) - tok := e.mint(t, tokenSpec{iss: org, repo: "val-" + randomHex(t, 4), scopes: []string{auth.ScopeRepoWrite}}) + // Each case creates a repository of its own name, so a case that + // stores one (a 500 before the fix) cannot turn the next into a + // repeat create. post := func(name, body, wantError string, want ...ipParam) ipCase { + tok := e.mint(t, tokenSpec{iss: org, repo: "val-" + randomHex(t, 4), scopes: []string{auth.ScopeRepoWrite}}) return ipCase{name: name, method: http.MethodPost, path: "/api/v1/repos", token: tok, body: body, wantError: wantError, want: want} } ipRun(t, e, []ipCase{ @@ -69,27 +72,30 @@ func TestSyncValuesTheRunRefusesAtRegistration(t *testing.T) { // The tracked create (CreateLinkedRepo) refuses the same tokens // before it stores a row, and a repeat create of a linked repo - // checks its body first too. - s := newPendingSyncStack(t, e, true) - linked := "val-linked-" + randomHex(t, 4) - if code, _, msg := s.create(t, linked, baseRepo("token")); code != http.StatusCreated { - t.Fatalf("tracked create = %d %q, want 201", code, msg) - } - for _, tc := range []struct{ token, want string }{ - {"to\nken", "invalid base_repo.auth.token (want no carriage return or line feed)"}, - {"enc:token", reserved}, - } { - name := "val-tracked-" + randomHex(t, 4) - if code, _, msg := s.create(t, name, baseRepo(tc.token)); code != http.StatusBadRequest || msg != tc.want { - t.Fatalf("tracked create with token %q = %d %q, want 400 %q", tc.token, code, msg, tc.want) + // checks its body first too. A subtest, so that its failure does + // not skip the credential cases below. + t.Run("tracked and repeat creates", func(t *testing.T) { + s := newPendingSyncStack(t, e, true) + linked := "val-linked-" + randomHex(t, 4) + if code, _, msg := s.create(t, linked, baseRepo("token")); code != http.StatusCreated { + t.Fatalf("tracked create = %d %q, want 201", code, msg) } - if r, err := e.store.GetRepoByURL(ctx, s.org, name); !errors.Is(err, meta.ErrNotFound) { - t.Fatalf("repo after a refused tracked create = %+v, %v; want not found", r, err) - } - if code, _, msg := s.create(t, linked, baseRepo(tc.token)); code != http.StatusBadRequest || msg != tc.want { - t.Fatalf("repeat create with token %q = %d %q, want 400 %q", tc.token, code, msg, tc.want) + for _, tc := range []struct{ token, want string }{ + {"to\nken", "invalid base_repo.auth.token (want no carriage return or line feed)"}, + {"enc:token", reserved}, + } { + name := "val-tracked-" + randomHex(t, 4) + if code, _, msg := s.create(t, name, baseRepo(tc.token)); code != http.StatusBadRequest || msg != tc.want { + t.Errorf("tracked create with token %q = %d %q, want 400 %q", tc.token, code, msg, tc.want) + } + if r, err := e.store.GetRepoByURL(ctx, s.org, name); !errors.Is(err, meta.ErrNotFound) { + t.Errorf("repo after a refused tracked create = %+v, %v; want not found", r, err) + } + if code, _, msg := s.create(t, linked, baseRepo(tc.token)); code != http.StatusBadRequest || msg != tc.want { + t.Errorf("repeat create with token %q = %d %q, want 400 %q", tc.token, code, msg, tc.want) + } } - } + }) repo, err := e.store.CreateRepo(ctx, org, "https://example.com/"+org+"/val-cred-"+randomHex(t, 4), "main") if err != nil { From 3914650ebdf533c1edc4dece326d977e4576584d Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 23:32:52 +0000 Subject: [PATCH 2/3] docs: close the wording points of the review of #901 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - deploy-production.md: the strict webhook policy also denies a cloud metadata endpoint that is a public address (Azure's 168.63.129.16). - configuration.md: the webhook `false` bullet no longer files the metadata endpoints under the IANA special-purpose ranges. - ADR-45: most metadata endpoints lie in the private classes, not all. - guard.go: the note on 168.63.129.16 no longer reads as past tense. - The rest of the code comments, ADR-44 and the boot warning test call OPENSTORAGE_SYNC_PUSH_ALLOW_PRIVATE a variable, not a spelling. Signed-off-by: Thomas Kosiewski --- _Generated with [`xum`](https://github.com/coder/xum) • Model: `anthropic:claude-opus-5-5` • Thinking: `xhigh`_ Change-Id: I8e3b4beb6d4674c53f67f9500ce58b6a2c49d3f0 Signed-off-by: Thomas Kosiewski --- cmd/openstorage/config_test.go | 6 +++--- ...sh-forwarding-rides-a-separate-push-claim-lane.md | 2 +- ...s-policy-is-unified-behind-one-dial-time-guard.md | 12 +++++++----- docs/how-to/deploy-production.md | 4 +++- docs/reference/configuration.md | 7 ++++--- internal/events/guard.go | 3 ++- internal/sync/egress.go | 2 +- internal/sync/worker.go | 2 +- 8 files changed, 22 insertions(+), 16 deletions(-) diff --git a/cmd/openstorage/config_test.go b/cmd/openstorage/config_test.go index 1a926446f..fd9170629 100644 --- a/cmd/openstorage/config_test.go +++ b/cmd/openstorage/config_test.go @@ -1651,7 +1651,7 @@ func TestResolveLFSMaxObjectsPerRepo(t *testing.T) { // TestLogSyncEgressWarnings covers the boot-warning paths (issue #139): // the transitional default announces the future flip, an explicit relaxed -// policy is loud like the webhook hatch, the deprecated spelling names its +// policy is loud like the webhook hatch, the deprecated variable names its // replacement (and whether it was ignored), and the explicit strict policy // boots silently. func TestLogSyncEgressWarnings(t *testing.T) { @@ -1668,11 +1668,11 @@ func TestLogSyncEgressWarnings(t *testing.T) { settings: syncEgressSettings{pushAllowPrivate: true, pullAllowPrivate: true}, want: []string{"sync egress guard relaxed", "shared (RFC 6598)", "never a cloud metadata endpoint", envSyncAllowPrivate}, }, - "deprecated spelling still honored names replacement": { + "deprecated variable still honored names replacement": { settings: syncEgressSettings{pushAllowPrivate: true, pullAllowPrivate: true, pullTransitional: true, legacyPushFlagSet: true}, want: []string{"deprecated env var", envSyncPushAllowPrivate, envSyncAllowPrivate}, }, - "deprecated spelling ignored when unified set": { + "deprecated variable ignored when unified set": { settings: syncEgressSettings{legacyPushFlagSet: true}, want: []string{"deprecated env var ignored", envSyncPushAllowPrivate}, }, diff --git a/docs/adr/044-outbound-push-forwarding-rides-a-separate-push-claim-lane.md b/docs/adr/044-outbound-push-forwarding-rides-a-separate-push-claim-lane.md index 695f2ccfe..e43c56407 100644 --- a/docs/adr/044-outbound-push-forwarding-rides-a-separate-push-claim-lane.md +++ b/docs/adr/044-outbound-push-forwarding-rides-a-separate-push-claim-lane.md @@ -130,7 +130,7 @@ permanently as configuration. The pull lane's egress predates this ADR; [ADR-45](045-sync-egress-policy-is-unified-behind-one-dial-time-guard.md) extends the same dial-time guard to it behind the unified `OPENSTORAGE_SYNC_ALLOW_PRIVATE` policy (this flag became the deprecated -push-only spelling). +push-only variable). **Failure semantics.** The forward itself is atomic (`git push --atomic`): a batch with one refused ref leaves the upstream entirely untouched, so a diff --git a/docs/adr/045-sync-egress-policy-is-unified-behind-one-dial-time-guard.md b/docs/adr/045-sync-egress-policy-is-unified-behind-one-dial-time-guard.md index 7b0e22b69..906a5a1bb 100644 --- a/docs/adr/045-sync-egress-policy-is-unified-behind-one-dial-time-guard.md +++ b/docs/adr/045-sync-egress-policy-is-unified-behind-one-dial-time-guard.md @@ -58,12 +58,14 @@ on resolution errors, never a substitute for dial-time enforcement. Under the transitional default this already rejects cloud metadata and special-purpose targets while admitting private forges. -**Cloud metadata stays denied under every setting.** The private classes -hold the instance-metadata and credential endpoints of the clouds -(`169.254.169.254` in link-local space, Alibaba Cloud's `100.100.100.200` -in RFC 6598 shared space, `fd20:ce::254` in ULA space, and more). The guard +**Cloud metadata stays denied under every setting.** Most of the +instance-metadata and credential endpoints of the clouds lie in the private +classes (`169.254.169.254` in link-local space, Alibaba Cloud's +`100.100.100.200` in RFC 6598 shared space, `fd20:ce::254` in ULA space, and +more); Azure's WireServer, `168.63.129.16`, is a public address. The guard denies each of these single addresses whatever the policy, so admitting a -private class never admits instance credentials. The list and its sources +private class never admits instance credentials, and the strict policy +denies the public one too. The list and its sources are in the [configuration reference](../reference/configuration.md#cloud-metadata-endpoints). diff --git a/docs/how-to/deploy-production.md b/docs/how-to/deploy-production.md index e9c60b0b0..59a53a688 100644 --- a/docs/how-to/deploy-production.md +++ b/docs/how-to/deploy-production.md @@ -497,7 +497,9 @@ Resolution rules: Webhook deliveries use a native SSRF guard. By default, every attempt resolves the registered hostname at dial time and fails closed unless **every** answer is a public, -global-unicast address outside the IANA special-purpose registries. The +global-unicast address outside the IANA special-purpose registries and not a +[cloud metadata endpoint](../reference/configuration.md#cloud-metadata-endpoints), such +as Azure's public `168.63.129.16`. The [reference](../reference/configuration.md#openstorage_webhook_allow_private) lists the denied ranges, including loopback, private, link-local, and cloud metadata targets. diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index fa4a0f0c9..33942f345 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -425,9 +425,10 @@ Controls the native webhook SSRF guard. - **Default:** `false`. Delivery rejects IANA special-purpose ranges, including loopback, RFC 1918, RFC 6598 shared, IPv4/IPv6 link-local, IPv6 ULA, translation/protocol - anycast, documentation, benchmarking, reserved, unspecified, multicast, broadcast, and - the [cloud metadata endpoints](#cloud-metadata-endpoints). IPv4-mapped IPv6 is - normalized first, so `::ffff:10.0.0.1` is private. + anycast, documentation, benchmarking, reserved, unspecified, multicast and broadcast, + and the [cloud metadata endpoints](#cloud-metadata-endpoints), one of which + (`168.63.129.16`) is a public address. IPv4-mapped IPv6 is normalized first, so + `::ffff:10.0.0.1` is private. - **`true`** (Go boolean; logs a startup warning): admits loopback, RFC 1918, RFC 6598 shared, link-local, and ULA receivers. Use it only for intentionally internal webhooks. It **still blocks** the [cloud metadata endpoints](#cloud-metadata-endpoints) (such diff --git a/internal/events/guard.go b/internal/events/guard.go index fa7a7bdec..543e59fd0 100644 --- a/internal/events/guard.go +++ b/internal/events/guard.go @@ -109,7 +109,8 @@ const dialTimeout = 10 * time.Second // Instance Metadata Service as serving metadata and bootstrapping VM // credentials // (learn.microsoft.com/azure/virtual-machines/metadata-security-protocol/overview). -// It is a public address, so the guard denied it under no policy. +// It is a public address, so without this entry no policy would deny +// it. // - 192.0.0.192: Oracle Cloud Infrastructure Classic metadata // (docs.oracle.com/en/cloud/iaas-classic/compute-iaas-cloud/stcsg/retrieving-instance-metadata.html), // inside 192.0.0.0/24, which specialUsePrefixes denies too. diff --git a/internal/sync/egress.go b/internal/sync/egress.go index ab66bc153..f5b4dd141 100644 --- a/internal/sync/egress.go +++ b/internal/sync/egress.go @@ -10,7 +10,7 @@ package sync // pioneered (issue #53) cannot be applied in-process; instead every // http(s) transfer runs through a short-lived loopback forward proxy // whose dials go through TargetGuard.DialContext. The operator escape -// hatch (OPENSTORAGE_SYNC_ALLOW_PRIVATE; deprecated push-only spelling +// hatch (OPENSTORAGE_SYNC_ALLOW_PRIVATE; deprecated push-only variable // OPENSTORAGE_SYNC_PUSH_ALLOW_PRIVATE) relaxes only the private receiver // classes (loopback, RFC 1918, RFC 6598 shared, link-local, ULA), exactly // like the webhook hatch: the cloud metadata endpoints, broadcast, and the diff --git a/internal/sync/worker.go b/internal/sync/worker.go index 7f252af4f..8297ac218 100644 --- a/internal/sync/worker.go +++ b/internal/sync/worker.go @@ -218,7 +218,7 @@ type Worker struct { // PushEgressAllowPrivate relaxes the outbound push egress guard's // private receiver classes (resolved from OPENSTORAGE_SYNC_ALLOW_PRIVATE // with OPENSTORAGE_SYNC_PUSH_ALLOW_PRIVATE as the deprecated push-only - // spelling, see egress.go): loopback, RFC 1918, RFC 6598 shared, + // variable, see egress.go): loopback, RFC 1918, RFC 6598 shared, // link-local and ULA upstreams for genuinely internal forges. The // cloud metadata endpoints and the other special-purpose ranges stay // denied either way — every http(s) push dials through From 469bed430a2fd26ae2ea6e1b4a275b7032b85a69 Mon Sep 17 00:00:00 2001 From: Thomas Kosiewski Date: Mon, 16 Nov 2026 23:52:53 +0000 Subject: [PATCH 3/3] test(api): read each commit route's envelope statuses in the unknown-op test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The review of #902 noted that the test of wcCheckRefPolicy wrapped the recorder in an envelopeWriter with no statuses, so it would keep passing if a commit write route later answered 403 in its result envelope. It now runs once for each route in commitWriteEnvelopeStatuses. The errtext README lists the unknown-op callers and the cut parse error of CheckRegistration. Signed-off-by: Thomas Kosiewski --- _Generated with [`xum`](https://github.com/coder/xum) • Model: `anthropic:claude-opus-5-5` • Thinking: `xhigh`_ Change-Id: I65a72822779feee84988fa6b8544a60a2aa509af Signed-off-by: Thomas Kosiewski --- internal/api/echo_bounds_test.go | 23 +++++++++++++---------- internal/errtext/README.md | 3 ++- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/internal/api/echo_bounds_test.go b/internal/api/echo_bounds_test.go index 5eae02ed8..fa22849cf 100644 --- a/internal/api/echo_bounds_test.go +++ b/internal/api/echo_bounds_test.go @@ -111,18 +111,21 @@ func TestUnknownPolicyOpQuotedBounded(t *testing.T) { t.Fatalf("wrefPolicyCheck = %.300q, %v; want %.300q", deny, err, want) } - // A commit write route answers a 403 in a problem body - // (commitWriteEnvelopeStatuses lists no 403). - rec := httptest.NewRecorder() - if wcCheckRefPolicy(&envelopeWriter{ResponseWriter: rec}, claims, refName) { - t.Fatal("wcCheckRefPolicy admitted an unknown op") - } - var wc problemBody - if err := json.Unmarshal(rec.Body.Bytes(), &wc); err != nil || rec.Code != http.StatusForbidden || wc.Detail != want { - t.Fatalf("wcCheckRefPolicy: %d %.300s; want 403 %.300q", rec.Code, rec.Body.Bytes(), want) + // Each commit write route answers a 403 in a problem body: + // its envelope statuses (commitWriteEnvelopeStatuses) hold no + // 403. + for path, statuses := range commitWriteEnvelopeStatuses { + rec := httptest.NewRecorder() + if wcCheckRefPolicy(&envelopeWriter{ResponseWriter: rec, statuses: statuses}, claims, refName) { + t.Fatalf("%s: wcCheckRefPolicy admitted an unknown op", path) + } + var wc problemBody + if err := json.Unmarshal(rec.Body.Bytes(), &wc); err != nil || rec.Code != http.StatusForbidden || wc.Detail != want { + t.Fatalf("%s: wcCheckRefPolicy: %d %.300s; want 403 %.300q", path, rec.Code, rec.Body.Bytes(), want) + } } - rec = httptest.NewRecorder() + rec := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/api/v1/repos/merge", nil).WithContext(auth.WithClaims(t.Context(), claims)) if (&server{}).wmCheckRefPolicy(rec, r, nil, refName, ref.ZeroSHA, newSHA, false, "main") { t.Fatal("wmCheckRefPolicy admitted an unknown op") diff --git a/internal/errtext/README.md b/internal/errtext/README.md index 056a5f372..f7da8d685 100644 --- a/internal/errtext/README.md +++ b/internal/errtext/README.md @@ -18,7 +18,8 @@ Bounds the request text that an error answer repeats, so that an answer stays sm - `api` cuts `error`, `detail` and each `invalid_params` name of a problem body (`newProblem`), and the `result.message` of a commit write (`wcRefUpdate`) [../api/errors.go:newProblem] [../api/handlers_write_commit.go:wcRefUpdate] - `api` quotes an unknown member name in an answer that names one unknown member (`unknownNameText`), and cuts each unknown member name at 32 bytes in the answer of a strictly decoded body that names several members (`Cut`) [../api/decode_body.go:unknownNameText] [../api/handlers_write_refs.go:strictProblem.err] - `api` quotes a refused sync provider, operation, access or host (`upstreamConfig.validate`, `validateSyncBaseRepo`) and the ref policy op of a token in a 403 (`refPolicyOpDenies`) [../api/handlers_sync.go:upstreamConfig.validate] [../api/handlers_sync.go:refPolicyOpDenies] -- `events` quotes a webhook URL or its host in a registration refusal (`CheckRegistration`) [../events/guard.go:CheckRegistration] +- `api` quotes a ref policy op that a ref, commit or merge write does not know (`unknown policy op "x" for `) [../api/handlers_write_refs.go:wrefPolicyCheck] [../api/handlers_write_commit.go:wcCheckRefPolicy] [../api/handlers_write_merge.go:server.wmCheckRefPolicy] +- `events` quotes a webhook URL or its host in a registration refusal, and cuts the parse error it adds after a URL that does not parse (`CheckRegistration`) [../events/guard.go:CheckRegistration] - `githttp` cuts the error text of the git and LFS routes [../githttp/server.go:httpError] [../githttp/lfs.go:lfsError] ## Verification