diff --git a/cmd/openstorage/README.md b/cmd/openstorage/README.md index 17797f9d..a7a59899 100644 --- a/cmd/openstorage/README.md +++ b/cmd/openstorage/README.md @@ -96,7 +96,7 @@ The `hook` subprocess also reads per-push variables that the Git server sets (na - `[config.go:resolveSweepSettings]` `OPENSTORAGE_REPO_RETENTION`, when non-zero, must be `>= jobs.MinRepoPurgeAge` and needs an active sweep. - `[config.go:resolveLFSGCSettings]` Grace and object budget must be `> 0`. Delete mode needs a non-zero interval. - `[config.go:resolveRepackThreshold]` The threshold must be `0` or `>= 2`. -- `[config.go:resolveSyncEgress]` `OPENSTORAGE_SYNC_ALLOW_PRIVATE`, when set to a non-empty value, governs both push and pull. A malformed value in either spelling is an error. +- `[config.go:resolveSyncEgress]` `OPENSTORAGE_SYNC_ALLOW_PRIVATE`, when set to a non-empty value, governs both push and pull. A malformed value in either variable is an error. - `[config.go:resolveSecretsCodec]` `decrypt-only` requires a configured key. - `[config.go:normalizeBaseURL]` The base URL must be `http` or `https` with a host and no query or fragment. - `[config.go:assertf]` Impossible states panic with `openstorage assertion failed:`. Input errors return errors. diff --git a/cmd/openstorage/config.go b/cmd/openstorage/config.go index 28d08309..29786a2c 100644 --- a/cmd/openstorage/config.go +++ b/cmd/openstorage/config.go @@ -198,13 +198,13 @@ const ( // metadata endpoints and the other special-purpose ranges stay denied); // "false" denies the private classes in both directions. Unset is the // transitional compatibility default: pushes stay strict (the - // deprecated per-direction spelling below may relax them) while pulls + // deprecated push-only variable below may relax them) while pulls // ALLOW private upstreams with a boot warning — internal-forge // mirrors are a mainstream released 0.6.x deployment shape, and a // future release flips the unset pull default to deny. envSyncAllowPrivate = "OPENSTORAGE_SYNC_ALLOW_PRIVATE" // envSyncPushAllowPrivate relaxes the outbound push egress guard's - // private receiver classes (issue #90). Deprecated spelling: it stays + // private receiver classes (issue #90). Deprecated variable: it stays // push-only (its historical meaning), and envSyncAllowPrivate wins // for both directions when set. envSyncPushAllowPrivate = "OPENSTORAGE_SYNC_PUSH_ALLOW_PRIVATE" @@ -1184,17 +1184,17 @@ type syncEgressSettings struct { // is unset, so pulls allow private upstreams (released 0.6.x behavior) // and boot warns about the future default flip to deny. pullTransitional bool - // legacyPushFlagSet records that the deprecated push-only spelling was + // legacyPushFlagSet records that the deprecated push-only variable was // present, for the boot-time deprecation notice (when the unified knob // is also set the legacy value is ignored entirely). legacyPushFlagSet bool } // resolveSyncEgress resolves the unified sync egress policy (issue #139). -// Both spellings parse fail-fast even when losing (a malformed boolean is +// Both variables parse fail-fast even when losing (a malformed boolean is // a boot error, never a silent fallback); the unified knob wins for both // directions. Unset keeps the transitional compatibility default: strict -// pushes (unless the deprecated push-only spelling relaxes them) and +// pushes (unless the deprecated push-only variable relaxes them) and // allow-private pulls, with wire.go warning about the future flip. func resolveSyncEgress(getenv func(string) string) (syncEgressSettings, error) { assertf(getenv != nil, "resolveSyncEgress: nil getenv") diff --git a/cmd/openstorage/config_test.go b/cmd/openstorage/config_test.go index 874f0049..fd917062 100644 --- a/cmd/openstorage/config_test.go +++ b/cmd/openstorage/config_test.go @@ -1344,8 +1344,8 @@ func TestResolveWebhookAllowPrivate(t *testing.T) { // TestResolveSyncEgress covers the unified sync egress policy resolution // (issue #139): the unified knob governs both directions when set; unset // keeps the transitional compatibility default (strict pushes unless the -// deprecated push-only spelling relaxes them, allow-private pulls with the -// boot warning); malformed values are boot errors for BOTH spellings. +// deprecated push-only variable relaxes them, allow-private pulls with the +// boot warning); malformed values are boot errors for BOTH variables. func TestResolveSyncEgress(t *testing.T) { for name, tc := range map[string]struct { env map[string]string @@ -1406,6 +1406,46 @@ func TestResolveSyncEgress(t *testing.T) { } } +// TestResolveServeConfigEmptySyncAllowPrivate: serve reads an +// OPENSTORAGE_SYNC_ALLOW_PRIVATE that is present but empty as unset, so the +// deprecated push-only variable still decides pushes and pulls keep the +// transitional default. resolveSyncEgress cannot tell an empty value from +// an absent one; this is the layer where the environment can. +func TestResolveServeConfigEmptySyncAllowPrivate(t *testing.T) { + for name, tc := range map[string]struct { + env map[string]string + want syncEgressSettings + }{ + "empty unified, legacy true": { + env: map[string]string{envSyncAllowPrivate: "", envSyncPushAllowPrivate: "true"}, + want: syncEgressSettings{pushAllowPrivate: true, pullAllowPrivate: true, pullTransitional: true, legacyPushFlagSet: true}, + }, + "empty unified, no legacy": { + env: map[string]string{envSyncAllowPrivate: ""}, + want: syncEgressSettings{pullAllowPrivate: true, pullTransitional: true}, + }, + "unified false wins over legacy true": { + env: map[string]string{envSyncAllowPrivate: "false", envSyncPushAllowPrivate: "true"}, + want: syncEgressSettings{legacyPushFlagSet: true}, + }, + } { + t.Run(name, func(t *testing.T) { + env := map[string]string{ + "OPENSTORAGE_ORG": "org1", + "OPENSTORAGE_ORG_PUBKEYS": "/tmp/org.pub.pem", + } + maps.Copy(env, tc.env) + cfg, err := resolveServeConfig(mapEnv(env), "", "127.0.0.1:0", "/tmp/cache", t.TempDir()) + if err != nil { + t.Fatal(err) + } + if cfg.syncEgress != tc.want { + t.Fatalf("syncEgress = %+v, want %+v", cfg.syncEgress, tc.want) + } + }) + } +} + func TestResolveSecretsCodec(t *testing.T) { // 32 zero bytes, base64 — a syntactically valid AES-256 key. validKey := base64.StdEncoding.EncodeToString(make([]byte, 32)) @@ -1611,7 +1651,7 @@ func TestResolveLFSMaxObjectsPerRepo(t *testing.T) { // TestLogSyncEgressWarnings covers the boot-warning paths (issue #139): // the transitional default announces the future flip, an explicit relaxed -// policy is loud like the webhook hatch, the deprecated spelling names its +// policy is loud like the webhook hatch, the deprecated variable names its // replacement (and whether it was ignored), and the explicit strict policy // boots silently. func TestLogSyncEgressWarnings(t *testing.T) { @@ -1628,11 +1668,11 @@ func TestLogSyncEgressWarnings(t *testing.T) { settings: syncEgressSettings{pushAllowPrivate: true, pullAllowPrivate: true}, want: []string{"sync egress guard relaxed", "shared (RFC 6598)", "never a cloud metadata endpoint", envSyncAllowPrivate}, }, - "deprecated spelling still honored names replacement": { + "deprecated variable still honored names replacement": { settings: syncEgressSettings{pushAllowPrivate: true, pullAllowPrivate: true, pullTransitional: true, legacyPushFlagSet: true}, want: []string{"deprecated env var", envSyncPushAllowPrivate, envSyncAllowPrivate}, }, - "deprecated spelling ignored when unified set": { + "deprecated variable ignored when unified set": { settings: syncEgressSettings{legacyPushFlagSet: true}, want: []string{"deprecated env var ignored", envSyncPushAllowPrivate}, }, diff --git a/cmd/openstorage/wire.go b/cmd/openstorage/wire.go index 52823b7c..ee90e0dc 100644 --- a/cmd/openstorage/wire.go +++ b/cmd/openstorage/wire.go @@ -38,7 +38,7 @@ import ( // logSyncEgressWarnings emits the boot-time sync egress policy notices // (issue #139) beside the webhook hatch warning: a relaxed guard must never // be silent, the transitional pull default announces its future flip to -// deny, and the deprecated push-only spelling names its replacement. +// deny, and the deprecated push-only variable names its replacement. func logSyncEgressWarnings(logger *slog.Logger, s syncEgressSettings) { assertf(logger != nil, "logSyncEgressWarnings: nil logger") switch { diff --git a/docs/adr/044-outbound-push-forwarding-rides-a-separate-push-claim-lane.md b/docs/adr/044-outbound-push-forwarding-rides-a-separate-push-claim-lane.md index 695f2ccf..e43c5640 100644 --- a/docs/adr/044-outbound-push-forwarding-rides-a-separate-push-claim-lane.md +++ b/docs/adr/044-outbound-push-forwarding-rides-a-separate-push-claim-lane.md @@ -130,7 +130,7 @@ permanently as configuration. The pull lane's egress predates this ADR; [ADR-45](045-sync-egress-policy-is-unified-behind-one-dial-time-guard.md) extends the same dial-time guard to it behind the unified `OPENSTORAGE_SYNC_ALLOW_PRIVATE` policy (this flag became the deprecated -push-only spelling). +push-only variable). **Failure semantics.** The forward itself is atomic (`git push --atomic`): a batch with one refused ref leaves the upstream entirely untouched, so a diff --git a/docs/adr/045-sync-egress-policy-is-unified-behind-one-dial-time-guard.md b/docs/adr/045-sync-egress-policy-is-unified-behind-one-dial-time-guard.md index 317ef280..906a5a1b 100644 --- a/docs/adr/045-sync-egress-policy-is-unified-behind-one-dial-time-guard.md +++ b/docs/adr/045-sync-egress-policy-is-unified-behind-one-dial-time-guard.md @@ -31,14 +31,14 @@ boot warning announcing the future flip of that default to deny. Syncing from internal RFC 1918 gitea/forgejo/GitLab forges is a mainstream released 0.6.x deployment shape; default-denying it is a breaking change that gets a deprecation window, not a surprise. `OPENSTORAGE_SYNC_PUSH_ALLOW_PRIVATE` -becomes the deprecated push-only spelling: still honored (its historical +becomes the deprecated push-only variable: still honored (its historical meaning, push only) when the unified knob is unset, ignored with a boot notice when it is set. One knob was chosen over per-direction flags because the threat and the deployment shape are symmetric — an internal-forge deployment wants both directions private, a hardened deployment wants neither — and a second per-direction flag would double the migration story for no expressible policy anyone asked for. Malformed booleans in either -spelling are boot errors. +variable are boot errors. **Ambient proxies no longer apply to sync transfers.** The pull fetch previously inherited `HTTP_PROXY`/`HTTPS_PROXY` from the environment; the @@ -58,12 +58,14 @@ on resolution errors, never a substitute for dial-time enforcement. Under the transitional default this already rejects cloud metadata and special-purpose targets while admitting private forges. -**Cloud metadata stays denied under every setting.** The private classes -hold the instance-metadata and credential endpoints of the clouds -(`169.254.169.254` in link-local space, Alibaba Cloud's `100.100.100.200` -in RFC 6598 shared space, `fd20:ce::254` in ULA space, and more). The guard +**Cloud metadata stays denied under every setting.** Most of the +instance-metadata and credential endpoints of the clouds lie in the private +classes (`169.254.169.254` in link-local space, Alibaba Cloud's +`100.100.100.200` in RFC 6598 shared space, `fd20:ce::254` in ULA space, and +more); Azure's WireServer, `168.63.129.16`, is a public address. The guard denies each of these single addresses whatever the policy, so admitting a -private class never admits instance credentials. The list and its sources +private class never admits instance credentials, and the strict policy +denies the public one too. The list and its sources are in the [configuration reference](../reference/configuration.md#cloud-metadata-endpoints). @@ -72,5 +74,5 @@ gain the dial-time guard with private targets still allowed by default; `serve` warns at boot until `OPENSTORAGE_SYNC_ALLOW_PRIVATE` is set explicitly. Set `true` to keep internal-forge sync across the future flip, `false` to adopt strict mode early. A future minor release flips the unset -pull default to deny; the deprecated push-only spelling is removed no +pull default to deny; the deprecated push-only variable is removed no earlier than that flip. diff --git a/docs/explanation/wire-compatibility.md b/docs/explanation/wire-compatibility.md index ce6513e8..2ee9c3cb 100644 --- a/docs/explanation/wire-compatibility.md +++ b/docs/explanation/wire-compatibility.md @@ -245,7 +245,7 @@ the [archived phase notes](../archive/phases/)): | A token that fails verification (malformed, bad signature, expired, unknown issuer): upstream's live docs and servers answer `403` `forbidden` (`Invalid or expired token`) on the REST API, and `403` on the Git and LFS transports too (checked 2026-11-04). The vendored spec documents no status for it | The REST API answers `403` `forbidden` (`Invalid or expired token`), as upstream does ([#664](https://github.com/coder/openstorage/issues/664)). The Git and LFS transports answer `401` with `WWW-Authenticate: Basic`, as for a missing token ([authentication](../reference/authentication.md#failure-mapping)) | Git asks for credentials only after a `401`, and only a `401` makes it erase a stored credential. After a `403`, git fails and keeps the expired token in the credential helper, so every later command fails the same way until the user removes the token | | A `crit` member in the JWT header: the vendored docs and spec say nothing about it. They list `alg` and `typ` as the header ([authentication guide](../upstream/getting-started/authentication.md)) | A token whose header has a `crit` member of any value fails verification, even when the signature is valid. The status is the one for any token that fails verification (the row above) | RFC 7515 §4.1.11: a recipient that does not understand a listed extension must refuse the token. open.storage understands none, so it refuses any `crit` | | Every path field and `path` parameter is a plain `type: string`. The spec says nothing about tree entry names whose bytes are not valid UTF-8 | A response path that is not valid UTF-8 still lists with one U+FFFD per invalid byte, and gets an extra sibling `_base64` with the raw bytes in standard base64 (for example `path_base64`, `paths_base64`, `old_path_base64`, `original_path_base64`). The sibling is absent for valid UTF-8 paths, so those responses are unchanged. This covers `files`, `files/metadata`, the diff routes, `blame` and `grep`. The merge conflict paths do not have it yet, and JSON request bodies cannot name such a file yet. The `path` query parameter takes the raw bytes percent-encoded ([path encoding](../reference/http-api.md#conventions)) | JSON strings hold Unicode text, so such a byte has no exact JSON form. A sibling field (like GitHub's `pathRaw`) is additive: no SDK reads it, and the SDKs ignore response keys they do not know. If upstream documents another form, we carry both ([#587](https://github.com/coder/openstorage/issues/587)) | -| Text the request asks the server to store (a webhook `secret`, a git credential `username` and `password`, the `owner`, `name` and `auth.token` of a sync link, a `pull-upstream` `ref`) is plain `type: string`. The spec documents no `400` for its content | Such a value that holds a NUL byte answers `400` `bad_request` ("invalid secret (want UTF-8 text without NUL)", with `invalid_params` naming the field where the route has them), on every backend. A `GET /repos` `cursor` whose repo ID holds a NUL byte or invalid UTF-8 answers the `400` of an undecodable cursor. A path or body ID with such text (a webhook or git credential `id`) answers the `404` of an unknown ID. A git credential `username` or `password`, or a `base_repo.auth.token`, with a carriage return or line feed answers `400` too, naming the field: the sync run hands the pair to git through a credential helper that reads one value per line, and failed such a credential on every run as `repository configuration error` | PostgreSQL refuses a NUL byte or invalid UTF-8 in a text parameter (SQLSTATE 22021) and answered `500` for most of these values, while SQLite and S3-WAL stored them. With `OPENSTORAGE_SECRETS_KEY` set in the default `encrypt` mode, the `secret`, `password` and `auth.token` reach PostgreSQL encrypted, so PostgreSQL stored them too, and such a value now answers `400` there as well. A `ref` is stored only by PostgreSQL's job queue, as `jsonb`, which cannot hold a NUL (SQLSTATE 22P05). Every backend now gives the same answer, as for `q` and `prefix` below. The PostgreSQL store answers `ErrNotFound` for such a key before it runs the query | +| Text the request asks the server to store (a webhook `secret`, a git credential `username` and `password`, the `owner`, `name` and `auth.token` of a sync link, a `pull-upstream` `ref`) is plain `type: string`. The spec documents no `400` for its content | Such a value that holds a NUL byte answers `400` `bad_request` ("invalid secret (want UTF-8 text without NUL)", with `invalid_params` naming the field where the route has them), on every backend. A `GET /repos` `cursor` whose repo ID holds a NUL byte or invalid UTF-8 answers the `400` of an undecodable cursor. A path or body ID with such text (a webhook or git credential `id`) answers the `404` of an unknown ID. A git credential `username` or `password`, or the `base_repo.auth.token` of a sync create, with a carriage return or line feed answers `400` too, naming the field: the sync run hands the pair to git through a credential helper that reads one value per line, and failed such a credential on every run as `repository configuration error` | PostgreSQL refuses a NUL byte or invalid UTF-8 in a text parameter (SQLSTATE 22021) and answered `500` for most of these values, while SQLite and S3-WAL stored them. With `OPENSTORAGE_SECRETS_KEY` set in the default `encrypt` mode, the `secret`, `password` and `auth.token` reach PostgreSQL encrypted, so PostgreSQL stored them too, and such a value now answers `400` there as well. A `ref` is stored only by PostgreSQL's job queue, as `jsonb`, which cannot hold a NUL (SQLSTATE 22P05). Every backend now gives the same answer, as for `q` and `prefix` below. The PostgreSQL store answers `ErrNotFound` for such a key before it runs the query | | The `owner` and `name` of a sync link (`upstream`, `base_repo`), its host (`upstream.host`, `base_repo.upstream_host`) and `base_repo.auth.token`, and a git credential `username` and `password` are plain `type: string` with no `maxLength`. The spec documents no `400` for their length | A sync link whose upstream URL (`https://{host}/{owner}/{name}.git`, or the host as given when it has a scheme, without trailing `/`s) is over 131,070 bytes, and a token, `username` or `password` over 131,042 bytes, answer `400` before anything is stored, with `invalid_params` naming each field ("invalid upstream URL (want at most 131070 bytes)": the `owner`, the `name`, and the host when the body sent one) ([HTTP API](../reference/http-api.md#repository-lifecycle-repowrite--orgread)) | The sync run gives git the URL as one argument and each credential value as one environment entry, and Linux refuses one of 128 KiB or more, counting its terminating NUL and the `/` that git adds to the URL or the name of the variable. Such a link failed every run as `repository configuration error`, and the client learned it only from the failed run. A link or credential stored without the check (before it, or by an older server during an upgrade) still fails its runs that way | | `commit_message` and the `name` and `email` of an `author` or `committer` (`commit-pack`, `diff-commit`, `restore-commit` and `POST /repos/merge`), the `author` of `POST /repos/notes` (also the deprecated body of `DELETE /repos/notes`), and the `query.pattern` of `POST /repos/grep` are plain `type: string`. The spec documents no `400` for their content | Such a value that holds a NUL byte answers `400` ("invalid commit_message (want UTF-8 text without NUL)"). `merge`, `notes` and `grep` name the field in `invalid_params`. `commit-pack` and `diff-commit` answer the result envelope with `status` `invalid`, and `restore-commit` the plain problem body. A merge request refuses the NUL even when no commit is made, as it refuses a newline in an ident | git refuses a NUL in a message, takes a pattern as a process argument and an ident as an environment value, and none of them can hold a NUL. These requests answered `500` | | Every `path` parameter, grep `paths` entry and `query.pattern`, blame `range`, and the `name` and `email` of an `author` or `committer` (commit writes, `merge`, `notes`) is a plain `type: string` with no `maxLength`. The spec documents no `400` for their length | A `path` or `paths` entry over 131,060 bytes, a `range` or `query.pattern` over 131,071 bytes, and a `name` or `email` over 131,051 bytes answer `400` before git starts, with the field and the limit in the message ("invalid path (want at most 131060 bytes)"). `merge`, `notes` and `grep` name the field in `invalid_params`, and `commit-pack` and `diff-commit` answer the result envelope with `status` `invalid`. A `commit_message` has no such limit: git reads it on standard input ([HTTP API](../reference/http-api.md#conventions)) | git takes each of these values as one argument or one environment entry, and Linux refuses one of 128 KiB or more, counting its terminating NUL. These requests answered `500` from 128 KiB on. The limit holds on every platform, so a client gets the same answer from every server | diff --git a/docs/how-to/deploy-production.md b/docs/how-to/deploy-production.md index e9c60b0b..59a53a68 100644 --- a/docs/how-to/deploy-production.md +++ b/docs/how-to/deploy-production.md @@ -497,7 +497,9 @@ Resolution rules: Webhook deliveries use a native SSRF guard. By default, every attempt resolves the registered hostname at dial time and fails closed unless **every** answer is a public, -global-unicast address outside the IANA special-purpose registries. The +global-unicast address outside the IANA special-purpose registries and not a +[cloud metadata endpoint](../reference/configuration.md#cloud-metadata-endpoints), such +as Azure's public `168.63.129.16`. The [reference](../reference/configuration.md#openstorage_webhook_allow_private) lists the denied ranges, including loopback, private, link-local, and cloud metadata targets. diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index b838589a..33942f34 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -48,7 +48,7 @@ Each setting takes its value from the first source that is set: | [`OPENSTORAGE_TRUSTED_PROXIES`](#openstorage_trusted_proxies) | unset | CIDR allowlist of reverse proxies for rate-limit client identity | | [`OPENSTORAGE_WEBHOOK_ALLOW_PRIVATE`](#openstorage_webhook_allow_private) | `false` | Permit private-network webhook receivers (cloud metadata endpoints and redirects stay blocked) | | [`OPENSTORAGE_SYNC_ALLOW_PRIVATE`](#openstorage_sync_allow_private) | unset (transitional) | Unified sync egress policy: permit private-network upstreams for pull and push (cloud metadata endpoints stay blocked) | -| [`OPENSTORAGE_SYNC_PUSH_ALLOW_PRIVATE`](#openstorage_sync_push_allow_private) | `false` | **Deprecated** push-only spelling of the sync egress policy | +| [`OPENSTORAGE_SYNC_PUSH_ALLOW_PRIVATE`](#openstorage_sync_push_allow_private) | `false` | **Deprecated** push-only variable of the sync egress policy | | [`OPENSTORAGE_SECRETS_KEY`](#openstorage_secrets_key) | unset (plaintext) | AES-256 key list for at-rest encryption of stored webhook secrets and upstream credentials | | [`OPENSTORAGE_SECRETS_MODE`](#openstorage_secrets_mode) | `encrypt` | `decrypt-only` stages the key fleet-wide before encrypted writes begin | | [`OPENSTORAGE_PACK_QUOTA_BYTES`](#openstorage_pack_quota_bytes) | 10 GiB | Per-repo storage quota, packs + LFS (`0` disables) | @@ -409,7 +409,7 @@ through an external proxy. ### `OPENSTORAGE_SYNC_PUSH_ALLOW_PRIVATE` -**Deprecated** push-only spelling of the sync egress policy, kept for 0.6.x deployments. +**Deprecated** push-only variable of the sync egress policy, kept for 0.6.x deployments. - **Default:** `false`. - If [`OPENSTORAGE_SYNC_ALLOW_PRIVATE`](#openstorage_sync_allow_private) is unset or @@ -425,9 +425,10 @@ Controls the native webhook SSRF guard. - **Default:** `false`. Delivery rejects IANA special-purpose ranges, including loopback, RFC 1918, RFC 6598 shared, IPv4/IPv6 link-local, IPv6 ULA, translation/protocol - anycast, documentation, benchmarking, reserved, unspecified, multicast, broadcast, and - the [cloud metadata endpoints](#cloud-metadata-endpoints). IPv4-mapped IPv6 is - normalized first, so `::ffff:10.0.0.1` is private. + anycast, documentation, benchmarking, reserved, unspecified, multicast and broadcast, + and the [cloud metadata endpoints](#cloud-metadata-endpoints), one of which + (`168.63.129.16`) is a public address. IPv4-mapped IPv6 is normalized first, so + `::ffff:10.0.0.1` is private. - **`true`** (Go boolean; logs a startup warning): admits loopback, RFC 1918, RFC 6598 shared, link-local, and ULA receivers. Use it only for intentionally internal webhooks. It **still blocks** the [cloud metadata endpoints](#cloud-metadata-endpoints) (such diff --git a/internal/api/echo_bounds_test.go b/internal/api/echo_bounds_test.go index 5eae02ed..fa22849c 100644 --- a/internal/api/echo_bounds_test.go +++ b/internal/api/echo_bounds_test.go @@ -111,18 +111,21 @@ func TestUnknownPolicyOpQuotedBounded(t *testing.T) { t.Fatalf("wrefPolicyCheck = %.300q, %v; want %.300q", deny, err, want) } - // A commit write route answers a 403 in a problem body - // (commitWriteEnvelopeStatuses lists no 403). - rec := httptest.NewRecorder() - if wcCheckRefPolicy(&envelopeWriter{ResponseWriter: rec}, claims, refName) { - t.Fatal("wcCheckRefPolicy admitted an unknown op") - } - var wc problemBody - if err := json.Unmarshal(rec.Body.Bytes(), &wc); err != nil || rec.Code != http.StatusForbidden || wc.Detail != want { - t.Fatalf("wcCheckRefPolicy: %d %.300s; want 403 %.300q", rec.Code, rec.Body.Bytes(), want) + // Each commit write route answers a 403 in a problem body: + // its envelope statuses (commitWriteEnvelopeStatuses) hold no + // 403. + for path, statuses := range commitWriteEnvelopeStatuses { + rec := httptest.NewRecorder() + if wcCheckRefPolicy(&envelopeWriter{ResponseWriter: rec, statuses: statuses}, claims, refName) { + t.Fatalf("%s: wcCheckRefPolicy admitted an unknown op", path) + } + var wc problemBody + if err := json.Unmarshal(rec.Body.Bytes(), &wc); err != nil || rec.Code != http.StatusForbidden || wc.Detail != want { + t.Fatalf("%s: wcCheckRefPolicy: %d %.300s; want 403 %.300q", path, rec.Code, rec.Body.Bytes(), want) + } } - rec = httptest.NewRecorder() + rec := httptest.NewRecorder() r := httptest.NewRequest(http.MethodPost, "/api/v1/repos/merge", nil).WithContext(auth.WithClaims(t.Context(), claims)) if (&server{}).wmCheckRefPolicy(rec, r, nil, refName, ref.ZeroSHA, newSHA, false, "main") { t.Fatal("wmCheckRefPolicy admitted an unknown op") diff --git a/internal/api/long_input_test.go b/internal/api/long_input_test.go index 70ef1442..d45231a7 100644 --- a/internal/api/long_input_test.go +++ b/internal/api/long_input_test.go @@ -606,19 +606,22 @@ func TestReadValueListsPastGitArgBudget(t *testing.T) { } } - // The list is checked before the repository is looked up and - // before git starts: a list over the budget answers its 400 for - // a repository that does not exist, not 404. + // On the v1 routes the list is checked before the repository is + // looked up and before git starts: a list over the budget answers + // its 400 for a repository that does not exist, not 404. (The + // path-style alias looks the repository up first and answers 404.) org := e.newOrg(t) missing := e.mint(t, tokenSpec{iss: org, repo: "https://example.com/" + org + "/missing", scopes: []string{auth.ScopeGitRead}}) for _, tc := range cases { - suffix, body := tc.req(list(tc.count, gitlimit.MaxArgListBytes+1, tc.value)) - method := http.MethodGet - if body != nil { - method = http.MethodPost - } - resp, out := e.doReq(t, method, "/api/v1/repos"+suffix, missing, body, "") - requireListTooLong(t, resp, out, tc.list, body != nil) + t.Run("missing repository/"+tc.name, func(t *testing.T) { + suffix, body := tc.req(list(tc.count, gitlimit.MaxArgListBytes+1, tc.value)) + method := http.MethodGet + if body != nil { + method = http.MethodPost + } + resp, out := e.doReq(t, method, "/api/v1/repos"+suffix, missing, body, "") + requireListTooLong(t, resp, out, tc.list, body != nil) + }) } // 200,000 entries of one byte fit a 1 MiB body. git got them as diff --git a/internal/api/sync_values_test.go b/internal/api/sync_values_test.go index 8a214239..fc803772 100644 --- a/internal/api/sync_values_test.go +++ b/internal/api/sync_values_test.go @@ -40,8 +40,11 @@ func TestSyncValuesTheRunRefusesAtRegistration(t *testing.T) { runBackends(t, func(t *testing.T, e *testEnv) { ctx := context.Background() org := e.newOrg(t) - tok := e.mint(t, tokenSpec{iss: org, repo: "val-" + randomHex(t, 4), scopes: []string{auth.ScopeRepoWrite}}) + // Each case creates a repository of its own name, so a case that + // stores one (a 500 before the fix) cannot turn the next into a + // repeat create. post := func(name, body, wantError string, want ...ipParam) ipCase { + tok := e.mint(t, tokenSpec{iss: org, repo: "val-" + randomHex(t, 4), scopes: []string{auth.ScopeRepoWrite}}) return ipCase{name: name, method: http.MethodPost, path: "/api/v1/repos", token: tok, body: body, wantError: wantError, want: want} } ipRun(t, e, []ipCase{ @@ -69,27 +72,30 @@ func TestSyncValuesTheRunRefusesAtRegistration(t *testing.T) { // The tracked create (CreateLinkedRepo) refuses the same tokens // before it stores a row, and a repeat create of a linked repo - // checks its body first too. - s := newPendingSyncStack(t, e, true) - linked := "val-linked-" + randomHex(t, 4) - if code, _, msg := s.create(t, linked, baseRepo("token")); code != http.StatusCreated { - t.Fatalf("tracked create = %d %q, want 201", code, msg) - } - for _, tc := range []struct{ token, want string }{ - {"to\nken", "invalid base_repo.auth.token (want no carriage return or line feed)"}, - {"enc:token", reserved}, - } { - name := "val-tracked-" + randomHex(t, 4) - if code, _, msg := s.create(t, name, baseRepo(tc.token)); code != http.StatusBadRequest || msg != tc.want { - t.Fatalf("tracked create with token %q = %d %q, want 400 %q", tc.token, code, msg, tc.want) + // checks its body first too. A subtest, so that its failure does + // not skip the credential cases below. + t.Run("tracked and repeat creates", func(t *testing.T) { + s := newPendingSyncStack(t, e, true) + linked := "val-linked-" + randomHex(t, 4) + if code, _, msg := s.create(t, linked, baseRepo("token")); code != http.StatusCreated { + t.Fatalf("tracked create = %d %q, want 201", code, msg) } - if r, err := e.store.GetRepoByURL(ctx, s.org, name); !errors.Is(err, meta.ErrNotFound) { - t.Fatalf("repo after a refused tracked create = %+v, %v; want not found", r, err) - } - if code, _, msg := s.create(t, linked, baseRepo(tc.token)); code != http.StatusBadRequest || msg != tc.want { - t.Fatalf("repeat create with token %q = %d %q, want 400 %q", tc.token, code, msg, tc.want) + for _, tc := range []struct{ token, want string }{ + {"to\nken", "invalid base_repo.auth.token (want no carriage return or line feed)"}, + {"enc:token", reserved}, + } { + name := "val-tracked-" + randomHex(t, 4) + if code, _, msg := s.create(t, name, baseRepo(tc.token)); code != http.StatusBadRequest || msg != tc.want { + t.Errorf("tracked create with token %q = %d %q, want 400 %q", tc.token, code, msg, tc.want) + } + if r, err := e.store.GetRepoByURL(ctx, s.org, name); !errors.Is(err, meta.ErrNotFound) { + t.Errorf("repo after a refused tracked create = %+v, %v; want not found", r, err) + } + if code, _, msg := s.create(t, linked, baseRepo(tc.token)); code != http.StatusBadRequest || msg != tc.want { + t.Errorf("repeat create with token %q = %d %q, want 400 %q", tc.token, code, msg, tc.want) + } } - } + }) repo, err := e.store.CreateRepo(ctx, org, "https://example.com/"+org+"/val-cred-"+randomHex(t, 4), "main") if err != nil { diff --git a/internal/errtext/README.md b/internal/errtext/README.md index 056a5f37..f7da8d68 100644 --- a/internal/errtext/README.md +++ b/internal/errtext/README.md @@ -18,7 +18,8 @@ Bounds the request text that an error answer repeats, so that an answer stays sm - `api` cuts `error`, `detail` and each `invalid_params` name of a problem body (`newProblem`), and the `result.message` of a commit write (`wcRefUpdate`) [../api/errors.go:newProblem] [../api/handlers_write_commit.go:wcRefUpdate] - `api` quotes an unknown member name in an answer that names one unknown member (`unknownNameText`), and cuts each unknown member name at 32 bytes in the answer of a strictly decoded body that names several members (`Cut`) [../api/decode_body.go:unknownNameText] [../api/handlers_write_refs.go:strictProblem.err] - `api` quotes a refused sync provider, operation, access or host (`upstreamConfig.validate`, `validateSyncBaseRepo`) and the ref policy op of a token in a 403 (`refPolicyOpDenies`) [../api/handlers_sync.go:upstreamConfig.validate] [../api/handlers_sync.go:refPolicyOpDenies] -- `events` quotes a webhook URL or its host in a registration refusal (`CheckRegistration`) [../events/guard.go:CheckRegistration] +- `api` quotes a ref policy op that a ref, commit or merge write does not know (`unknown policy op "x" for `) [../api/handlers_write_refs.go:wrefPolicyCheck] [../api/handlers_write_commit.go:wcCheckRefPolicy] [../api/handlers_write_merge.go:server.wmCheckRefPolicy] +- `events` quotes a webhook URL or its host in a registration refusal, and cuts the parse error it adds after a URL that does not parse (`CheckRegistration`) [../events/guard.go:CheckRegistration] - `githttp` cuts the error text of the git and LFS routes [../githttp/server.go:httpError] [../githttp/lfs.go:lfsError] ## Verification diff --git a/internal/events/guard.go b/internal/events/guard.go index 0e73cbf0..9d52bfb3 100644 --- a/internal/events/guard.go +++ b/internal/events/guard.go @@ -111,7 +111,8 @@ const dialTimeout = 10 * time.Second // Instance Metadata Service as serving metadata and bootstrapping VM // credentials // (learn.microsoft.com/azure/virtual-machines/metadata-security-protocol/overview). -// It is a public address, so the guard denied it under no policy. +// It is a public address, so without this entry no policy would deny +// it. // - 192.0.0.192: Oracle Cloud Infrastructure Classic metadata // (docs.oracle.com/en/cloud/iaas-classic/compute-iaas-cloud/stcsg/retrieving-instance-metadata.html), // inside 192.0.0.0/24, which specialUsePrefixes denies too. diff --git a/internal/sync/egress.go b/internal/sync/egress.go index ab66bc15..f5b4dd14 100644 --- a/internal/sync/egress.go +++ b/internal/sync/egress.go @@ -10,7 +10,7 @@ package sync // pioneered (issue #53) cannot be applied in-process; instead every // http(s) transfer runs through a short-lived loopback forward proxy // whose dials go through TargetGuard.DialContext. The operator escape -// hatch (OPENSTORAGE_SYNC_ALLOW_PRIVATE; deprecated push-only spelling +// hatch (OPENSTORAGE_SYNC_ALLOW_PRIVATE; deprecated push-only variable // OPENSTORAGE_SYNC_PUSH_ALLOW_PRIVATE) relaxes only the private receiver // classes (loopback, RFC 1918, RFC 6598 shared, link-local, ULA), exactly // like the webhook hatch: the cloud metadata endpoints, broadcast, and the diff --git a/internal/sync/worker.go b/internal/sync/worker.go index 7f252af4..8297ac21 100644 --- a/internal/sync/worker.go +++ b/internal/sync/worker.go @@ -218,7 +218,7 @@ type Worker struct { // PushEgressAllowPrivate relaxes the outbound push egress guard's // private receiver classes (resolved from OPENSTORAGE_SYNC_ALLOW_PRIVATE // with OPENSTORAGE_SYNC_PUSH_ALLOW_PRIVATE as the deprecated push-only - // spelling, see egress.go): loopback, RFC 1918, RFC 6598 shared, + // variable, see egress.go): loopback, RFC 1918, RFC 6598 shared, // link-local and ULA upstreams for genuinely internal forges. The // cloud metadata endpoints and the other special-purpose ranges stay // denied either way — every http(s) push dials through