diff --git a/.agents/skills/verify-lane-schedule/SKILL.md b/.agents/skills/verify-lane-schedule/SKILL.md index ae63fe6d..d553b4a0 100644 --- a/.agents/skills/verify-lane-schedule/SKILL.md +++ b/.agents/skills/verify-lane-schedule/SKILL.md @@ -226,7 +226,7 @@ With every fact TRUE in the code, setting each fact FALSE in the model alone fai ## Coverage record -Sync commit: `2e98dad6dd886941543d266b99fed69d8b34b17d` (the last commit at which someone confirmed that the model describes this code; only `check.sh --record` moves it). +Sync commit: `ad51accd05a13ac21c2bd3ea8569ffb19bb17cda` (the last commit at which someone confirmed that the model describes this code; only `check.sh --record` moves it). Every declaration in these files is covered: diff --git a/.agents/skills/verify-lane-schedule/model/fingerprints.txt b/.agents/skills/verify-lane-schedule/model/fingerprints.txt index d11ce3d7..9cde39af 100644 --- a/.agents/skills/verify-lane-schedule/model/fingerprints.txt +++ b/.agents/skills/verify-lane-schedule/model/fingerprints.txt @@ -68,11 +68,7 @@ internal/api/handlers_sync.go:const_deprecationHeader 188405d30be79bf4a07cacfd25 internal/api/handlers_sync.go:const_pullUpstreamAcceptedMsg 76a52910111a2580c149b7ea535374dd1196bde9753bf900ed6667fc5763bdb4 internal/api/handlers_sync.go:const_pushUpstreamAcceptedMsg a2ae388107c13efba0294df359ec78e2d5b92bbb2774f068e5507891b5fea698 internal/api/handlers_sync.go:const_upstreamAppAccessMsg e754e2f035f781ed53cc2ee4cf96a4ee8988a87fa93954aca9989c2040b75599 -internal/api/handlers_sync.go:curlIPv4Name 3a642528f1cbc1e513c3d9280e277ce371ba7208b1f880b54b9b851fb92ae32b -internal/api/handlers_sync.go:curlIPv4Part 0d60932194160d2f0f58fc5f0ae6295b84337e94d3a2552565f4ad398cc7cbb4 internal/api/handlers_sync.go:deprecated 3227984a4c52242a8804346ff26305a40483c81f04ead54d0ab913daa01daa24 -internal/api/handlers_sync.go:dialableAuthority 272fc206a7652abebab23cc8b90a106a7a2f4571277f9c52ae0ef0b63f3397bc -internal/api/handlers_sync.go:dialableHostname 4aae5d4917b5a62b0b9483a694a95d585a56913a1756a60835291e61d5eb5880 internal/api/handlers_sync.go:import_"bytes" 151fc787b0679fc8767e367d1971b9f907660a3d72d8fa3888877ef9f8ebd9c3 internal/api/handlers_sync.go:import_"cmp" 6ee637994d3b529ead32b904453999f4347fa28f1bba3d141f490cbf02e1f7cc internal/api/handlers_sync.go:import_"context" 35d620b75b6b66f8c228f26b970a741d77137e4e691aa896119dd9d000eaadd2 @@ -85,9 +81,7 @@ internal/api/handlers_sync.go:import_"github.com/coder/openstorage/internal/meta internal/api/handlers_sync.go:import_"github.com/coder/openstorage/internal/sync" 3b3f0175cf3dee88544af1c7af3be7e737e79ae40f672de5cc2c6cc7e6d1f40e internal/api/handlers_sync.go:import_"io" 93c8d110e63e2ae040bf1dfd2ae6e7c4b4ff1bdc806be48a9ae34456827e0486 internal/api/handlers_sync.go:import_"net/http" 480be5af96d08206ab18c2509cb88ca7831e506d1256768f81e9dd07ebf030a8 -internal/api/handlers_sync.go:import_"net/netip" ee93e0a6774d156ffb7dee8e4b29220dbe7dc488e036f596c8ae70f318ffc70e internal/api/handlers_sync.go:import_"net/url" dbfdd047eb1b2a38b79ee147694d9d1308190f3c019908fa1ebf5e3fd13e2343 -internal/api/handlers_sync.go:import_"strconv" 4259f4eb8a51f255fe15fe2b5dea61adc7a54b5df7345920fed692a971b4af46 internal/api/handlers_sync.go:import_"strings" 347f34c3a37316b52e87798b22bb4c8e6173f20f936b730bf65dc5ce6c1fb407 internal/api/handlers_sync.go:import_"time" 9dae0539db6e1e0ec43a0c7ef2211f001e3ddc574b4fc19244eab79b05e23f5a internal/api/handlers_sync.go:isStrictPullBody ec4bff99487fd47a4c11c3cc2714cee70b94035771aa148211f4abbfd7e798f5 @@ -96,7 +90,6 @@ internal/api/handlers_sync.go:parseUpstreamConfig dd27982847b44628b2a14a4364e4a2 internal/api/handlers_sync.go:policyDeniesPullUpstream 098960c7bef537c91ca905ce894432882eef7ef465a3ddb0b0f9e7224be746e7 internal/api/handlers_sync.go:refPolicyOpDenies 14c74420941dbb32b29b6bccd73d4c8029dc794fba6a8df30beaf91104b7c0f1 internal/api/handlers_sync.go:registerSyncRoutes 30a124203d6812c4b8fa939a4866caae6a74ac2b0b71e36a7de2fed3e0c25134 -internal/api/handlers_sync.go:resolverName 3e71c7fdad4e07921f939372c51f47fe4662e0140f0f73ee453bb27eee32f5b3 internal/api/handlers_sync.go:sentHost 06d61ad20c0eda8ee6d85423eba7a886af4d12740a62517242e024cb01e1381d internal/api/handlers_sync.go:syncServer.answerTakenSyncName 22cf55fac3487ceec250be953518f3cd9b54265fa9061a6f94c4dd2d8567b0e5 internal/api/handlers_sync.go:syncServer.completeOwnSyncCreate a301e0bad45ed03f793aa89eca3036f4e652051a33060812075c97ccf1076b16 @@ -118,11 +111,10 @@ internal/api/handlers_sync.go:type_usyncBaseRepo fa63c2d8bb62db2a56acff9a8de55e5 internal/api/handlers_sync.go:type_usyncBaseRepoAuth cbce39e3f3df0443cfcb499a9e439e361335ee86e6e97c205c21b18b48141cd6 internal/api/handlers_sync.go:type_usyncPullUpstreamRequest b98132ea1ae7690f3f00aece7dd8fb2a00f69d0e9ea5f1f95c198c983cc18b0b internal/api/handlers_sync.go:type_usyncPushUpstreamRequest 3ad57ba2d21ad394741cb6b3ced05a273c72b38ca84d7a73b34d79b4510f4284 -internal/api/handlers_sync.go:upstreamConfig.validate 4c75ed7ea1b26bcef15daa0358e3d79ee704eb27b8d1fbdb9861d7a0d4f2f509 +internal/api/handlers_sync.go:upstreamConfig.validate 92dd3c7bf8e33cf7987a72b13d5c3b2b28745032f8a945ebee8ccfa7597819af internal/api/handlers_sync.go:upstreamHostname f65b41a625e2606e71c567cbc6d99d991e59b6ef4da9e976294b9ff3895aac34 internal/api/handlers_sync.go:upstreamURLProblem e66d2c29ef48b6891a62cca5ace9afbd52d579f5f4ae11aeba3cc61b3f3c488d -internal/api/handlers_sync.go:validUpstreamHost 2aa28ab36ae31c94b9311a490f3bfb3707653ce95f7995727c31a37f58cf3c30 -internal/api/handlers_sync.go:validateSyncBaseRepo df7737d1b71ebccd3220b074e9002ee36edc83b549338e321394c012a9a6a807 +internal/api/handlers_sync.go:validateSyncBaseRepo 53ae861e7da72aa70b924740aaf81d91c6be1c0644b83a47c1dc3106960628dd internal/api/handlers_sync.go:withStrictPullBody 449923a2f639f42183416d391cac81b304bcf8b6cda9d207474e66e16178766f internal/api/handlers_sync.go:withUpstreamDetach b16659bb293b5f9eb05b5ade652dfe4fb224e8a7b6c475829007ee1933a164ec internal/events/refstore.go:BaseRepoID d45f84f8f7f8b9e27d21c7b17553527e76db8c3a5fa34718d2768840b351c8ae @@ -833,7 +825,7 @@ internal/sync/worker.go:type_RepoInfo e8a02330a52c30199fa31037b4286d8d23ab173f23 internal/sync/worker.go:type_RepoLookup 819f1bc4b699b113c5c1400e9137b05dd20a8fa12cd1d0d621cca9ac686e29df internal/sync/worker.go:type_Worker 02235720d66907846dcf03ab0427a9eec32fb417af3c0462bbf5a381c76dfdf1 internal/sync/worker.go:type_syncFailure 6f04984a8621d6e7bf3665dd7af1eb8f7634d923899e5e61b18c60e246c3c59c -internal/sync/worker.go:upstreamURLFailure a4f2c466197974a038dfc2edd51f12da58db9bb332df8709573a6fbd3101cd20 +internal/sync/worker.go:upstreamURLFailure f3cab3222d358d74d09d01aa01bf85a19919c4493a8d010c79af0eee15a8c9f3 internal/sync/worker.go:var_authFailureMarkers 28df1810afcf9a0f750de4a7b73b0a925f66ba08df738a9c43a062abc68c8d40 internal/sync/worker.go:var_errClaimStolen 5869b81a47a71928975f5577f5b485bef2e6d0d98f3fd0f31a81b191a67f74a6 internal/sync/worker.go:var_errDetached 09503fc22b7999840131138daf5a27e594cb8b6773f3be5b2001b2bbb0d356e2 diff --git a/docs/reference/http-api.md b/docs/reference/http-api.md index ab08fa62..b2a4448d 100644 --- a/docs/reference/http-api.md +++ b/docs/reference/http-api.md @@ -953,6 +953,17 @@ pass the total limit together: one git start can carry the URL twice and both cr entries, about 512 KiB at the limits, so a server with a stack limit of 2 MiB or less can fail such a run as a failed fetch or push. +A pull or push run whose stored host `POST /repos` refuses (`is not a valid host`, +[Repository lifecycle](#repository-lifecycle-repowrite--orgread)) also fails as +`repository configuration error` before git starts. The server's `WARN` line for the +failed run gives the cause `upstream host is not a valid host` and does not repeat the +host. Only a link stored before `POST /repos` refused its host has one, such as a name +with letters outside ASCII (`bücher.example`), a number that curl reads as an IPv4 +address (`2130706433`), or a base URL with an upper-case scheme (`HTTPS://`). Such a +link fails every run: to sync that upstream, link a repository again through +`POST /repos` with the host in a form it takes, such as `xn--bcher-kva.example`, the +ASCII form of `bücher.example`. + #### Deprecation header Every response of the four deprecated routes carries `Deprecation: @1791331200`: diff --git a/internal/api/README.md b/internal/api/README.md index 5590074b..f51366c6 100644 --- a/internal/api/README.md +++ b/internal/api/README.md @@ -33,7 +33,7 @@ The REST API (`/api/v1/...`) and its `/api/repos/{repo_name}` alias layer. `New` - Merges need git >= 2.38 and refuse unrelated histories unless allowed; a stale target SHA is 409 [handlers_write_merge.go:wmCheckGitVersion] [handlers_write_merge.go:handleMerge] - Fork and copy check the source token for `git:read` and the pack size against quota before creating any row [handlers_fork.go:handleForkRepo] - `publishPack` denies over-quota packs before upload and fails closed on a second stale usage floor [publish.go:publishPack] -- `validUpstreamHost` rejects userinfo, query, fragment (even an empty one), a space, a scheme other than lower-case `http`/`https`, a port outside 1-65535, and a hostname that is neither an IP address without a zone nor a name that Go's resolver asks DNS for and that curl does not read as a numerical IPv4 address (one to four decimal, octal or hex numbers within IPv4's range) [handlers_sync.go:validUpstreamHost] [handlers_sync.go:dialableHostname] [handlers_sync.go:curlIPv4Name] +- A sync create refuses a host that `upsync.ValidUpstreamHost` refuses, with the reason `upsync.InvalidUpstreamHostReason` (`is not a valid host`) for `upstream.host` or `base_repo.upstream_host` [handlers_sync.go:upstreamConfig.validate] [handlers_sync.go:validateSyncBaseRepo] - Text the request asks to store answers 400 on every backend when it holds a NUL byte (a JSON body decodes invalid UTF-8 to U+FFFD, so only a NUL reaches the check): a webhook `secret`, a credential `username` and `password`, the `owner`, `name` and `auth.token` of a sync link, and a `pull-upstream` `ref`. A webhook or credential ID with a NUL byte or invalid UTF-8 answers the 404 of an unknown ID [errors.go:invalidTextFields] [handlers_webhooks.go:handleCreate] [handlers_credentials.go:handleCreateCredential] [handlers_sync.go:validateSyncBaseRepo] [handlers_sync.go:parseUpstreamConfig] [handlers_sync.go:handlePullUpstream] - A request value that git takes as one argument or one environment entry answers 400 before git starts when it does not fit `gitlimit.MaxArgBytes` with what the server puts around it: a read `path` or grep `paths` entry (`maxGitPathBytes`), a blame `range` and a grep `query.pattern`, and the `name` and `email` of an author, a committer or a note author (`maxGitIdentBytes`). A commit message goes to `commit-tree` on stdin. A list of such values (the grep `paths`, the diff `path` params, the blame `range` params) also answers 400 when its entries do not fit `gitlimit.MaxArgListBytes` together [handlers_read_query.go:listTooLong] [handlers_read_core.go:readPathOK] [handlers_read_query.go:cleanPathParams] [handlers_read_query.go:handleBlame] [handlers_read_query.go:handleGrep] [handlers_read_query.go:grepPathProblems] [handlers_write_commit.go:wcAuthorProblems] [handlers_write_refs.go:wrefHandleNote] [handlers_write_commit.go:wcScratch.writeCommit] - A sync link or credential that the sync worker could not hand to git answers 400 when it is stored: an upstream URL (`upsync.Upstream.CloneURL`) over `upsync.MaxUpstreamURLBytes`, naming the owner, the name and the host field the body sent, and a `base_repo.auth.token`, credential `username` or `password` over `upsync.MaxCredentialBytes`. The worker checks the same limits again for values stored before [handlers_sync.go:upstreamURLProblem] [handlers_sync.go:validateSyncBaseRepo] [handlers_credentials.go:gcredValidate] diff --git a/internal/api/handlers_sync.go b/internal/api/handlers_sync.go index 84464aef..9a25eef5 100644 --- a/internal/api/handlers_sync.go +++ b/internal/api/handlers_sync.go @@ -26,9 +26,7 @@ import ( "fmt" "io" "net/http" - "net/netip" neturl "net/url" - "strconv" "strings" "time" @@ -152,8 +150,8 @@ func validateSyncBaseRepo(spec *usyncBaseRepo, branchErr error, errs *bodyErrors var hostErr error host = spec.UpstreamHost switch { - case host != "" && !validUpstreamHost(host): - hostErr = newFieldError("base_repo.upstream_host", "is not a valid host", "invalid base_repo upstream_host "+errtext.Quote("", spec.UpstreamHost)) + case host != "" && !upsync.ValidUpstreamHost(host): + hostErr = newFieldError("base_repo.upstream_host", upsync.InvalidUpstreamHostReason, "invalid base_repo upstream_host "+errtext.Quote("", spec.UpstreamHost)) case host == "" && known: if host = defaultHost; host == "" { hostErr = newFieldError("base_repo.upstream_host", "is required for this provider", @@ -322,8 +320,8 @@ func (cfg upstreamConfig) validate(errs *bodyErrors) (host string) { } host = cfg.Host switch { - case host != "" && !validUpstreamHost(host): - hostErr = newFieldError("upstream.host", "is not a valid host", "invalid upstream host "+errtext.Quote("", cfg.Host)) + case host != "" && !upsync.ValidUpstreamHost(host): + hostErr = newFieldError("upstream.host", upsync.InvalidUpstreamHostReason, "invalid upstream host "+errtext.Quote("", cfg.Host)) case host == "" && known: if host = defaultHost; host == "" { hostErr = newFieldError("upstream.host", "is required for this provider", "upstream host is required for provider "+errtext.Quote("", cfg.Provider)) @@ -338,164 +336,8 @@ func (cfg upstreamConfig) validate(errs *bodyErrors) (host string) { return host } -// validUpstreamHost accepts the two documented upstream_host shapes — a -// bare hostname[:port] or an http(s):// base URL — and nothing else. Both -// need a hostname: upstreamHostname asserts one, so a host such as ":80" -// must be refused here and not reach it. -// upsync.CloneURL uses the value verbatim as the base of a git fetch URL, -// so file://, ssh://, transport helpers (ext::), embedded credentials, and -// query/fragment noise must be rejected here, where the attacker-controlled -// value enters (the worker additionally pins GIT_ALLOW_PROTOCOL). -// -// It also refuses hosts that a sync run cannot use as given, so the create -// answers 400 instead of storing a link that the registration check could -// not vet: a hostname that is neither an IP address nor a name that the -// run's resolver asks DNS for (dialableHostname), a port outside 1-65535, -// and a base URL with a query or a fragment, even an empty one -// (`https://h?`), which would end the path of every fetch URL, with a -// space, which curl refuses, or with a scheme that is not lower case -// (`HTTPS://`), which GIT_ALLOW_PROTOCOL refuses. -func validUpstreamHost(host string) bool { - assertf(host != "", "validUpstreamHost: empty host") - if !strings.Contains(host, "://") { - // Bare host[:port]: must round-trip as exactly the authority - // component (no path, userinfo, or control characters). A zone - // is written %25, which the parse decodes, so it never - // round-trips. - u, err := neturl.Parse("https://" + host) - return err == nil && u.Host == host && u.User == nil && - u.Path == "" && u.RawQuery == "" && u.Fragment == "" && dialableAuthority(u) - } - if !strings.HasPrefix(host, "http://") && !strings.HasPrefix(host, "https://") { - return false - } - u, err := neturl.Parse(host) - if err != nil { - return false - } - // A path is allowed (sub-path-hosted forges); credentials belong in - // the credential store, never in the URL. url.Parse drops an empty - // query or fragment and keeps a space in a path, so the bytes are - // checked. - return u.User == nil && !strings.ContainsAny(host, "?# ") && dialableAuthority(u) -} - -// dialableAuthority reports whether the host and port of u name something -// that a sync run can dial: a dialableHostname and, when u has one, a port -// from 1 to 65535. An empty port (`forge.example:`) is the default one. -func dialableAuthority(u *neturl.URL) bool { - if port := u.Port(); port != "" { - if n, err := strconv.Atoi(port); err != nil || n < 1 || n > 65535 { - return false - } - } - return dialableHostname(u.Hostname()) -} - -// dialableHostname reports whether name is an IP address without a zone, -// or a name that the resolver of a sync run asks DNS for and that curl -// does not read as an IPv4 address (curlIPv4Name). A zone-scoped address -// (fe80::1%eth0) is refused: the egress guard refuses it under every -// policy, and the egress answer would repeat its zone, which only the -// upstream URL limit bounds. -func dialableHostname(name string) bool { - if ip, err := netip.ParseAddr(name); err == nil { - return ip.Zone() == "" - } - return resolverName(name) && !curlIPv4Name(name) -} - -// resolverName reports whether the Go resolver of a sync run asks DNS for -// name. A run dials through the egress proxy (upsync.startEgressProxy), -// whose resolver refuses any other name (net.isDomainName, after RFC 1035 -// and RFC 3696): at most 253 bytes, or 254 with a final dot; labels of 1 to -// 63 letters, digits, '_' and '-' that do not start or end with '-'; and -// not only digits and dots, which curl reads as an IPv4 address -// (2130706433 is 127.0.0.1) that the registration check never saw. The -// resolver reads /etc/hosts before it applies the rule, but a name that -// only /etc/hosts holds is no upstream host. -// -// A name with other letters goes in its ASCII form -// (xn--bcher-kva.example): the registration check cannot look bücher.example -// up, and whether git and curl convert it depends on the locale that the -// server's git runs in. -func resolverName(name string) bool { - if name == "" || len(name) > 254 || len(name) == 254 && name[253] != '.' { - return false - } - prev := byte('.') - letters := false // a letter, '_' or '-': all digits would be an address - label := 0 - for i := 0; i < len(name); i++ { - c := name[i] - switch { - case 'a' <= c && c <= 'z', 'A' <= c && c <= 'Z', c == '_': - letters = true - label++ - case '0' <= c && c <= '9': - label++ - case c == '-': - if prev == '.' { - return false - } - letters = true - label++ - case c == '.': - if prev == '.' || prev == '-' || label > 63 { - return false - } - label = 0 - default: - return false - } - prev = c - } - return prev != '-' && label <= 63 && letters -} - -// curlIPv4Name reports whether curl reads name as an IPv4 address, as its -// URL parser does (ipv4_normalize): one to four labels, a final dot aside, -// each a decimal, octal (leading 0) or hexadecimal (0x) number of at most -// 32 bits, all but the last at most 255, and the last within the bits the -// others leave, such as 2130706433, 0x7f000001, 0x7f.1 or 1.2.3.0x4. curl -// dials such a name as the address (127.0.0.1, 1.2.3.4), which the -// registration check, which reads it as a name, never saw. A name that only -// looks numeric, such as 0x100.1 or 0x100000000, stays a name for curl. -func curlIPv4Name(name string) bool { - labels := strings.Split(strings.TrimSuffix(name, "."), ".") - if len(labels) > 4 { - return false - } - last := len(labels) - 1 - for i, label := range labels { - v, ok := curlIPv4Part(label) - if !ok || i < last && v > 0xff || i == last && v > 0xffffffff>>(8*last) { - return false - } - } - return true -} - -// curlIPv4Part parses one label of a numerical IPv4 address as curl does: -// 0x or 0X and hexadecimal digits, 0 and octal digits, or decimal digits, -// at most 32 bits. -func curlIPv4Part(label string) (uint64, bool) { - base, digits := 10, label - switch { - case len(label) >= 2 && label[0] == '0' && (label[1] == 'x' || label[1] == 'X'): - base, digits = 16, label[2:] - case len(label) >= 1 && label[0] == '0': - base = 8 - } - if digits == "" { - return 0, false - } - v, err := strconv.ParseUint(digits, base, 32) - return v, err == nil -} - -// upstreamHostname extracts the hostname the sync fetch will dial from a -// validUpstreamHost-validated value (bare host[:port] or http(s):// base +// upstreamHostname extracts the hostname the sync fetch will dial from an +// upsync.ValidUpstreamHost-validated value (bare host[:port] or http(s):// base // URL) for the registration-time egress courtesy check (issue #139). func upstreamHostname(host string) string { assertf(host != "", "upstreamHostname: empty host") diff --git a/internal/api/upstream_host_shape_test.go b/internal/api/upstream_host_shape_test.go index 58e5f00b..bec7a485 100644 --- a/internal/api/upstream_host_shape_test.go +++ b/internal/api/upstream_host_shape_test.go @@ -8,135 +8,6 @@ import ( "testing" ) -// TestValidUpstreamHost: an upstream host is a bare host[:port] or an -// http(s) base URL whose hostname a sync run can dial: an IP address -// without a zone, or a name that Go's resolver looks up (net.isDomainName), -// with a port from 1 to 65535 and no query or fragment. -func TestValidUpstreamHost(t *testing.T) { - label63 := strings.Repeat("a", 63) - // 63+1+63+1+63+1+61 = 253 bytes, the longest name without a final dot. - name253 := label63 + "." + label63 + "." + label63 + "." + strings.Repeat("b", 61) - for _, host := range []string{ - "forge.example", - "forge.example.", - "FORGE.Example:8443", - "a_b.example", - "xn--bcher-kva.example", - "localhost", - "1forge.example", - label63 + ".example", - "forge." + label63, - "1-1", - "1_1", - "cafe", - "0xg.example", - // Names that only look numeric stay names for curl: five labels, - // a part out of IPv4's range, an empty or a bad number. - "0x1.0x2.0x3.0x4.0x5", - "0x0.0x0.0x0.0x0.0x0", - "0x100.1", - "0x100000000", - "0xff.0x1000000", - "0xff.0xff.0x10000", - "0x", - "0x.1", - "1.0x", - "09.0x1", - "0xg", - "axe", - "6x6", - // A 63-byte label of letters, digits and dashes. - "a" + strings.Repeat("1", 30) + strings.Repeat("-", 30) + "bc", - name253, - name253 + ".", - "10.0.0.1", - "10.0.0.1:1", - "[::1]:65535", - "[2001:db8::1]", - "https://forge.example", - "https://forge.example/sub/path/", - "http://127.0.0.1:3000", - "https://[2001:db8::1]:443/git", - "https://forge.example/a%3Fb", - "forge.example:", - "https://forge.example:/git", - } { - if !validUpstreamHost(host) { - t.Errorf("validUpstreamHost(%q) = false, want true", host) - } - } - for _, host := range []string{ - // Names that the resolver refuses. - "<<<.com", - "https://<<<.com", - "bücher.example", - "https://bücher.example", - "a..b.example", - ".forge.example", - "-forge.example", - "forge-.example", - "forge.-example", - "forge.example-", - label63 + "a.example", - "a" + strings.Repeat("1", 63) + ".example", - "a" + strings.Repeat("-", 62) + "a.example", - "forge." + label63 + "a", - name253 + "a", - name253 + "a.", - ".", - // Only digits and dots, but no IP address: curl would read - // 2130706433 as 127.0.0.1, Go's resolver refuses it. - "2130706433", - "1.2.3", - "https://1.2.3", - "01.2.3.4", - "1.2.3.4.5", - // Hexadecimal forms that curl reads as IPv4 too. - "0x7f000001", - "0x7f.1", - "0X7F.0.0.1", - "1.2.3.0x4", - "0x7f.1.", - "0xa9fea9fe", - "0xA9FEA9FE", - "0xA.0xA.0xA.0xA", - "0xffffffff", - "0xff.0xffffff", - "0xff.0xff.0xffff", - "0377.0377.0377.0377", - "http://0x7f000001:3000/git", - // A zone. - "https://1.2.3.4%25eth0", - "https://[fe80::1%25eth0]", - "[fe80::1%25eth0]", - // Ports. - "forge.example:0", - "forge.example:65536", - "https://forge.example:99999/git", - // A query or fragment, even an empty one. - "https://forge.example?", - "https://forge.example#", - "https://forge.example/git?", - "https://forge.example/git?a=b", - "https://forge.example/git#x", - // A space, which curl refuses, and a scheme that is not lower - // case, which GIT_ALLOW_PROTOCOL refuses. - "https://forge.example/a b", - "HTTPS://forge.example", - "Http://forge.example", - // Unchanged refusals. - ":80", - "https://:80", - "forge.example/git", - "ssh://forge.example", - "https://u:p@forge.example", - } { - if validUpstreamHost(host) { - t.Errorf("validUpstreamHost(%q) = true, want false", host) - } - } -} - // TestSyncCreateRefusesUndialableHosts: POST /repos refuses, with 400 // before anything is stored, a host that no sync run could use, in both // host fields. Before, the create answered 201 and every run failed. diff --git a/internal/sync/README.md b/internal/sync/README.md index b8adedf0..5701ab9c 100644 --- a/internal/sync/README.md +++ b/internal/sync/README.md @@ -6,6 +6,7 @@ Upstream git mirror sync (pull) and outbound push-forward. The package defines t ## Interface - Data types: `Credential`, `Upstream` (`CloneURL`), `Job` [sync.go:Credential]; `RejectedRef`, `EventRepository`, `Event` [events.go:Event] - `ProviderHost`, `NewCredentialID` (`ggc_` + 16 hex) [sync.go:ProviderHost] +- `ValidUpstreamHost`, `InvalidUpstreamHostReason` — the upstream hosts that `POST /repos` stores and a run takes, and the reason (`is not a valid host`) for any other [upstreamhost.go:ValidUpstreamHost] - `PushPendingNone`, `PushPendingPush`, `PushPendingForce` — outbound pending levels 0, 1, 2 [sync.go:PushPendingNone] - Errors: `ErrNotFound`, `ErrCredentialExists`, `ErrUpstreamExists`, `ErrSyncInProgress`, `ErrPushInProgress`, `ErrClaimLost`, `ErrShutdown` [sync.go:ErrNotFound] [local_schedule.go:ErrShutdown] - Contracts: `CredentialStore`, `UpstreamStore`, `EventSink`, `Queue`, `DefaultBranchAdopter`, `FirstSyncScheduler`, `ShutdownRestorer`, `LocalLanes` [sync.go:Queue] [default_branch.go:DefaultBranchAdopter] [first_sync.go:FirstSyncScheduler] [metastore.go:ShutdownRestorer] [local_schedule.go:LocalLanes] @@ -23,11 +24,12 @@ Upstream git mirror sync (pull) and outbound push-forward. The package defines t ## Invariants - `assertf` panics on programming errors only, never on untrusted input [sync.go:assertf] +- `ValidUpstreamHost` rejects userinfo, query, fragment (even an empty one), a space, a scheme other than lower-case `http`/`https`, a port outside 1-65535, and a hostname that is neither an IP address without a zone nor a name that Go's resolver asks DNS for and that curl does not read as a numerical IPv4 address (one to four decimal, octal or hex numbers within IPv4's range) [upstreamhost.go:ValidUpstreamHost] [upstreamhost.go:dialableHostname] [upstreamhost.go:curlIPv4Name] - `ValidBranchName` rejects empty names, names over 255 bytes, `..`, bad components, and `.lock` or `.` suffixes [default_branch.go:ValidBranchName] - `Worker.assertReady` panics when a required dependency is unset; `Run` and `RunPush` assert non-empty org and repo ID [worker.go:assertReady] - `Run` returns nil without events when the repo no longer exists [worker.go:Run] - A transfer refuses to push when the holder changed (`errClaimStolen`), the row is gone (`errDetached`), or the claim lease is inside `claimFenceMargin` [worker.go:transfer] -- `hermeticGitEnv` strips inherited `GIT_*` variables and sets `GIT_ALLOW_PROTOCOL` [worker.go:hermeticGitEnv]; stored credentials containing CR, LF, or NUL fail with `FailConfig`, and so does one over `MaxCredentialBytes` [worker.go:Worker.upstreamAuth]; an upstream URL over `MaxUpstreamURLBytes` fails with `FailConfig` before any git starts [worker.go:upstreamURLFailure]. The API refuses both limits at registration (`POST /repos`, git credential routes) with the same constants; the run checks stay for stored values +- `hermeticGitEnv` strips inherited `GIT_*` variables and sets `GIT_ALLOW_PROTOCOL` [worker.go:hermeticGitEnv]; stored credentials containing CR, LF, or NUL fail with `FailConfig`, and so does one over `MaxCredentialBytes` [worker.go:Worker.upstreamAuth]; a stored host that `ValidUpstreamHost` refuses (a `file://` test fixture aside) and an upstream URL over `MaxUpstreamURLBytes` fail with `FailConfig` before any git starts, and the error never names the host or the URL [worker.go:upstreamURLFailure]. The API refuses both limits and such a host at registration (`POST /repos`, git credential routes) with the same constants and `ValidUpstreamHost`; the run checks stay for stored values - Only `FailUnreachable` push failures retry, at most `MaxPushTransientRetries` times; pushes use `--atomic` [worker_push.go:RunPush] - `proxyEnv` removes inherited proxy variables in any case; the egress proxy maps policy denials to 403 and dial failures to 502 [egress.go:proxyEnv] - `MemStore.BeginSync` returns `ErrSyncInProgress` while an unexpired claim exists; legacy claims get a grace period; `RequestPush` never downgrades force; a stolen push claim inherits force [memstore.go:BeginPush] diff --git a/internal/sync/stored_host_test.go b/internal/sync/stored_host_test.go new file mode 100644 index 00000000..d15f27bc --- /dev/null +++ b/internal/sync/stored_host_test.go @@ -0,0 +1,95 @@ +package sync + +import ( + "bytes" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + stdsync "sync" + "sync/atomic" + "testing" +) + +// lockedBuffer is a bytes.Buffer that a logger can write from several +// goroutines. +type lockedBuffer struct { + mu stdsync.Mutex + buf bytes.Buffer +} + +func (b *lockedBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *lockedBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +// TestRunRefusesStoredInvalidHost: a pull or push run whose stored host +// POST /repos now refuses (ValidUpstreamHost), as a link that an older +// release stored can hold, fails as configuration before any git starts. +// The run's error and the server log give POST /repos's reason, never the +// host. Before, the run handed such a host to git: the loopback forms here +// reached the upstream through the allow-private egress policy. +func TestRunRefusesStoredInvalidHost(t *testing.T) { + var reached atomic.Bool + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + reached.Store(true) + http.Error(w, "not a git server", http.StatusNotFound) + })) + defer srv.Close() + port := srv.URL[strings.LastIndexByte(srv.URL, ':'):] + + const cause = "repository configuration error: upstream host is not a valid host" + for _, host := range []string{ + "bücher.example", + "https://bücher.example", + "http://2130706433" + port, // curl dials 127.0.0.1 + "http://0x7f.1" + port, // curl dials 127.0.0.1 + "http://127.0.0.1" + port + "?", + "http://127.0.0.1" + port + "/a b", + "HTTP://127.0.0.1" + port, + "forge.example:65536", + "https://[fe80::1%25eth0]", + } { + for _, lane := range []string{"pull", "push"} { + // A % in the name would reach the file:// URL of the + // fixture's t.TempDir. + t.Run(lane+" "+strings.ReplaceAll(host, "%", "_"), func(t *testing.T) { + reached.Store(false) + f := newWorkerFixture(t) + var logs lockedBuffer + f.worker.Logger = slog.New(slog.NewTextHandler(&logs, nil)) + f.putUpstream(t, "repo_ih", "own", "proj", host) + var err error + want := []string{EventSyncStarted, EventSyncFailed} + if lane == "pull" { + err = f.worker.Run(t.Context(), Job{Org: "acme", RepoID: "repo_ih"}) + } else { + seedLocalRepo(t, f, "repo_ih") + err = f.worker.RunPush(t.Context(), Job{Org: "acme", RepoID: "repo_ih"}) + want = []string{EventPushUpstreamStarted, EventPushUpstreamFailed} + } + if err == nil || err.Error() != cause { + t.Fatalf("run = %v; want %q", err, cause) + } + events := f.rec.list() + assertEventTypes(t, events, want...) + if events[1].Error != FailConfig { + t.Fatalf("error = %q, want %q", events[1].Error, FailConfig) + } + if reached.Load() { + t.Fatal("the run dialed the upstream of a host that POST /repos refuses") + } + if got := logs.String(); !strings.Contains(got, `cause="`+cause+`"`) || strings.Contains(got, host) { + t.Fatalf("the failure log lacks the reason or names the host: %s", got) + } + }) + } + } +} diff --git a/internal/sync/upstreamhost.go b/internal/sync/upstreamhost.go new file mode 100644 index 00000000..d62c2ccf --- /dev/null +++ b/internal/sync/upstreamhost.go @@ -0,0 +1,171 @@ +package sync + +import ( + "net/netip" + neturl "net/url" + "strconv" + "strings" +) + +// InvalidUpstreamHostReason is the reason that POST /repos gives for a host +// that ValidUpstreamHost refuses (the invalid_params reason of +// `upstream.host` or `base_repo.upstream_host`). A run that finds such a +// host stored fails with the same reason (upstreamURLFailure). +const InvalidUpstreamHostReason = "is not a valid host" + +// ValidUpstreamHost accepts the two documented upstream_host shapes — a +// bare hostname[:port] or an http(s):// base URL — and nothing else. Both +// need a hostname, so a host such as ":80" is refused. +// CloneURL uses the value verbatim as the base of a git fetch URL, so +// file://, ssh://, transport helpers (ext::), embedded credentials, and +// query/fragment noise must be rejected where the attacker-controlled +// value enters, POST /repos (the worker additionally pins +// GIT_ALLOW_PROTOCOL). +// +// It also refuses hosts that a sync run cannot use as given, so the create +// answers 400 instead of storing a link that the registration check could +// not vet: a hostname that is neither an IP address nor a name that the +// run's resolver asks DNS for (dialableHostname), a port outside 1-65535, +// and a base URL with a query or a fragment, even an empty one +// (`https://h?`), which would end the path of every fetch URL, with a +// space, which curl refuses, or with a scheme that is not lower case +// (`HTTPS://`), which GIT_ALLOW_PROTOCOL refuses. A run refuses such a +// host too when an older release stored it (upstreamURLFailure). +func ValidUpstreamHost(host string) bool { + assertf(host != "", "ValidUpstreamHost: empty host") + if !strings.Contains(host, "://") { + // Bare host[:port]: must round-trip as exactly the authority + // component (no path, userinfo, or control characters). A zone + // is written %25, which the parse decodes, so it never + // round-trips. + u, err := neturl.Parse("https://" + host) + return err == nil && u.Host == host && u.User == nil && + u.Path == "" && u.RawQuery == "" && u.Fragment == "" && dialableAuthority(u) + } + if !strings.HasPrefix(host, "http://") && !strings.HasPrefix(host, "https://") { + return false + } + u, err := neturl.Parse(host) + if err != nil { + return false + } + // A path is allowed (sub-path-hosted forges); credentials belong in + // the credential store, never in the URL. url.Parse drops an empty + // query or fragment and keeps a space in a path, so the bytes are + // checked. + return u.User == nil && !strings.ContainsAny(host, "?# ") && dialableAuthority(u) +} + +// dialableAuthority reports whether the host and port of u name something +// that a sync run can dial: a dialableHostname and, when u has one, a port +// from 1 to 65535. An empty port (`forge.example:`) is the default one. +func dialableAuthority(u *neturl.URL) bool { + if port := u.Port(); port != "" { + if n, err := strconv.Atoi(port); err != nil || n < 1 || n > 65535 { + return false + } + } + return dialableHostname(u.Hostname()) +} + +// dialableHostname reports whether name is an IP address without a zone, +// or a name that the resolver of a sync run asks DNS for and that curl +// does not read as an IPv4 address (curlIPv4Name). A zone-scoped address +// (fe80::1%eth0) is refused: the egress guard refuses it under every +// policy, and the egress answer would repeat its zone, which only the +// upstream URL limit bounds. +func dialableHostname(name string) bool { + if ip, err := netip.ParseAddr(name); err == nil { + return ip.Zone() == "" + } + return resolverName(name) && !curlIPv4Name(name) +} + +// resolverName reports whether the Go resolver of a sync run asks DNS for +// name. A run dials through the egress proxy (startEgressProxy), +// whose resolver refuses any other name (net.isDomainName, after RFC 1035 +// and RFC 3696): at most 253 bytes, or 254 with a final dot; labels of 1 to +// 63 letters, digits, '_' and '-' that do not start or end with '-'; and +// not only digits and dots, which curl reads as an IPv4 address +// (2130706433 is 127.0.0.1) that the registration check never saw. The +// resolver reads /etc/hosts before it applies the rule, but a name that +// only /etc/hosts holds is no upstream host. +// +// A name with other letters goes in its ASCII form +// (xn--bcher-kva.example): the registration check cannot look bücher.example +// up, and whether git and curl convert it depends on the locale that the +// server's git runs in. +func resolverName(name string) bool { + if name == "" || len(name) > 254 || len(name) == 254 && name[253] != '.' { + return false + } + prev := byte('.') + letters := false // a letter, '_' or '-': all digits would be an address + label := 0 + for i := 0; i < len(name); i++ { + c := name[i] + switch { + case 'a' <= c && c <= 'z', 'A' <= c && c <= 'Z', c == '_': + letters = true + label++ + case '0' <= c && c <= '9': + label++ + case c == '-': + if prev == '.' { + return false + } + letters = true + label++ + case c == '.': + if prev == '.' || prev == '-' || label > 63 { + return false + } + label = 0 + default: + return false + } + prev = c + } + return prev != '-' && label <= 63 && letters +} + +// curlIPv4Name reports whether curl reads name as an IPv4 address, as its +// URL parser does (ipv4_normalize): one to four labels, a final dot aside, +// each a decimal, octal (leading 0) or hexadecimal (0x) number of at most +// 32 bits, all but the last at most 255, and the last within the bits the +// others leave, such as 2130706433, 0x7f000001, 0x7f.1 or 1.2.3.0x4. curl +// dials such a name as the address (127.0.0.1, 1.2.3.4), which the +// registration check, which reads it as a name, never saw. A name that only +// looks numeric, such as 0x100.1 or 0x100000000, stays a name for curl. +func curlIPv4Name(name string) bool { + labels := strings.Split(strings.TrimSuffix(name, "."), ".") + if len(labels) > 4 { + return false + } + last := len(labels) - 1 + for i, label := range labels { + v, ok := curlIPv4Part(label) + if !ok || i < last && v > 0xff || i == last && v > 0xffffffff>>(8*last) { + return false + } + } + return true +} + +// curlIPv4Part parses one label of a numerical IPv4 address as curl does: +// 0x or 0X and hexadecimal digits, 0 and octal digits, or decimal digits, +// at most 32 bits. +func curlIPv4Part(label string) (uint64, bool) { + base, digits := 10, label + switch { + case len(label) >= 2 && label[0] == '0' && (label[1] == 'x' || label[1] == 'X'): + base, digits = 16, label[2:] + case len(label) >= 1 && label[0] == '0': + base = 8 + } + if digits == "" { + return 0, false + } + v, err := strconv.ParseUint(digits, base, 32) + return v, err == nil +} diff --git a/internal/sync/upstreamhost_test.go b/internal/sync/upstreamhost_test.go new file mode 100644 index 00000000..b5c38bba --- /dev/null +++ b/internal/sync/upstreamhost_test.go @@ -0,0 +1,135 @@ +package sync + +import ( + "strings" + "testing" +) + +// TestValidUpstreamHost: an upstream host is a bare host[:port] or an +// http(s) base URL whose hostname a sync run can dial: an IP address +// without a zone, or a name that Go's resolver looks up (net.isDomainName), +// with a port from 1 to 65535 and no query or fragment. +func TestValidUpstreamHost(t *testing.T) { + label63 := strings.Repeat("a", 63) + // 63+1+63+1+63+1+61 = 253 bytes, the longest name without a final dot. + name253 := label63 + "." + label63 + "." + label63 + "." + strings.Repeat("b", 61) + for _, host := range []string{ + "forge.example", + "forge.example.", + "FORGE.Example:8443", + "a_b.example", + "xn--bcher-kva.example", + "localhost", + "1forge.example", + label63 + ".example", + "forge." + label63, + "1-1", + "1_1", + "cafe", + "0xg.example", + // Names that only look numeric stay names for curl: five labels, + // a part out of IPv4's range, an empty or a bad number. + "0x1.0x2.0x3.0x4.0x5", + "0x0.0x0.0x0.0x0.0x0", + "0x100.1", + "0x100000000", + "0xff.0x1000000", + "0xff.0xff.0x10000", + "0x", + "0x.1", + "1.0x", + "09.0x1", + "0xg", + "axe", + "6x6", + // A 63-byte label of letters, digits and dashes. + "a" + strings.Repeat("1", 30) + strings.Repeat("-", 30) + "bc", + name253, + name253 + ".", + "10.0.0.1", + "10.0.0.1:1", + "[::1]:65535", + "[2001:db8::1]", + "https://forge.example", + "https://forge.example/sub/path/", + "http://127.0.0.1:3000", + "https://[2001:db8::1]:443/git", + "https://forge.example/a%3Fb", + "forge.example:", + "https://forge.example:/git", + } { + if !ValidUpstreamHost(host) { + t.Errorf("ValidUpstreamHost(%q) = false, want true", host) + } + } + for _, host := range []string{ + // Names that the resolver refuses. + "<<<.com", + "https://<<<.com", + "bücher.example", + "https://bücher.example", + "a..b.example", + ".forge.example", + "-forge.example", + "forge-.example", + "forge.-example", + "forge.example-", + label63 + "a.example", + "a" + strings.Repeat("1", 63) + ".example", + "a" + strings.Repeat("-", 62) + "a.example", + "forge." + label63 + "a", + name253 + "a", + name253 + "a.", + ".", + // Only digits and dots, but no IP address: curl would read + // 2130706433 as 127.0.0.1, Go's resolver refuses it. + "2130706433", + "1.2.3", + "https://1.2.3", + "01.2.3.4", + "1.2.3.4.5", + // Hexadecimal forms that curl reads as IPv4 too. + "0x7f000001", + "0x7f.1", + "0X7F.0.0.1", + "1.2.3.0x4", + "0x7f.1.", + "0xa9fea9fe", + "0xA9FEA9FE", + "0xA.0xA.0xA.0xA", + "0xffffffff", + "0xff.0xffffff", + "0xff.0xff.0xffff", + "0377.0377.0377.0377", + "http://0x7f000001:3000/git", + // A zone. + "https://1.2.3.4%25eth0", + "https://[fe80::1%25eth0]", + "[fe80::1%25eth0]", + // Ports. + "forge.example:0", + "forge.example:65536", + "https://forge.example:99999/git", + // A query or fragment, even an empty one. + "https://forge.example?", + "https://forge.example#", + "https://forge.example/git?", + "https://forge.example/git?a=b", + "https://forge.example/git#x", + // A space, which curl refuses, and a scheme that is not lower + // case, which GIT_ALLOW_PROTOCOL refuses. + "https://forge.example/a b", + "HTTPS://forge.example", + "Http://forge.example", + // Unchanged refusals. + ":80", + "https://:80", + "forge.example/git", + "ssh://forge.example", + "https://u:p@forge.example", + } { + if ValidUpstreamHost(host) { + t.Errorf("ValidUpstreamHost(%q) = true, want false", host) + } + } +} diff --git a/internal/sync/worker.go b/internal/sync/worker.go index 8297ac21..f7ee9eeb 100644 --- a/internal/sync/worker.go +++ b/internal/sync/worker.go @@ -1043,10 +1043,18 @@ const MaxCredentialBytes = gitlimit.MaxArgBytes - len("OPENSTORAGE_ASKPASS_PASSW // checks it again for links registered before. const MaxUpstreamURLBytes = gitlimit.MaxArgBytes - len("/") -// upstreamURLFailure fails a run whose upstream URL is over -// MaxUpstreamURLBytes, before any git starts: such a link fails the same -// way on every run, so it is configuration. The error never names the URL. +// upstreamURLFailure fails a run whose upstream URL a run cannot use, +// before any git starts: a host that ValidUpstreamHost refuses, such as +// one with non-ASCII letters that an older release stored, or a URL over +// MaxUpstreamURLBytes. Such a link fails the same way on every run, so it +// is configuration. The error gives POST /repos's reason for the host +// (InvalidUpstreamHostReason) and never names the host or the URL. A +// file:// host is the tests' fixture: POST /repos never stores one, and +// GIT_ALLOW_PROTOCOL refuses it in production. func upstreamURLFailure(up Upstream) *syncFailure { + if !strings.HasPrefix(up.UpstreamHost, "file://") && !ValidUpstreamHost(up.UpstreamHost) { + return &syncFailure{code: FailConfig, cause: errors.New("upstream host " + InvalidUpstreamHostReason)} + } if len(up.CloneURL()) <= MaxUpstreamURLBytes { return nil } diff --git a/internal/sync/worker_push.go b/internal/sync/worker_push.go index 038eed99..e6ccb14f 100644 --- a/internal/sync/worker_push.go +++ b/internal/sync/worker_push.go @@ -16,7 +16,7 @@ package sync // // Egress posture: the push targets the CloneURL the pull lane already // fetches from — a host validated at link time (providerHosts / -// validUpstreamHost) — under the same hermetic environment and +// ValidUpstreamHost) — under the same hermetic environment and // GIT_ALLOW_PROTOCOL pin, PLUS a dial-time destination guard (egress.go): // writes into private networks are a stronger SSRF primitive than the // pull lane's reads, so every http(s) push dials through a vetting proxy