diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de17234..64b69b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,13 +20,18 @@ jobs: fail-fast: false matrix: include: - # Primary target: newest stable Elixir (1.19.x) on newest OTP (28.x). - # `erlef/setup-beam` resolves "1.19" to the latest 1.19.y.z that has - # an OTP-28 build on hex.pm. Owns the quality gates (format, - # credo, dialyzer, integration). + # Primary target: newest stable Elixir (1.20.x) on newest OTP (29.x). + # `erlef/setup-beam` resolves "1.20" to the latest 1.20.y.z that has + # an OTP-29 build. Owns the quality gates (format, credo, dialyzer, + # integration). Elixir 1.20 supports OTP 27-29. + - elixir: "1.20" + otp: "29" + primary: true + # Previous stable line — a compile + test smoke to catch regressions + # against 1.19/OTP-28. Quality gates run once, on the primary. - elixir: "1.19" otp: "28" - primary: true + primary: false # Minimum-supported version — matches the `elixir: "~> 1.17"` # constraint in mix.exs. If we ever need a 1.18+ feature, # bump both the constraint and this matrix entry. Compile + @@ -159,3 +164,34 @@ jobs: CF_ARTIFACT_REMOTE: ${{ secrets.CF_ARTIFACT_REMOTE }} CF_ARTIFACT_TOKEN: ${{ secrets.CF_ARTIFACT_TOKEN }} run: mix test --warnings-as-errors --only cloudflare + + consumer-smoke: + name: Consumer install smoke (package + use) + runs-on: ubuntu-24.04 + # Separate from the `test` job on purpose: this builds the package + # exactly as it ships to Hex, depends on it from a throwaway project + # compiled in :prod, and uses the public API end-to-end. It catches + # missing `files:` entries and compile-time coupling to dev/test/ + # optional deps — neither of which the in-repo suite can see. + # Live network (clones a public repo); runs on every push and PR. + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Elixir / OTP + uses: erlef/setup-beam@v1 + with: + elixir-version: "1.19" + otp-version: "28" + version-type: loose + + - name: Install local Hex & rebar + run: | + mix local.hex --force + mix local.rebar --force + + - name: Fetch dependencies + run: mix deps.get + + - name: Consumer-install smoke (clones elixir-ai-tools/just_bash) + run: bash scripts/consumer_smoke.sh diff --git a/.gitignore b/.gitignore index d2cb7f0..9209ff6 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,10 @@ ex_git-*.tar # Temporary files, for example, from tests. /tmp/ +# Local scratch: internal notes, third-party product critiques, repro +# scripts. Never published to GitHub or hex. +/scratch/ + # Environment / secrets. Never commit. .env .env.* @@ -33,3 +37,6 @@ ex_git-*.tar # mix dialyzer --plt; never commit them. priv/plts/ + +# hex.build package artifact +/exgit-*.tar diff --git a/.tool-versions b/.tool-versions index 9d1de14..088edd2 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,2 @@ -elixir 1.19.5-otp-28 -erlang 28.4 +erlang 29.0.2 +elixir 1.20.2-otp-29 diff --git a/CHANGELOG.md b/CHANGELOG.md index e69e673..6526173 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,65 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.1.0] — 2026-07-01 + +Initial release: pure-Elixir git client for clone, fetch, push over +smart HTTP v2, with lazy partial-clone support and a path-oriented FS +API for agents. + +See [README](./README.md) and [BENCHMARKS on the smoketest +repo](https://github.com/ivarvong/exgit_smoketest/blob/main/BENCHMARKS.md). + +### Security — credential redaction + ref bounds + +- **Telemetry no longer leaks URL-embedded credentials.** A token + embedded in a remote URL (`https://token@host/…`, as git clients + commonly accept) is now redacted to `***` before the URL enters any + `:telemetry` span metadata (`ls_refs`, `fetch`, `push`) or the + `[:exgit, :security, :ref_rejected]` event. Previously such a token + could reach telemetry exporters / log aggregators. Prefer the + `:auth` field regardless — it was already redacted. +- **`ls-refs` responses are capped at 1,000,000 refs.** A hostile or + broken server can no longer stream unbounded refs into client + memory; the transport stream halts once the cap trips and the + caller gets `{:error, {:too_many_refs, cap}}`. Tunable via the + `:max_refs` option on `Exgit.Transport.HTTP.new/2`. Real repos + (linux, esp-idf) sit far below the default. +- **Dependencies bumped to clear HTTP-stack advisories.** `req` + `0.5.17 → 0.6.2` and `mint` `1.7.1 → 1.9.0` resolve the decompression- + bomb DoS (CVE-2026-49755), multipart header injection + (CVE-2026-49756), and HTTP/2 CONTINUATION flood (CVE-2026-49754). + The cross-origin credential-leak test suite was re-run against the + new `req` line. The only remaining `mix hex.audit` advisories are in + `cowlib`, reachable solely through the `only: :test` `bypass` + dependency — never part of the published package or a consumer's + runtime. + +### Notes + +- The optional `:vfs` integration depends on the pre-1.0 `vfs` + package (`~> 0.1.0`), which itself depends on exgit. The cycle is + handled with `runtime: false` + a compile-time `Code.ensure_loaded?` + guard, but treat the integration as pre-release and pin `vfs` if you + rely on it. + +### Added — size-aware reads + +- **`Exgit.FS.size/3`** — byte size of a blob at a path *without* + materializing its content. The size-aware companion to + `read_path/4`: gate on it before pulling a blob into memory. + O(1) for the in-memory store; on-disk loose objects inflate only + the header. Resolving the path may fetch trees (small) on a lazy + clone, but never the blob — an un-fetched blob returns + `{:error, :not_local}` instead of triggering a possibly-multi-GB + fetch. Directories return `{:error, :not_a_blob}`; gitlink + (submodule) entries return `{:error, :submodule}` — as do + `read_path/4` lookups on them, while `stat/3` reports + `%{type: :submodule}` without fetching. +- **`Exgit.ObjectStore.object_size/2`** — new protocol callback + backing the above. Memory keeps a parallel `sha => size` index + (no extra decompression); `Promisor` answers from cache or + returns `{:error, :not_local}` without fetching. ### Added — observability + workload bench @@ -18,7 +76,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **`Exgit.Repository.memory_report/1`** — structured memory report (object counts by type, cache_bytes, max_cache_bytes, mode, backend) with consistent shape across all object-store - backends. Suitable for emission into observability stacks. + backends; counts report `:unknown` for backends that can't be + introspected (Disk). Suitable for emission into observability + stacks. - **`bench/agent_workload.exs`** — realistic agent-session benchmark: clone + prefetch + ls + grep + reads, with `:cold` and `:hot` variants. Reports per-op breakdown + peak cache @@ -392,11 +452,3 @@ taking the opportunity to land the right shapes before v0.1. corrupt or tampered objects return tagged errors instead of crashing. -## [0.1.0] — 2026-04-17 - -Initial release: pure-Elixir git client for clone, fetch, push over -smart HTTP v2, with lazy partial-clone support and a path-oriented FS -API for agents. - -See [README](./README.md) and [BENCHMARKS on the smoketest -repo](https://github.com/ivarvong/exgit_smoketest/blob/main/BENCHMARKS.md). diff --git a/README.md b/README.md index e61ac15..9d22121 100644 --- a/README.md +++ b/README.md @@ -338,11 +338,14 @@ What's in place: eviction for long-running agent loops. - **Protocol v2 `symrefs`** — `Exgit.clone/2` picks the server's actual HEAD target instead of guessing main/master. -- **399 tests, 29 properties, 0 failures** across default, slow, - real_git, and live-integration tiers. -- **CI gates**: Elixir 1.17 / OTP 27 on ubuntu-24.04 with - warnings-as-errors, Credo, Dialyzer, format check, partial-clone - roundtrip against GitHub. +- **870 tests (52 properties), 0 failures** across the default, + slow, and real_git tiers; live-integration and authenticated + tiers run in CI on top. +- **CI gates**: three-tier matrix on ubuntu-24.04 — Elixir 1.20 / + OTP 29 (primary: warnings-as-errors, Credo, Dialyzer, format + check, partial-clone roundtrip against GitHub), plus 1.19 / OTP 28 + and 1.17 / OTP 27 (minimum-supported) compile-and-test tiers, and + a consumer-install smoke test of the built hex package. See [CHANGELOG.md](./CHANGELOG.md) for details and [SECURITY.md](./SECURITY.md) for the threat model. @@ -370,7 +373,9 @@ changes require a major-version bump. Functions annotated `@doc experimental: true` — currently `FS.prefetch/3` and `Repository.materialize/2` — are explicitly -exempt from SemVer guarantees until marked stable. The `:lazy` and +exempt from SemVer guarantees until marked stable. The whole +`Exgit.Index` module is likewise experimental (see its `@moduledoc`) +and may change in any release. The `:lazy` and `:filter` options on `Exgit.clone/2` are also experimental: the threading contract (`{:ok, result, repo}` shape) may evolve before v1.0. diff --git a/lib/exgit.ex b/lib/exgit.ex index 8ca5bf7..7cde28e 100644 --- a/lib/exgit.ex +++ b/lib/exgit.ex @@ -21,6 +21,34 @@ defmodule Exgit do alias Exgit.{Config, ObjectStore, Pack, RefStore, Repository, Transport} + @typedoc """ + A remote endpoint for `clone/2`, `fetch/3`, and `push/3`. + + Either a URL string (`"https://..."` or `"file://..."`), one of the + built-in transport structs (`Exgit.Transport.File`, + `Exgit.Transport.HTTP`), or any struct implementing the + `Exgit.Transport` protocol. + """ + @type remote :: String.t() | Transport.File.t() | Transport.HTTP.t() | struct() + + @doc """ + Create an empty bare repository. + + {:ok, repo} = Exgit.init() + {:ok, repo} = Exgit.init(path: "/tmp/repo.git") + + ## Options + + * `:path` — create the repo on disk at this path, backed by + `ObjectStore.Disk`/`RefStore.Disk`. Default: in-memory + (lost on process exit). + + ## Returns + + `{:ok, %Repository{}}` with `HEAD` pointing at `refs/heads/main`. + When `:path` is set and any directory or file write fails, returns + `{:error, {:init_failed, reason}}` with the underlying posix reason. + """ @spec init(keyword()) :: {:ok, Repository.t()} | {:error, term()} def init(opts \\ []) do case Keyword.get(opts, :path) do @@ -29,6 +57,22 @@ defmodule Exgit do end end + @doc """ + Open an existing on-disk repository. + + {:ok, repo} = Exgit.open("/tmp/repo.git") + + `path` must point at a git directory — a bare repo or a `.git` + directory — containing a `HEAD` file and an `objects` directory. + The repo's `config` file is read when present; a missing or + unparseable config falls back to an empty `Exgit.Config`. + + ## Returns + + `{:ok, %Repository{}}` backed by `ObjectStore.Disk`/`RefStore.Disk`, + or `{:error, {:not_a_repository, reason}}` when the path doesn't + look like a git directory. + """ @spec open(Path.t(), keyword()) :: {:ok, Repository.t()} | {:error, term()} def open(path, _opts \\ []) do head_path = Path.join(path, "HEAD") @@ -114,8 +158,7 @@ defmodule Exgit do `Exgit.Repository.materialize/2` to convert `:lazy` → `:eager` after reading what you need. """ - @spec clone(String.t() | Transport.File.t() | Transport.HTTP.t(), keyword()) :: - {:ok, Repository.t()} | {:error, term()} + @spec clone(remote(), keyword()) :: {:ok, Repository.t()} | {:error, term()} def clone(source, opts \\ []) do cond do disk_partial_clone?(opts) -> @@ -239,7 +282,6 @@ defmodule Exgit do spec -> case Exgit.Filter.encode(spec) do - :none -> {:ok, :none} {:ok, wire} -> {:ok, wire} {:error, _} = err -> err end @@ -314,7 +356,30 @@ defmodule Exgit do end end - @spec fetch(Repository.t(), String.t() | term(), keyword()) :: + @doc """ + Fetch updates from a remote into an existing repository. + + {:ok, repo} = Exgit.fetch(repo, "https://github.com/user/repo") + + Lists the remote's refs, fetches any objects reachable from them, + and records remote-tracking refs under `refs/remotes//` + (for branches) and `refs/tags/` (for tags). Local branches and + `HEAD` are never moved. + + ## Options + + * `:remote` — remote name to record tracking refs under. + Default: `"origin"`. + + * `:prefix` — ref prefixes to list on the remote. + Default: `["refs/heads/", "refs/tags/"]`. + + ## Returns + + `{:ok, %Repository{}}` with the updated object and ref stores, or + `{:error, reason}` when listing refs or fetching the pack fails. + """ + @spec fetch(Repository.t(), remote(), keyword()) :: {:ok, Repository.t()} | {:error, term()} def fetch(%Repository{} = repo, source, opts \\ []) do transport = to_transport(source, opts) @@ -335,23 +400,57 @@ defmodule Exgit do ref to the same name on the remote (creating it if absent). * `{:delete, "refs/heads/branch"}` — delete the ref on the remote. + When `:refspecs` is omitted, the repo's current branch — `HEAD`'s + symref target — is pushed. If `HEAD` is not a symbolic ref (e.g. + detached or unset), the push fails with `{:error, :no_refspecs}`. + + ## Errors + + * `{:error, :no_refspecs}` — `:refspecs` omitted and `HEAD` + doesn't point at a branch. + * `{:error, {:local_ref_not_found, name}}` — a named refspec + doesn't resolve in the local ref store. The whole push fails; + nothing is sent to the remote. (Deleting a ref the remote + doesn't have remains a silent no-op.) + Returns `{:ok, %{ref_results: [...]}}` on success. """ - @spec push(Repository.t(), String.t() | term(), keyword()) :: + @spec push(Repository.t(), remote(), keyword()) :: {:ok, map()} | {:error, term()} def push(%Repository{} = repo, dest, opts \\ []) do transport = to_transport(dest, opts) - refspecs = Keyword.get(opts, :refspecs, ["refs/heads/main"]) - ref_names = Enum.map(refspecs, &refspec_ref_name/1) - remote_refs = load_remote_refs_for_push(transport, ref_names) - {updates, objects} = plan_push_updates(refspecs, repo, remote_refs) + with {:ok, refspecs} <- push_refspecs(repo, opts) do + ref_names = Enum.map(refspecs, &refspec_ref_name/1) + remote_refs = load_remote_refs_for_push(transport, ref_names) - if updates == [] do - {:ok, %{ref_results: []}} - else - pack = build_push_pack(updates, objects) - Transport.push(transport, Enum.reverse(updates), pack, []) + case plan_push_updates(refspecs, repo, remote_refs) do + {:ok, [], _objects} -> + {:ok, %{ref_results: []}} + + {:ok, updates, objects} -> + pack = build_push_pack(updates, objects) + Transport.push(transport, Enum.reverse(updates), pack, []) + + {:error, _} = err -> + err + end + end + end + + # When the caller doesn't name refspecs, push the current branch — + # HEAD's symref target. A detached or unset HEAD leaves nothing + # sensible to push, so the caller must name refspecs explicitly. + defp push_refspecs(repo, opts) do + case Keyword.fetch(opts, :refspecs) do + {:ok, refspecs} -> + {:ok, refspecs} + + :error -> + case RefStore.read(repo.ref_store, "HEAD") do + {:ok, {:symbolic, ref_name}} -> {:ok, [ref_name]} + _ -> {:error, :no_refspecs} + end end end @@ -366,7 +465,7 @@ defmodule Exgit do end defp plan_push_updates(refspecs, repo, remote_refs) do - Enum.reduce(refspecs, {[], []}, fn refspec, {upd, objs} -> + Enum.reduce_while(refspecs, {:ok, [], []}, fn refspec, {:ok, upd, objs} -> case plan_push(refspec, repo, remote_refs) do {:update, ref_name, old_sha, new_sha, sha_for_objects} -> new_objs = @@ -374,10 +473,13 @@ defmodule Exgit do do: collect_push_objects(repo.object_store, sha_for_objects, remote_refs), else: [] - {[{ref_name, old_sha, new_sha} | upd], objs ++ new_objs} + {:cont, {:ok, [{ref_name, old_sha, new_sha} | upd], objs ++ new_objs}} :skip -> - {upd, objs} + {:cont, {:ok, upd, objs}} + + {:error, _} = err -> + {:halt, err} end end) end @@ -421,7 +523,7 @@ defmodule Exgit do {:update, ref_name, old_sha, sha, sha} _ -> - :skip + {:error, {:local_ref_not_found, ref_name}} end end @@ -431,7 +533,7 @@ defmodule Exgit do # URL string that we can map to the built-in File/HTTP transports. defp to_transport(url, opts) when is_binary(url) do if String.starts_with?(url, "file://") do - path = String.trim_leading(url, "file://") + path = String.replace_prefix(url, "file://", "") Transport.File.new(path) else Transport.HTTP.new(url, opts) @@ -461,6 +563,8 @@ defmodule Exgit do config: config, path: path )} + else + {:error, reason} -> {:error, {:init_failed, reason}} end end @@ -519,7 +623,7 @@ defmodule Exgit do remote_ref = cond do String.starts_with?(ref, "refs/heads/") -> - branch = String.trim_leading(ref, "refs/heads/") + branch = String.replace_prefix(ref, "refs/heads/", "") "refs/remotes/#{remote_name}/#{branch}" String.starts_with?(ref, "refs/tags/") -> @@ -585,7 +689,7 @@ defmodule Exgit do end defp collect_push_objects(store, sha, remote_refs) do - remote_shas = MapSet.new(Map.values(remote_refs)) + remote_shas = Map.new(Map.values(remote_refs), &{&1, true}) collect_reachable(store, [sha], remote_shas) end @@ -593,17 +697,28 @@ defmodule Exgit do # `seen` accumulator so shared subtrees are visited exactly once, and # is bounded by O(heap) rather than O(stack) so it handles deep # histories (millions of commits) without stack overflow. + # `seen` is a plain map used as a membership set (`sha => true`) rather + # than a `MapSet`. MapSet is an opaque type, and threading it through this + # self-recursive accumulator — whose base clause binds `_seen` as a + # wildcard — trips a Dialyzer opacity false-positive. A plain map is + # semantically identical here (set of seen shas), not opaque, and a touch + # faster. + @typep seen_set :: %{optional(binary()) => true} + + @spec collect_reachable(ObjectStore.t(), [binary()], seen_set()) :: [Exgit.Object.t()] defp collect_reachable(store, initial_shas, seen) do do_collect_reachable(store, initial_shas, seen, []) end + @spec do_collect_reachable(ObjectStore.t(), [binary()], seen_set(), [Exgit.Object.t()]) :: + [Exgit.Object.t()] defp do_collect_reachable(_store, [], _seen, acc), do: Enum.reverse(acc) defp do_collect_reachable(store, [sha | rest], seen, acc) do - if MapSet.member?(seen, sha) do + if Map.has_key?(seen, sha) do do_collect_reachable(store, rest, seen, acc) else - seen = MapSet.put(seen, sha) + seen = Map.put(seen, sha, true) case ObjectStore.get(store, sha) do {:ok, obj} -> diff --git a/lib/exgit/error.ex b/lib/exgit/error.ex index 6f8eb4b..874b1ce 100644 --- a/lib/exgit/error.ex +++ b/lib/exgit/error.ex @@ -20,7 +20,8 @@ defmodule Exgit.Error do Well-known codes: * `:not_found` — object, ref, or path missing. - * `:invalid_ref_name` — ref name failed `Exgit.RefName.valid?/1`. + * `:invalid_ref_name` — ref name failed `Exgit.RefName.valid?/1` + (defensive validation of ref names, including wire input). * `:invalid_hex_header` — a commit/tag header expected 40-char hex and got something else. * `:malformed_tree_entry` — tree decode failed structurally. @@ -30,7 +31,6 @@ defmodule Exgit.Error do * `:zlib_error` — zlib decompression failed on untrusted input. * `:resolved_too_large`, `:object_too_large`, `:pack_too_large` — pack parser resource-limit trip. - * `:invalid_ref_name` — defensive ref name validation tripped. * `:http_error` — non-2xx response from a transport. * `:compare_and_swap_failed` — ref store CAS check failed. diff --git a/lib/exgit/fs.ex b/lib/exgit/fs.ex index ca0d2a1..293eaab 100644 --- a/lib/exgit/fs.ex +++ b/lib/exgit/fs.ex @@ -32,7 +32,11 @@ defmodule Exgit.FS do @type ref :: String.t() | binary() @type path :: String.t() - @type stat :: %{type: :blob | :tree, mode: String.t(), size: non_neg_integer() | nil} + @type stat :: %{ + type: :blob | :tree | :submodule, + mode: String.t(), + size: non_neg_integer() | nil + } @doc """ Read the blob at `path`. Returns `{:ok, {mode, %Blob{}}, repo}` or @@ -41,23 +45,26 @@ defmodule Exgit.FS do ## Options - * `:resolve_lfs_pointers` (default `false`) — when `true`, blobs - detected as git-lfs pointer files are returned as + * `:detect_lfs_pointers` (default `false`) — when `true`, blobs + that parse as git-lfs pointer files are returned as `{:ok, {mode, {:lfs_pointer, info}}, repo}` instead of `{:ok, {mode, %Blob{}}, repo}`. `info` is a map with `:oid`, `:size`, and `:raw` (the original pointer bytes). - An agent reading blobs without this flag against an - LFS-using repo will silently receive ~130-byte pointer - text as if it were file content — a correctness cliff. - See `Exgit.LFS` for detection details. + Detection only — the actual LFS content is never fetched + (that requires a separate batch-API protocol against the LFS + server). Callers that need the real bytes can hand `info.raw` + to an LFS client. An agent reading blobs without this flag + against an LFS-using repo will silently receive ~130-byte + pointer text as if it were file content — a correctness + cliff. See `Exgit.LFS` for detection details. """ @spec read_path(Repository.t(), ref(), path(), keyword()) :: {:ok, {String.t(), Blob.t() | {:lfs_pointer, Exgit.LFS.pointer_info()}}, Repository.t()} - | {:error, :not_found | :not_a_blob | term()} + | {:error, :not_found | :not_a_blob | :submodule | term()} def read_path(%Repository{} = repo, reference, path, opts \\ []) do - resolve_lfs? = Keyword.get(opts, :resolve_lfs_pointers, false) + detect_lfs? = Keyword.get(opts, :detect_lfs_pointers, false) Exgit.Telemetry.span( [:exgit, :fs, :read_path], @@ -65,17 +72,26 @@ defmodule Exgit.FS do fn -> with {:ok, tree_sha, repo} <- resolve_tree(repo, reference), {:ok, {mode, sha}, repo} <- walk_path(repo, tree_sha, normalize_path(path)), + :ok <- reject_gitlink(mode), {:ok, obj, repo} <- fetch_object(repo, sha) do - wrap_blob(obj, mode, repo, resolve_lfs?) + wrap_blob(obj, mode, repo, detect_lfs?) end end ) end + # A gitlink's SHA names a commit in the submodule's own repository — + # it can never be fetched from this repo's store, so fail before + # `fetch_object` attempts a doomed lookup (or, on a lazy clone, a + # doomed network fetch). Mirrors `size/3`. + defp reject_gitlink(mode) do + if gitlink_mode?(mode), do: {:error, :submodule}, else: :ok + end + # Post-process a fetched object into the `read_path` return shape. # Split out so the main `with` chain stays flat; credo flags the # in-line nested `case + if + case` otherwise. - defp wrap_blob(%Blob{data: data} = b, mode, repo, true = _resolve_lfs?) do + defp wrap_blob(%Blob{data: data} = b, mode, repo, true = _detect_lfs?) do case Exgit.LFS.parse(data) do {:ok, info} -> {:ok, {mode, {:lfs_pointer, info}}, repo} {:error, _} -> {:ok, {mode, b}, repo} @@ -246,6 +262,10 @@ defmodule Exgit.FS do @doc """ Stat the path. Returns `{:ok, stat, repo}`. + + Gitlink (submodule) entries stat as `%{type: :submodule, size: nil}` + without fetching anything — the entry's SHA lives in the submodule's + own repository. """ @spec stat(Repository.t(), ref(), path()) :: {:ok, stat(), Repository.t()} | {:error, term()} def stat(%Repository{} = repo, reference, path) do @@ -257,18 +277,78 @@ defmodule Exgit.FS do {:ok, %{type: :tree, mode: "40000", size: nil}, repo} _ -> - with {:ok, {mode, sha}, repo} <- walk_path(repo, tree_sha, segments), - {:ok, obj, repo} <- fetch_object(repo, sha) do - case obj do - %Blob{data: d} -> {:ok, %{type: :blob, mode: mode, size: byte_size(d)}, repo} - %Tree{} -> {:ok, %{type: :tree, mode: mode, size: nil}, repo} - _ -> {:error, :unknown_type} - end + with {:ok, {mode, sha}, repo} <- walk_path(repo, tree_sha, segments) do + stat_entry(repo, mode, sha) end end end end + @doc """ + Size in bytes of the blob at `path` — WITHOUT reading its content. + + The size-aware companion to `read_path/4`: use it to decide whether a + blob is too large to pull into memory *before* you pull it. For the + in-memory store this is O(1) (the size is indexed, not recomputed); + for on-disk loose objects it inflates only the header. + + Resolving the path may fetch *tree* objects (small) on a lazy clone, + but the blob itself is never fetched. For a lazy/partial clone whose + blob has not been materialized yet, returns `{:error, :not_local}` + rather than triggering a possibly-multi-GB fetch — call `read_path/4` + when you actually want the bytes. Directories return + `{:error, :not_a_blob}`. Gitlink (submodule) entries return + `{:error, :submodule}` — the entry's SHA names a commit in the + submodule's own repository, so it has no size here and no amount + of prefetching will make it local. + + {:ok, size, repo} = Exgit.FS.size(repo, "HEAD", "go.mod") + + """ + @spec size(Repository.t(), ref(), path()) :: + {:ok, non_neg_integer(), Repository.t()} | {:error, term()} + def size(%Repository{} = repo, reference, path) do + with {:ok, tree_sha, repo} <- resolve_tree(repo, reference), + {:ok, {mode, sha}, repo} <- walk_path(repo, tree_sha, normalize_path(path)) do + cond do + dir_mode?(mode) -> + {:error, :not_a_blob} + + gitlink_mode?(mode) -> + {:error, :submodule} + + true -> + case ObjectStore.object_size(repo.object_store, sha) do + {:ok, size} -> {:ok, size, repo} + {:error, _} = err -> err + end + end + end + end + + # Stat one resolved tree entry. Gitlinks short-circuit before any + # object lookup — the commit they name is never in this repo's store. + defp stat_entry(repo, mode, sha) do + if gitlink_mode?(mode) do + {:ok, %{type: :submodule, mode: mode, size: nil}, repo} + else + with {:ok, obj, repo} <- fetch_object(repo, sha) do + case obj do + %Blob{data: d} -> {:ok, %{type: :blob, mode: mode, size: byte_size(d)}, repo} + %Tree{} -> {:ok, %{type: :tree, mode: mode, size: nil}, repo} + _ -> {:error, :unknown_type} + end + end + end + end + + defp dir_mode?("40000"), do: true + defp dir_mode?("040000"), do: true + defp dir_mode?(_), do: false + + defp gitlink_mode?("160000"), do: true + defp gitlink_mode?(_), do: false + @doc """ Return true if the path exists under the given reference. @@ -862,19 +942,22 @@ defmodule Exgit.FS do end @doc """ - Glob paths matching `pattern`. Streaming; does not grow the cache. + List the file paths matching `pattern`, like `Path.wildcard/1`. + + Walks the tree lazily (pure `ObjectStore.get/2`, does not grow the + cache) and returns the sorted list of matching paths. Unlike + `walk/2` and `grep/4` this is not a stream — sorting requires + collecting every match, so the whole tree is traversed before the + call returns. An unmatched pattern returns `[]`. """ - @spec glob(Repository.t(), ref(), String.t()) :: {:ok, [String.t()]} + @spec glob(Repository.t(), ref(), String.t()) :: [String.t()] def glob(%Repository{} = repo, reference, pattern) do regex = compile_glob(pattern) - paths = - walk(repo, reference) - |> Stream.map(&elem(&1, 0)) - |> Stream.filter(&Regex.match?(regex, &1)) - |> Enum.sort() - - {:ok, paths} + walk(repo, reference) + |> Stream.map(&elem(&1, 0)) + |> Stream.filter(&Regex.match?(regex, &1)) + |> Enum.sort() end @type grep_match :: %{ diff --git a/lib/exgit/index.ex b/lib/exgit/index.ex index 5dce2be..b1a74e4 100644 --- a/lib/exgit/index.ex +++ b/lib/exgit/index.ex @@ -127,7 +127,7 @@ defmodule Exgit.Index do # remote-controlled. defp valid_checksum?(data) when byte_size(data) >= 20 do content_size = byte_size(data) - 20 - <> = data + <> = data :crypto.hash(:sha, content) == checksum end @@ -193,7 +193,7 @@ defmodule Exgit.Index do # the real length is unknown and we must read until the next NUL. defp read_name(data, len) when len < 0xFFF do case data do - <> -> {:ok, name, rest} + <> -> {:ok, name, rest} _ -> {:error, :truncated_name} end end @@ -222,7 +222,7 @@ defmodule Exgit.Index do _ = pad if byte_size(data) >= pad_remaining do - <<_::binary-size(pad_remaining), rest::binary>> = data + <<_::binary-size(^pad_remaining), rest::binary>> = data {:ok, rest} else {:error, :truncated_padding} diff --git a/lib/exgit/lfs.ex b/lib/exgit/lfs.ex index fe69d65..625eb65 100644 --- a/lib/exgit/lfs.ex +++ b/lib/exgit/lfs.ex @@ -15,10 +15,12 @@ defmodule Exgit.LFS do text, not the file contents — a silent correctness cliff if the agent doesn't know to check. - `Exgit.FS.read_path/4` with `resolve_lfs_pointers: true` uses + `Exgit.FS.read_path/4` with `detect_lfs_pointers: true` uses `parse/1` here to surface detected pointers as a structured `{:lfs_pointer, info}` tuple instead of returning the pointer - text as if it were content. + text as if it were content. Detection is the whole story — + fetching the pointed-to content from the LFS server is left to + the caller. ## Strictness diff --git a/lib/exgit/object/commit.ex b/lib/exgit/object/commit.ex index d821427..f977b66 100644 --- a/lib/exgit/object/commit.ex +++ b/lib/exgit/object/commit.ex @@ -92,7 +92,7 @@ defmodule Exgit.Object.Commit do def decode(bytes) when is_binary(bytes) do case :binary.match(bytes, "\n\n") do {pos, 2} -> - <> = bytes + <> = bytes case parse_headers(raw_headers) do {:ok, headers} -> diff --git a/lib/exgit/object/tag.ex b/lib/exgit/object/tag.ex index e48b21a..b17e78d 100644 --- a/lib/exgit/object/tag.ex +++ b/lib/exgit/object/tag.ex @@ -2,80 +2,108 @@ defmodule Exgit.Object.Tag do @moduledoc """ Git annotated tag object. - `decode/1` validates the `object` header as a 40-char hex string - and stores it as the raw 20-byte binary. Accessors on a decoded - tag are infallible — a hostile remote cannot DoS a walk/diff by - shipping a tag with non-hex header bytes. See + Like `Exgit.Object.Commit`, a tag is represented as a message plus an + **ordered list of headers**. Unknown headers and continuation lines + (multi-line values such as embedded signatures) are preserved verbatim + and in order, so `decode |> encode` is byte-exact and re-encoding a + decoded tag never changes its SHA. + + The `:object` field caches the validated `object` header as a raw + 20-byte binary; `encode/1` reads only `:headers` and `:message`. + Convenience accessors (`type/1`, `tag/1`, `tagger/1`) extract + well-known headers. + + `decode/1` validates the `object` header as a 40-char hex string. + Accessors on a decoded tag are infallible — a hostile remote cannot + DoS a walk/diff by shipping a tag with non-hex header bytes. See `test/exgit/security/tag_malformed_hex_test.exs` for the regression. """ alias Exgit.Object.Hex - @enforce_keys [:object, :type, :tag, :message] - defstruct [:object, :type, :tag, :message, tagger: nil] + @enforce_keys [:object, :headers, :message] + defstruct [:object, :headers, :message] + @type header :: {name :: String.t(), value :: String.t()} @type t :: %__MODULE__{ object: binary(), - type: String.t(), - tag: String.t(), - tagger: String.t() | nil, + headers: [header()], message: String.t() } @spec new(keyword()) :: t() def new(opts) do - %__MODULE__{ - object: Keyword.fetch!(opts, :object), - type: Keyword.get(opts, :type, "commit"), - tag: Keyword.fetch!(opts, :tag), - tagger: Keyword.get(opts, :tagger), - message: Keyword.fetch!(opts, :message) - } + object = Keyword.fetch!(opts, :object) + type = Keyword.get(opts, :type, "commit") + tag = Keyword.fetch!(opts, :tag) + tagger = Keyword.get(opts, :tagger) + message = Keyword.fetch!(opts, :message) + + # `Hex.encode/1` accepts either a raw 20-byte sha or a 40-char hex + # string; the field always holds the raw form, the header the hex. + object_hex = Hex.encode(object) + + headers = + [{"object", object_hex}, {"type", type}, {"tag", tag}] ++ + if(tagger, do: [{"tagger", tagger}], else: []) + + %__MODULE__{object: Hex.decode!(object_hex), headers: headers, message: message} + end + + @spec type(t()) :: String.t() + def type(%__MODULE__{} = t), do: header!(t, "type") + + @spec tag(t()) :: String.t() + def tag(%__MODULE__{} = t), do: header!(t, "tag") + + @spec tagger(t()) :: String.t() | nil + def tagger(%__MODULE__{headers: hs}) do + Enum.find_value(hs, fn + {"tagger", v} -> v + _ -> nil + end) + end + + defp header!(%__MODULE__{headers: hs}, name) do + case Enum.find(hs, fn {n, _} -> n == name end) do + {_, v} -> v + nil -> raise KeyError, key: name + end end @spec encode(t()) :: iolist() - def encode(%__MODULE__{} = t) do + def encode(%__MODULE__{headers: headers, message: message}) do [ - "object ", - Hex.encode(t.object), - ?\n, - "type ", - t.type, - ?\n, - "tag ", - t.tag, - ?\n, - encode_tagger(t.tagger), + Enum.map(headers, &encode_header/1), ?\n, - t.message + message ] end - defp encode_tagger(nil), do: [] - defp encode_tagger(tagger), do: ["tagger ", tagger, ?\n] + defp encode_header({name, value}) do + case String.split(value, "\n") do + [single] -> [name, ?\s, single, ?\n] + [first | rest] -> [name, ?\s, first, ?\n, Enum.map(rest, fn l -> [?\s, l, ?\n] end)] + end + end @spec decode(binary()) :: {:ok, t()} | {:error, term()} def decode(bytes) when is_binary(bytes) do case :binary.match(bytes, "\n\n") do {pos, 2} -> - <> = bytes + <> = bytes case parse_headers(raw_headers) do {:ok, headers} -> - with {:ok, object} <- Map.fetch(headers, :object), - {:ok, type} <- Map.fetch(headers, :type), - {:ok, tag} <- Map.fetch(headers, :tag) do - {:ok, - %__MODULE__{ - object: object, - type: type, - tag: tag, - tagger: Map.get(headers, :tagger), - message: message - }} - else - :error -> {:error, :missing_header} + # A tag must have object + type + tag. The `object` header + # is additionally validated as 40-char hex so that the + # `:object` field (and downstream walks) are infallible — + # we never `raise` on the wire bytes. + with :ok <- ensure_header(headers, "type"), + :ok <- ensure_header(headers, "tag"), + {:ok, object} <- fetch_object(headers) do + {:ok, %__MODULE__{object: object, headers: headers, message: message}} end {:error, _} = err -> @@ -89,33 +117,50 @@ defmodule Exgit.Object.Tag do e -> {:error, {:decode_failed, e}} end - # Parse headers into a map. If any `object` header has non-hex content, - # return an error — a hostile remote could otherwise DoS downstream - # accessors. We never `raise` on the wire bytes. - defp parse_headers(raw) do - raw - |> String.split("\n") - |> Enum.reduce_while({:ok, %{}}, fn line, {:ok, acc} -> - case String.split(line, " ", parts: 2) do - ["object", hex] -> - case Hex.decode(hex) do - {:ok, bin} -> {:cont, {:ok, Map.put(acc, :object, bin)}} - :error -> {:halt, {:error, {:invalid_hex_header, "object", hex}}} - end + defp ensure_header(headers, name) do + if Enum.any?(headers, fn {n, _} -> n == name end), + do: :ok, + else: {:error, {:missing_header, name}} + end - ["type", val] -> - {:cont, {:ok, Map.put(acc, :type, val)}} + defp fetch_object(headers) do + case Enum.find(headers, fn {n, _} -> n == "object" end) do + {_, hex} -> + case Hex.decode(hex) do + {:ok, bin} -> {:ok, bin} + :error -> {:error, {:invalid_hex_header, "object", hex}} + end - ["tag", val] -> - {:cont, {:ok, Map.put(acc, :tag, val)}} + nil -> + {:error, {:missing_header, "object"}} + end + end + + # Parse headers preserving order. Continuation lines (lines starting with + # a space) extend the previous header's value with a leading newline. + # A continuation line appearing before any header is an error. + defp parse_headers(raw) do + lines = String.split(raw, "\n") - ["tagger", val] -> - {:cont, {:ok, Map.put(acc, :tagger, val)}} + Enum.reduce_while(lines, {:ok, []}, fn line, {:ok, acc} -> + case {line, acc} do + {" " <> rest, [{name, val} | tail]} -> + {:cont, {:ok, [{name, val <> "\n" <> rest} | tail]}} - _ -> - {:cont, {:ok, acc}} + {" " <> _, []} -> + {:halt, {:error, :leading_continuation}} + + {line, acc} -> + case String.split(line, " ", parts: 2) do + [name, val] when name != "" -> {:cont, {:ok, [{name, val} | acc]}} + _ -> {:halt, {:error, :malformed_header}} + end end end) + |> case do + {:ok, rev} -> {:ok, Enum.reverse(rev)} + {:error, _} = err -> err + end end @spec sha(t()) :: Exgit.Object.sha() diff --git a/lib/exgit/object/tree.ex b/lib/exgit/object/tree.ex index 7a8087f..91f1cf6 100644 --- a/lib/exgit/object/tree.ex +++ b/lib/exgit/object/tree.ex @@ -9,10 +9,10 @@ defmodule Exgit.Object.Tree do tree's SHA and corrupt verification. `new/1` applies git's canonical ordering (dirs sort as if they had a - trailing `/`) and — by default — normalizes regular file modes via - `canonical_mode/1`. Pass `strict: true` to error on unknown modes - instead of silently coercing, or build the struct directly for a - raw, unvalidated tree. + trailing `/`) and — by default — normalizes modes via + `canonical_mode/1`. Pass `strict: true` to raise on non-canonical + modes instead of silently coercing, or build the struct directly for + a raw, unvalidated tree. > #### Round-trip note {: .warning} > @@ -44,10 +44,14 @@ defmodule Exgit.Object.Tree do Options: - * `:strict` — when `true`, reject unknown modes with a - `:invalid_mode` error. Default: `false` (unknown modes are - silently coerced to `100644` / `100755` based on the executable - bit). + * `:strict` — when `true`, raise `ArgumentError` on any mode that + is not one of the five canonical git modes (`40000`, `100644`, + `100755`, `120000`, `160000`). Default: `false`, which + canonicalizes via `canonical_mode/1`: zero-padded modes normalize + to their canonical form (`"040000"` → `"40000"`, `"0120000"` → + `"120000"`), other octal modes coerce to `100644` / `100755` + based on the executable bit, and non-octal strings pass through + unchanged. """ @spec new([entry()], keyword()) :: t() def new(entries, opts \\ []) when is_list(entries) do @@ -68,10 +72,14 @@ defmodule Exgit.Object.Tree do end @doc """ - Normalize a file mode to one of the canonical git file modes. Used by - `new/1` but NOT by `decode/1`. Unknown modes are returned unchanged - by the one-arg form; the two-arg form with `strict: true` raises - on unknown input. + Normalize a mode string to the canonical git spelling. Used by `new/1` + but NOT by `decode/1`. Octal modes canonicalize by their file-type + bits: directories (`"040000"` → `"40000"`), symlinks (`"0120000"` → + `"120000"`) and gitlinks (`"0160000"` → `"160000"`) keep their type; + everything else coerces to `100644` / `100755` based on the + executable bit. Non-octal strings are returned unchanged by the + one-arg form; the two-arg form with `strict: true` raises + `ArgumentError` on any non-canonical input. """ @spec canonical_mode(String.t()) :: String.t() def canonical_mode(mode), do: canonical_mode(mode, false) @@ -87,7 +95,7 @@ defmodule Exgit.Object.Tree do # Parse as octal (git modes are always octal strings). case Integer.parse(mode, 8) do {n, ""} when not strict -> - if band(n, 0o111) != 0, do: "100755", else: "100644" + canonicalize_octal(n) {_n, ""} when strict -> raise ArgumentError, "invalid git tree mode: #{inspect(mode)}" @@ -100,6 +108,18 @@ defmodule Exgit.Object.Tree do end end + # Canonicalize by the file-type bits (upper 4 octal digits of a + # 16-bit st_mode). A zero-padded "040000" must stay a directory — + # coercing it to a blob mode would silently corrupt the tree. + defp canonicalize_octal(n) do + case band(n, 0o170000) do + 0o040000 -> "40000" + 0o120000 -> "120000" + 0o160000 -> "160000" + _ -> if band(n, 0o111) != 0, do: "100755", else: "100644" + end + end + @spec encode(t()) :: iolist() def encode(%__MODULE__{entries: entries}) do Enum.map(entries, fn {mode, name, sha} -> @@ -180,7 +200,7 @@ defmodule Exgit.Object.Tree do defp take_until(data, byte) do case :binary.match(data, <>) do {pos, 1} -> - <> = data + <> = data {:ok, before, rest} :nomatch -> diff --git a/lib/exgit/object_store.ex b/lib/exgit/object_store.ex index f87853c..272218e 100644 --- a/lib/exgit/object_store.ex +++ b/lib/exgit/object_store.ex @@ -1,4 +1,16 @@ defprotocol Exgit.ObjectStore do + @moduledoc """ + Storage protocol for git objects, keyed by binary SHA-1. + + Implemented by `Exgit.ObjectStore.Memory` (in-heap), + `Exgit.ObjectStore.Disk` (git's on-disk layout), and + `Exgit.ObjectStore.Promisor` (lazy, fetch-on-demand). + + Stores are pure values: every write (`put/2`, `import_objects/2`, + `close_write/2`) returns the updated store, which the caller must + thread forward. Reads never mutate. + """ + @spec get(t, binary()) :: {:ok, Exgit.Object.t()} | {:error, term()} def get(store, sha) @@ -8,6 +20,27 @@ defprotocol Exgit.ObjectStore do @spec has?(t, binary()) :: boolean() def has?(store, sha) + @doc """ + Uncompressed byte size of an object WITHOUT materializing its content. + + The point is a constant-memory size check: callers can decide whether a + blob is too large to read before paying to inflate it into the heap. + + Returns `{:ok, byte_count}` when the store can answer, or: + + * `{:error, :not_found}` — the object is genuinely absent from the + store. + * `{:error, :not_local}` — a meaningful, non-fatal result for lazy + stores (`Promisor`): it means "knowing this size requires a network + fetch", so the caller can opt in rather than have a multi-GB fetch + triggered behind a size check. + * `{:error, term()}` — store-specific failure (e.g. a corrupt loose + object on disk). + """ + @spec object_size(t, binary()) :: + {:ok, non_neg_integer()} | {:error, :not_found | :not_local | term()} + def object_size(store, sha) + @spec import_objects(t, [{atom(), binary(), binary()}]) :: {:ok, t} def import_objects(store, raw_objects) diff --git a/lib/exgit/object_store/disk.ex b/lib/exgit/object_store/disk.ex index e683960..4a62522 100644 --- a/lib/exgit/object_store/disk.ex +++ b/lib/exgit/object_store/disk.ex @@ -79,6 +79,122 @@ defmodule Exgit.ObjectStore.Disk do else: {:error, {:sha_mismatch, expected_sha}} end + @doc """ + Uncompressed object size without materializing the object. + + For a loose object this reads and inflates only enough compressed + bytes to cover the header (`" \\0"`) — constant memory + AND constant I/O, no matter how large the blob. Packed objects fall + back to a full read (the delta chain must be resolved to know the + final size), so this is cheap for loose objects and O(object) for + packed ones. + """ + @spec object_size(t(), binary()) :: {:ok, non_neg_integer()} | {:error, term()} + def object_size(%__MODULE__{root: root}, sha) when byte_size(sha) == 20 do + hex = Base.encode16(sha, case: :lower) + <> = hex + path = Path.join([root, "objects", prefix, rest]) + + case loose_object_size(path) do + {:ok, size} -> {:ok, size} + {:error, :enoent} -> packed_object_size(root, sha) + {:error, reason} -> {:error, reason} + end + end + + defp loose_object_size(path) do + case :file.open(path, [:read, :raw, :binary]) do + {:ok, fd} -> + try do + with {:ok, header} <- inflate_until_null(fd) do + case parse_loose_header(header) do + {:ok, _type, size} -> {:ok, size} + {:error, _} = err -> err + end + end + after + :file.close(fd) + end + + {:error, reason} -> + {:error, reason} + end + end + + # The git object header is tiny; cap the scan so a corrupt or hostile + # object can't make us inflate unbounded output hunting for a NUL. + @max_header_bytes 64 + + # Compressed bytes read from disk per iteration while hunting for the + # header's NUL. The header fits in the first chunk for any real object. + @header_read_chunk_bytes 256 + + # Stream-inflate only until the header's NUL terminator, returning the + # bytes before it. Bounded memory AND bounded I/O: reads small + # compressed chunks from `fd` and never inflates the full object. + defp inflate_until_null(fd) do + z = :zlib.open() + + try do + :zlib.inflateInit(z) + feed_until_null(z, fd, <<>>) + rescue + _ -> {:error, :zlib_error} + catch + _, _ -> {:error, :zlib_error} + after + :zlib.close(z) + end + end + + defp feed_until_null(z, fd, acc) do + case :file.read(fd, @header_read_chunk_bytes) do + {:ok, chunk} -> + case drain_until_null(z, chunk, acc) do + {:need_input, acc} -> feed_until_null(z, fd, acc) + done -> done + end + + # Truncated file: zlib still wants input but there is none. + :eof -> + {:error, :malformed_object_header} + + {:error, reason} -> + {:error, reason} + end + end + + # Inflate `input` and any pending output, scanning for the NUL. + # `{:need_input, acc}` means zlib produced nothing more from this + # chunk — the caller must read more compressed bytes. + defp drain_until_null(z, input, acc) do + {status, out} = :zlib.safeInflate(z, input) + out = IO.iodata_to_binary(out) + acc = acc <> out + + case :binary.match(acc, <<0>>) do + {pos, 1} -> + {:ok, binary_part(acc, 0, pos)} + + :nomatch -> + cond do + byte_size(acc) > @max_header_bytes -> {:error, :malformed_object_header} + status == :finished -> {:error, :malformed_object_header} + out == <<>> -> {:need_input, acc} + true -> drain_until_null(z, [], acc) + end + end + end + + defp packed_object_size(root, sha) do + case raw_from_packs(root, sha) do + # `content` is the already-inflated (delta-resolved) object body; + # its byte_size IS the object size. No decode→re-encode round trip. + {:ok, _type, content} -> {:ok, byte_size(content)} + {:error, _} = err -> err + end + end + @spec put_object(t(), Exgit.Object.t()) :: {:ok, binary()} | {:error, term()} def put_object(%__MODULE__{root: root}, object) do {sha, raw} = object_raw(object) @@ -199,7 +315,7 @@ defmodule Exgit.ObjectStore.Disk do defp parse_loose_object(raw) do case :binary.match(raw, <<0>>) do {pos, 1} -> - <> = raw + <> = raw with {:ok, type, size} <- parse_loose_header(header), :ok <- check_loose_size(content, size) do @@ -250,6 +366,15 @@ defmodule Exgit.ObjectStore.Disk do defp hex_char?(_), do: false defp get_from_packs(root, sha) do + case raw_from_packs(root, sha) do + {:ok, type, content} -> Exgit.Object.decode(type, content) + {:error, _} = err -> err + end + end + + # Locate `sha` in any pack and return its inflated (delta-resolved) + # type + content, without decoding into an `Exgit.Object`. + defp raw_from_packs(root, sha) do pack_dir = Path.join([root, "objects", "pack"]) case File.ls(pack_dir) do @@ -276,7 +401,7 @@ defmodule Exgit.ObjectStore.Disk do # the header we preserved). Parse at that offset, NOT the on-disk # offset. case Exgit.Pack.Reader.parse_at(pack_slice, 12) do - {:ok, {type, ^sha, content}} -> Exgit.Object.decode(type, content) + {:ok, {type, ^sha, content}} -> {:ok, type, content} _ -> find_in_packs(dir, rest, sha) end else @@ -446,6 +571,19 @@ defimpl Exgit.ObjectStore, for: Exgit.ObjectStore.Disk do ) end + def object_size(store, sha) do + Telemetry.span( + [:exgit, :object_store, :object_size], + %{store: :disk, sha: sha}, + fn -> + case Disk.object_size(store, sha) do + {:ok, _} = ok -> {:span, ok, %{hit?: true}} + other -> {:span, other, %{hit?: false}} + end + end + ) + end + def import_objects(store, raw_objects) do failures = Enum.reduce(raw_objects, [], fn {type, sha, content}, errs -> diff --git a/lib/exgit/object_store/memory.ex b/lib/exgit/object_store/memory.ex index 01fa19c..710b32e 100644 --- a/lib/exgit/object_store/memory.ex +++ b/lib/exgit/object_store/memory.ex @@ -3,10 +3,19 @@ defmodule Exgit.ObjectStore.Memory do # Stores objects as {type_atom, zlib_compressed_content} for memory efficiency. # Objects are decoded on demand when `get` is called. + # + # `sizes` is a parallel index of `sha => uncompressed_byte_size`, kept in + # lockstep with `objects` by every insert path (put/import/streaming close). + # It lets `object_size/2` answer the size of a blob in O(1) WITHOUT + # `:zlib.uncompress`-ing the whole object into the heap — the difference + # between a constant-memory size check and materializing a multi-GB blob. - defstruct objects: %{} + defstruct objects: %{}, sizes: %{} - @type t :: %__MODULE__{objects: %{binary() => {atom(), binary()}}} + @type t :: %__MODULE__{ + objects: %{binary() => {atom(), binary()}}, + sizes: %{binary() => non_neg_integer()} + } @spec new() :: t() def new, do: %__MODULE__{} @@ -24,26 +33,63 @@ defmodule Exgit.ObjectStore.Memory do end @spec put_object(t(), Exgit.Object.t()) :: {:ok, binary(), t()} - def put_object(%__MODULE__{objects: objects} = store, object) do + def put_object(%__MODULE__{objects: objects, sizes: sizes} = store, object) do sha = Exgit.Object.sha(object) type = Exgit.Object.type(object) content = Exgit.Object.encode(object) |> IO.iodata_to_binary() compressed = :zlib.compress(content) - {:ok, sha, %{store | objects: Map.put(objects, sha, {type, compressed})}} + + {:ok, sha, + %{ + store + | objects: Map.put(objects, sha, {type, compressed}), + sizes: Map.put(sizes, sha, byte_size(content)) + }} end @spec has_object?(t(), binary()) :: boolean() def has_object?(%__MODULE__{objects: objects}, sha), do: Map.has_key?(objects, sha) + @doc """ + Uncompressed byte size of the stored object, in O(1) — without + decompressing it. Returns `{:error, :not_found}` if absent. + """ + @spec object_size(t(), binary()) :: {:ok, non_neg_integer()} | {:error, :not_found} + def object_size(%__MODULE__{sizes: sizes}, sha) do + case Map.fetch(sizes, sha) do + {:ok, size} -> {:ok, size} + :error -> {:error, :not_found} + end + end + + @doc """ + Remove `sha` from the store, keeping the `objects` and `sizes` + indexes in lockstep. Returns `{:ok, freed_compressed_bytes, store}` + so callers tracking a byte budget (e.g. the Promisor's cache + accounting) know how much was reclaimed, or `{:error, :not_found}` + if the object is absent. + """ + @spec delete_object(t(), binary()) :: {:ok, non_neg_integer(), t()} | {:error, :not_found} + def delete_object(%__MODULE__{objects: objects, sizes: sizes} = store, sha) do + case Map.pop(objects, sha) do + {nil, _} -> + {:error, :not_found} + + {{_type, compressed}, new_objects} -> + {:ok, byte_size(compressed), + %{store | objects: new_objects, sizes: Map.delete(sizes, sha)}} + end + end + @spec import_objects(t(), [{atom(), binary(), binary()}]) :: {:ok, t()} - def import_objects(%__MODULE__{objects: objects} = store, raw_objects) do - new_objects = - Enum.reduce(raw_objects, objects, fn {type, sha, content}, acc -> + def import_objects(%__MODULE__{objects: objects, sizes: sizes} = store, raw_objects) do + {new_objects, new_sizes} = + Enum.reduce(raw_objects, {objects, sizes}, fn {type, sha, content}, {objs, szs} -> compressed = :zlib.compress(content) - Map.put(acc, sha, {type, compressed}) + {Map.put(objs, sha, {type, compressed}), Map.put(szs, sha, byte_size(content))} end) - {:ok, %{store | objects: new_objects}} + {:ok, %{store | objects: new_objects, sizes: new_sizes}} end end @@ -86,6 +132,19 @@ defimpl Exgit.ObjectStore, for: Exgit.ObjectStore.Memory do ) end + def object_size(store, sha) do + Telemetry.span( + [:exgit, :object_store, :object_size], + %{store: :memory, sha: sha}, + fn -> + case Memory.object_size(store, sha) do + {:ok, _} = ok -> {:span, ok, %{hit?: true}} + other -> {:span, other, %{hit?: false}} + end + end + ) + end + def import_objects(store, raw_objects), do: Memory.import_objects(store, raw_objects) @@ -113,27 +172,37 @@ defimpl Exgit.ObjectStore, for: Exgit.ObjectStore.Memory do store_type: type, sha_ctx: sha_ctx, deflate: z, - acc: [] + acc: [], + # Actual uncompressed bytes seen, summed across chunks. We use this + # rather than `expected_size` so a caller that over- or under-declares + # the size can't desync the `sizes` index from the stored content. + usize: 0 } {:ok, handle} end - def write_chunk(_store, %{deflate: z, sha_ctx: sha_ctx, acc: acc} = handle, chunk) + def write_chunk(_store, %{deflate: z, sha_ctx: sha_ctx, acc: acc, usize: usize} = handle, chunk) when is_binary(chunk) do sha_ctx = :crypto.hash_update(sha_ctx, chunk) compressed_chunks = :zlib.deflate(z, chunk) - {:ok, %{handle | sha_ctx: sha_ctx, acc: [acc | compressed_chunks]}} + + {:ok, + %{handle | sha_ctx: sha_ctx, acc: [acc | compressed_chunks], usize: usize + byte_size(chunk)}} end - def close_write(%Memory{objects: objects} = store, %{__type__: :memory_write} = handle) do + def close_write( + %Memory{objects: objects, sizes: sizes} = store, + %{__type__: :memory_write} = handle + ) do sha = :crypto.hash_final(handle.sha_ctx) final_chunks = :zlib.deflate(handle.deflate, <<>>, :finish) :zlib.deflateEnd(handle.deflate) :zlib.close(handle.deflate) compressed = IO.iodata_to_binary([handle.acc | final_chunks]) new_objects = Map.put(objects, sha, {handle.store_type, compressed}) - {:ok, sha, %{store | objects: new_objects}} + new_sizes = Map.put(sizes, sha, handle.usize) + {:ok, sha, %{store | objects: new_objects, sizes: new_sizes}} end def cancel_write(_store, %{__type__: :memory_write, deflate: z}) do diff --git a/lib/exgit/object_store/promisor.ex b/lib/exgit/object_store/promisor.ex index de3d3a4..3f63396 100644 --- a/lib/exgit/object_store/promisor.ex +++ b/lib/exgit/object_store/promisor.ex @@ -325,6 +325,22 @@ defmodule Exgit.ObjectStore.Promisor do ObjectStore.Memory.has_object?(cache, sha) end + @doc """ + Uncompressed byte size of `sha` IF it is already cached locally — + without triggering a fetch. Returns `{:error, :not_local}` when the + object has not been fetched yet, so a size check can never silently + pull a multi-GB blob over the network. + """ + @spec object_size(t(), binary()) :: + {:ok, non_neg_integer()} | {:error, :not_local} + def object_size(%__MODULE__{cache: cache}, sha) do + if ObjectStore.Memory.has_object?(cache, sha) do + ObjectStore.Memory.object_size(cache, sha) + else + {:error, :not_local} + end + end + @doc "Merge `raw_objects` into the cache." @spec import_objects(t(), [{atom(), binary(), binary()}]) :: {:ok, t()} def import_objects(%__MODULE__{cache: cache} = p, raw_objects) do @@ -525,23 +541,16 @@ defmodule Exgit.ObjectStore.Promisor do _ -> {_key, sha, q2} = :gb_trees.take_smallest(q) - # Drop the commit object from the Memory cache. Track the - # byte delta. We pattern-match `p.cache` into a - # `%ObjectStore.Memory{}` binding first so the subsequent - # struct-update is visible to Elixir 1.19's type checker - # (a struct update on a field-access expression is rejected - # under --warnings-as-errors because the type is dynamic() - # at that site). - %ObjectStore.Memory{objects: objs} = cache = p.cache - - {dropped_bytes, new_objs} = - case Map.pop(objs, sha) do - {nil, o} -> {0, o} - {{_type, compressed}, o} -> {byte_size(compressed), o} + # Drop the commit object from the Memory cache via its delete + # helper — it keeps the `objects` and `sizes` indexes in + # lockstep, so `object_size/2` can never report a stale size + # for an evicted object. + {dropped_bytes, new_cache} = + case ObjectStore.Memory.delete_object(p.cache, sha) do + {:ok, freed, cache} -> {freed, cache} + {:error, :not_found} -> {0, p.cache} end - new_cache = %ObjectStore.Memory{cache | objects: new_objs} - %{ p | cache: new_cache, @@ -645,6 +654,19 @@ defimpl Exgit.ObjectStore, for: Exgit.ObjectStore.Promisor do ) end + def object_size(store, sha) do + Telemetry.span( + [:exgit, :object_store, :object_size], + %{store: :promisor, sha: sha}, + fn -> + case Promisor.object_size(store, sha) do + {:ok, _} = ok -> {:span, ok, %{hit?: true}} + other -> {:span, other, %{hit?: false}} + end + end + ) + end + def import_objects(store, raw_objects), do: Promisor.import_objects(store, raw_objects) diff --git a/lib/exgit/pack/delta.ex b/lib/exgit/pack/delta.ex index b6aae76..57098b7 100644 --- a/lib/exgit/pack/delta.ex +++ b/lib/exgit/pack/delta.ex @@ -64,7 +64,7 @@ defmodule Exgit.Pack.Delta do {:error, :insert_exceeds_result_size} true -> - <> = rest + <> = rest apply_instructions(more, base, remaining - byte, [data | acc]) end end diff --git a/lib/exgit/pack/index.ex b/lib/exgit/pack/index.ex index dc3ecd3..edcf23b 100644 --- a/lib/exgit/pack/index.ex +++ b/lib/exgit/pack/index.ex @@ -53,8 +53,8 @@ defmodule Exgit.Pack.Index do crc_size = total_objects * 4 offset_size = total_objects * 4 - <> = rest + <> = rest {large_offset_table, rest} = extract_large_offsets(offsets, total_objects, rest) @@ -63,7 +63,7 @@ defmodule Exgit.Pack.Index do # Verify index checksum idx_body_size = byte_size(data) - 20 - <> = data + <> = data if :crypto.hash(:sha, idx_body) != claimed_checksum do {:error, :index_checksum_mismatch} @@ -118,8 +118,8 @@ defmodule Exgit.Pack.Index do crc_size = total * 4 offset_size = total * 4 - <> = rest + <> = rest # Large-offset table comes next in the file. We keep the whole tail # as a subbinary; `offset_at/2` only slices into it when a lookup @@ -237,7 +237,7 @@ defmodule Exgit.Pack.Index do large_count -> large_size = large_count * 8 - <> = rest + <> = rest {large, rest} end end diff --git a/lib/exgit/pkt_line.ex b/lib/exgit/pkt_line.ex index a0cea95..6fee2e3 100644 --- a/lib/exgit/pkt_line.ex +++ b/lib/exgit/pkt_line.ex @@ -12,17 +12,44 @@ defmodule Exgit.PktLine do * `0002` — response-end `encode/1` emits the framed bytes for a data payload; - `decode_stream/1` parses a concatenated pkt-line stream into a - list of `{:data, bin} | :flush | :delim | :response_end` tokens. - Decoders never raise on truncated input; see - `Exgit.PropertiesTest` for the roundtrip property. + `decode_stream/1` parses a concatenated pkt-line stream into + `{:data, bin} | :flush | :delim | :response_end` tokens. + Decoders never raise on malformed or truncated input — bad + framing surfaces as an `{:error, {:malformed_pkt_line, snippet}}` + value; see `Exgit.PropertiesTest` for the fuzz property. """ @type packet :: {:data, binary()} | :flush | :delim | :response_end + @type decode_error :: {:error, {:malformed_pkt_line, binary()}} + # Largest payload a single pkt-line can carry: git caps the framed + # length at 65520 bytes (LARGE_PACKET_MAX), minus the 4-byte header. + @max_payload 65_516 + + # How many bytes of the offending input to include in a + # `:malformed_pkt_line` error detail. + @error_snippet_bytes 40 + + @doc """ + Frame a data payload as a pkt-line. + + Payloads are capped at #{@max_payload} bytes — git's + `LARGE_PACKET_MAX` (65520) minus the 4-byte length header. Raises + `ArgumentError` for oversized payloads: the length header is 4 hex + digits, so silently encoding anything larger would corrupt the + wire stream. Callers sending bulk data (e.g. sideband frames) must + chunk it below the limit. + """ @spec encode(iodata()) :: iolist() def encode(data) do payload = IO.iodata_to_binary(data) + + if byte_size(payload) > @max_payload do + raise ArgumentError, + "pkt-line payload is #{byte_size(payload)} bytes; " <> + "max is #{@max_payload} (LARGE_PACKET_MAX minus the 4-byte header)" + end + len = byte_size(payload) + 4 [len |> Integer.to_string(16) |> String.pad_leading(4, "0"), payload] end @@ -36,9 +63,20 @@ defmodule Exgit.PktLine do @spec response_end() :: binary() def response_end, do: "0002" + @doc """ + Lazily decode a concatenated pkt-line stream. + + Yields `t:packet/0` tokens. On malformed or truncated input the + stream yields a final `{:error, {:malformed_pkt_line, snippet}}` + token — `snippet` is up to #{@error_snippet_bytes} bytes of the + offending input — and halts. Never raises on hostile bytes. + """ @spec decode_stream(binary()) :: Enumerable.t() def decode_stream(bytes) when is_binary(bytes) do Stream.unfold(bytes, fn + :halted -> + nil + <<>> -> nil @@ -51,25 +89,39 @@ defmodule Exgit.PktLine do <<"0002", rest::binary>> -> {:response_end, rest} - <> -> + <> = buf -> with {len, ""} <- Integer.parse(hex_len, 16), true <- len >= 4, payload_len = len - 4, - <> <- rest do - {{:data, payload}, rest} + <> <- rest do + {{:data, payload}, tail} else - _ -> - raise ArgumentError, - "malformed pkt-line at: #{inspect(binary_part(hex_len <> rest, 0, min(byte_size(hex_len <> rest), 40)))}" + _ -> {{:error, {:malformed_pkt_line, snippet(buf)}}, :halted} end truncated -> - raise ArgumentError, "truncated pkt-line header: #{inspect(truncated)}" + {{:error, {:malformed_pkt_line, snippet(truncated)}}, :halted} end) end - @spec decode_all(binary()) :: [packet()] + @doc """ + Eagerly decode a concatenated pkt-line stream. + + Returns the packet list, or `{:error, {:malformed_pkt_line, snippet}}` + when the input contains bad or truncated framing. A malformed stream + is rejected whole — packets decoded before the bad framing are + discarded. Never raises on hostile bytes. + """ + @spec decode_all(binary()) :: [packet()] | decode_error() def decode_all(bytes) when is_binary(bytes) do - decode_stream(bytes) |> Enum.to_list() + tokens = bytes |> decode_stream() |> Enum.to_list() + + case List.last(tokens) do + {:error, _} = error -> error + _ -> tokens + end end + + defp snippet(bytes), + do: binary_part(bytes, 0, min(byte_size(bytes), @error_snippet_bytes)) end diff --git a/lib/exgit/pkt_line/decoder.ex b/lib/exgit/pkt_line/decoder.ex index 270a042..956a4cc 100644 --- a/lib/exgit/pkt_line/decoder.ex +++ b/lib/exgit/pkt_line/decoder.ex @@ -36,11 +36,12 @@ defmodule Exgit.PktLine.Decoder do Feed a chunk of bytes into the decoder. Returns the updated decoder and any complete packets that became decodable from `buffer ++ chunk`. - Returns `{:error, reason}` on malformed framing (length-prefix that - is not valid hex or claims a length below the 4-byte header). + Returns `{:error, {:malformed_pkt_line, hex}}` on malformed framing + (a length-prefix that is not valid hex or claims a length below the + 4-byte header). """ @spec feed(t(), binary()) :: - {:ok, t(), [PktLine.packet()]} | {:error, term()} + {:ok, t(), [PktLine.packet()]} | PktLine.decode_error() def feed(%__MODULE__{buffer: buf}, chunk) when is_binary(chunk) do drain(<>, []) end @@ -68,7 +69,7 @@ defmodule Exgit.PktLine.Decoder do payload_len = len - 4 case rest do - <> -> + <> -> drain(tail, [{:data, payload} | acc]) # Payload not fully arrived — preserve the WHOLE pkt-line @@ -78,7 +79,7 @@ defmodule Exgit.PktLine.Decoder do end _ -> - {:error, {:malformed_length, hex}} + {:error, {:malformed_pkt_line, hex}} end end diff --git a/lib/exgit/repo_handle.ex b/lib/exgit/repo_handle.ex index 4d5cf10..e014538 100644 --- a/lib/exgit/repo_handle.ex +++ b/lib/exgit/repo_handle.ex @@ -40,7 +40,7 @@ defmodule Exgit.RepoHandle do reason — normal stop, crash, `Process.exit/2`, supervisor shutdown — the table is automatically destroyed by the BEAM. Callers holding a dead handle get `{:error, :dead_handle}` from - `get/1`. + `fetch/1`. Clients that want the handle to outlive a supervision tree are responsible for wiring it into the right supervisor themselves. @@ -53,17 +53,17 @@ defmodule Exgit.RepoHandle do {:ok, task} = Exgit.FS.prefetch_async(handle) # Meanwhile, foreground reads work against the current snapshot. - repo_snapshot = Exgit.RepoHandle.get(handle) + repo_snapshot = Exgit.RepoHandle.fetch!(handle) Exgit.FS.grep(repo_snapshot, "HEAD", "auth", max_count: 10) # Wait for prefetch to finish, then do a full-repo search. :ok = Exgit.FS.await_prefetch(task) - fresh_snapshot = Exgit.RepoHandle.get(handle) + fresh_snapshot = Exgit.RepoHandle.fetch!(handle) Exgit.FS.grep(fresh_snapshot, "HEAD", "auth") |> Enum.to_list() - ## Why ETS and not `Agent.get/1` + ## Why ETS and not `Agent.get/2` - `Agent.get/1` still sends a message to the agent process and + `Agent.get/2` still sends a message to the agent process and copies the state back. For a `Repository` with a large Promisor cache that copy would be 10-100 MB per read — unacceptable for a LiveView that reads on every keystroke. @@ -111,25 +111,8 @@ defmodule Exgit.RepoHandle do copy of the repo into this process's mailbox. Safe to call on every hot-loop iteration. - Raises `ArgumentError` if the handle is dead or if the table - doesn't exist. Callers that want to tolerate dead handles - should wrap in `try/rescue` or use `fetch/1`. - """ - @spec get(t()) :: Repository.t() - def get(handle) do - case fetch(handle) do - {:ok, repo} -> - repo - - {:error, reason} -> - raise ArgumentError, - "Exgit.RepoHandle.get/1: #{inspect(reason)} for handle #{inspect(handle)}" - end - end - - @doc """ - Non-raising variant of `get/1`. Returns `{:ok, repo}` on success - or `{:error, :dead_handle}` / `{:error, :no_table}`. + Returns `{:ok, repo}` on success or `{:error, :dead_handle}` / + `{:error, :no_table}`. See `fetch!/1` for a raising variant. """ @spec fetch(t()) :: {:ok, Repository.t()} | {:error, :dead_handle | :no_table} def fetch(handle) do @@ -147,6 +130,24 @@ defmodule Exgit.RepoHandle do ArgumentError -> {:error, :no_table} end + @doc """ + Same as `fetch/1`, but returns the repo directly and raises + `ArgumentError` if the handle is dead or if the table doesn't + exist. Callers that want to tolerate dead handles should use + `fetch/1`. + """ + @spec fetch!(t()) :: Repository.t() + def fetch!(handle) do + case fetch(handle) do + {:ok, repo} -> + repo + + {:error, reason} -> + raise ArgumentError, + "Exgit.RepoHandle.fetch!/1: #{inspect(reason)} for handle #{inspect(handle)}" + end + end + @doc """ Apply `fun` to the current repo value and store the result. @@ -192,7 +193,7 @@ defmodule Exgit.RepoHandle do Without dedup, each caller fires its own identical network call — wasteful. - With `fetch_once/3`: + With `fetch_once/4`: * First caller for `key` runs `fetch_fn(current_repo)` OUTSIDE the handle (in a linked Task) so the handle stays responsive @@ -218,9 +219,16 @@ defmodule Exgit.RepoHandle do ## Errors `{:error, :dead_handle}` if the handle isn't running. Propagates - `fetch_fn`'s errors verbatim. + `fetch_fn`'s errors verbatim. If `fetch_fn` raises, throws, or + exits — or the fetch task is killed — all waiters receive + `{:error, {:fetch_crashed, reason}}`. """ - @spec fetch_once(t(), term(), (Repository.t() -> {:ok, Repository.t()} | {:error, term()})) :: + @spec fetch_once( + t(), + term(), + (Repository.t() -> {:ok, Repository.t()} | {:error, term()}), + timeout() + ) :: {:ok, Repository.t()} | {:error, term()} def fetch_once(handle, key, fetch_fn, timeout \\ 300_000) when is_function(fetch_fn, 1) do GenServer.call(handle, {:fetch_once, key, fetch_fn}, timeout) @@ -261,10 +269,11 @@ defmodule Exgit.RepoHandle do true = :ets.insert(table, {:repo, repo}) - # `in_flight`: %{key => %{task_ref: ref, waiters: [from, ...]}} + # `in_flight`: %{key => %{monitor_ref: ref, waiters: [from, ...]}} # Entries are created when the first fetch_once for a key - # arrives; removed when the task completes and all waiters - # have been replied to. + # arrives; removed when the task completes (or dies — the + # monitor guarantees waiters are always replied to) and all + # waiters have been replied to. {:ok, %{table: table, in_flight: %{}}} end @@ -302,16 +311,27 @@ defmodule Exgit.RepoHandle do # updates. We'll reply to `from` (and any waiters that # register before the task finishes) when it completes. handle_pid = self() + table = state.table - _ = + start_result = Task.Supervisor.start_child(Exgit.TaskSupervisor, fn -> - [{:repo, repo}] = :ets.lookup(state.table, :repo) + [{:repo, repo}] = :ets.lookup(table, :repo) result = safe_fetch(fetch_fn, repo) send(handle_pid, {:fetch_once_done, key, result}) end) - entry = %{waiters: [from]} - {:noreply, %{state | in_flight: Map.put(state.in_flight, key, entry)}} + case start_result do + {:ok, pid} -> + # Monitor the task so waiters get a reply even if it + # dies without sending :fetch_once_done (e.g. a brutal + # kill that safe_fetch can't catch). + monitor_ref = Process.monitor(pid) + entry = %{monitor_ref: monitor_ref, waiters: [from]} + {:noreply, %{state | in_flight: Map.put(state.in_flight, key, entry)}} + + other -> + {:reply, {:error, {:fetch_task_start_failed, other}}, state} + end %{waiters: waiters} = entry -> # Already in-flight for this key. Add to waiters. @@ -328,7 +348,11 @@ defmodule Exgit.RepoHandle do # ourselves somehow. Log and continue. {:noreply, state} - {%{waiters: waiters}, remaining} -> + {%{monitor_ref: monitor_ref, waiters: waiters}, remaining} -> + # The task sent its result before exiting, so the pending + # :DOWN (if any) is stale — flush it. + Process.demonitor(monitor_ref, [:flush]) + # Commit the result to ETS if it succeeded, then reply to # all waiters with the same return value. reply_value = @@ -349,10 +373,29 @@ defmodule Exgit.RepoHandle do end end + def handle_info({:DOWN, ref, :process, _pid, reason}, state) do + # A fetch task died without sending :fetch_once_done (a brutal + # kill — anything catchable is turned into an error tuple by + # safe_fetch). Reply to all waiters so nobody hangs until the + # call timeout, and clear the entry so the next fetch_once for + # this key re-runs the fetch. + case Enum.find(state.in_flight, fn {_key, entry} -> entry.monitor_ref == ref end) do + nil -> + {:noreply, state} + + {key, %{waiters: waiters}} -> + for from <- Enum.reverse(waiters) do + GenServer.reply(from, {:error, {:fetch_crashed, reason}}) + end + + {:noreply, %{state | in_flight: Map.delete(state.in_flight, key)}} + end + end + defp safe_fetch(fun, repo) do fun.(repo) - rescue - e -> {:error, {:fetch_fn_raised, e, __STACKTRACE__}} + catch + kind, reason -> {:error, {:fetch_crashed, {kind, reason, __STACKTRACE__}}} end @impl true diff --git a/lib/exgit/repo_registry.ex b/lib/exgit/repo_registry.ex index b8215e7..3f06b50 100644 --- a/lib/exgit/repo_registry.ex +++ b/lib/exgit/repo_registry.ex @@ -296,7 +296,13 @@ defmodule Exgit.RepoRegistry do Registry.unregister(@registry_name, url) end - {:noreply, %{state | urls: MapSet.difference(state.urls, MapSet.new(dead_urls))}} + # Fold with `MapSet.delete/2` rather than `MapSet.difference/2`. `state` + # is an untyped GenServer map, so `state.urls` reads as `any()`, which + # trips a MapSet opacity warning through `difference` (but not through + # `delete`, as the `MapSet.delete` calls elsewhere in this module show). + # The dead set is tiny (an exit-path cleanup), so the fold is cheap. + urls = Enum.reduce(dead_urls, state.urls, fn url, acc -> MapSet.delete(acc, url) end) + {:noreply, %{state | urls: urls}} end ## Internal diff --git a/lib/exgit/repository.ex b/lib/exgit/repository.ex index d8a312f..9e435d4 100644 --- a/lib/exgit/repository.ex +++ b/lib/exgit/repository.ex @@ -88,14 +88,17 @@ defmodule Exgit.Repository do * `:max_cache_bytes` — configured cap (`:infinity` if unbounded) * `:mode` — repo mode (`:eager` or `:lazy`) * `:backend` — the object-store module + + Counts and `:cache_bytes` are `:unknown` for backends that can't + be introspected (`Disk`, user-defined stores). """ @type memory_report :: %{ - object_count: non_neg_integer(), - cache_bytes: non_neg_integer(), - commit_count: non_neg_integer(), - tree_count: non_neg_integer(), - blob_count: non_neg_integer(), - tag_count: non_neg_integer(), + object_count: non_neg_integer() | :unknown, + cache_bytes: non_neg_integer() | :unknown, + commit_count: non_neg_integer() | :unknown, + tree_count: non_neg_integer() | :unknown, + blob_count: non_neg_integer() | :unknown, + tag_count: non_neg_integer() | :unknown, max_cache_bytes: non_neg_integer() | :infinity, mode: mode(), backend: module() @@ -109,10 +112,9 @@ defmodule Exgit.Repository do cache growth, and alert when a configured cap is approached. Returns consistent shape across all object-store backends - (`Memory`, `Disk`, `Promisor`, `SharedPromisor`); counts for - backends without per-type bookkeeping (like `Disk`) are - `:unknown`. The `:cache_bytes` and `:max_cache_bytes` fields - are always present. + (`Memory`, `Disk`, `Promisor`, `SharedPromisor`); counts and + `:cache_bytes` for backends without per-type bookkeeping (like + `Disk`) are `:unknown`. All keys are always present. ## Examples @@ -189,18 +191,18 @@ defmodule Exgit.Repository do defp store_report(other) do # Backends we don't introspect deeply (ObjectStore.Disk, - # user-defined stores). Report a degraded shape with - # placeholders so callers can still depend on the keys - # existing. + # user-defined stores). Report `:unknown` rather than a fake 0 + # so monitoring callers can't mistake "not introspected" for + # "empty"; the keys still always exist. _ = other %{ - object_count: 0, - cache_bytes: 0, - commit_count: 0, - tree_count: 0, - blob_count: 0, - tag_count: 0, + object_count: :unknown, + cache_bytes: :unknown, + commit_count: :unknown, + tree_count: :unknown, + blob_count: :unknown, + tag_count: :unknown, max_cache_bytes: :infinity } end diff --git a/lib/exgit/telemetry.ex b/lib/exgit/telemetry.ex index cc044e5..5d557d3 100644 --- a/lib/exgit/telemetry.ex +++ b/lib/exgit/telemetry.ex @@ -43,6 +43,10 @@ defmodule Exgit.Telemetry do - `hit?: false` when it had to go to the network - omitted for stores that don't track this distinction * `[:exgit, :object_store, :put]` — `%{store, sha}` + * `[:exgit, :object_store, :object_size]` — `%{store, sha, hit?}` + - `hit?: false` covers both `{:error, :not_found}` and + `{:error, :not_local}` (a `Promisor` never fetches for a + size probe) * `[:exgit, :object_store, :has?]` — `%{store, sha, present?}` * `[:exgit, :object_store, :fetch_and_cache]` — `%{sha, object_count, cache_bytes}` (stop event only) diff --git a/lib/exgit/transport/http.ex b/lib/exgit/transport/http.ex index 79cd7af..5c726bb 100644 --- a/lib/exgit/transport/http.ex +++ b/lib/exgit/transport/http.ex @@ -33,6 +33,12 @@ defmodule Exgit.Transport.HTTP do alias Exgit.PktLine alias Exgit.PktLine.Decoder + # Default backstop cap on refs accepted from a single ls-refs response. + # Real repos top out in the tens of thousands (linux, esp-idf); this + # bound exists only to stop a hostile server from streaming unbounded + # refs into client memory. Tunable per-transport via `:max_refs`. + @max_refs 1_000_000 + @enforce_keys [:url] defstruct [ :url, @@ -81,6 +87,9 @@ defmodule Exgit.Transport.HTTP do # git hosts do redirect (canonicalization, repo renames) — set # `:same_origin` for hosts where this is common. redirect: false, + # Cap on refs accepted from a single ls-refs response before the + # transport aborts with {:error, {:too_many_refs, cap}}. + max_refs: @max_refs, # Cached server capabilities (protocol v2 advertisements). `nil` # means "not yet discovered"; an `{:ok, caps}` / `{:error, _}` # tuple means the result is memoized. A fresh struct has `nil` @@ -100,6 +109,30 @@ defmodule Exgit.Transport.HTTP do @type t :: %__MODULE__{url: String.t(), auth: auth(), user_agent: String.t()} + @doc """ + Build a transport for the git smart-HTTP remote at `url`. + + ## Options + + * `:auth` — credentials; a bare auth tuple (auto-wrapped in a + host-bound `%Exgit.Credentials{}`) or an explicit + `%Exgit.Credentials{}` struct. See the module doc. + * `:user_agent` — the `user-agent` header value. + * `:connect_timeout` — connect timeout in milliseconds + (default 10_000). + * `:receive_timeout` — response-body timeout in milliseconds + (default 300_000); `:infinity` disables. + * `:verify_tls` — TLS peer verification (default `true`). + * `:connect_options` — extra transport options merged into the + library's TLS defaults (custom CA bundles, mTLS, SNI). + * `:redirect` — redirect policy: `false` (default), + `:same_origin`, or `:follow`. + * `:max_refs` — cap on refs accepted from a single ls-refs + response (default #{@max_refs}). `ls_refs/2` aborts with + `{:error, {:too_many_refs, cap}}` when a server exceeds it — + a backstop against a hostile server streaming unbounded refs + into client memory. + """ @spec new(String.t(), keyword()) :: t() def new(url, opts \\ []) do trimmed_url = String.trim_trailing(url, "/") @@ -112,7 +145,8 @@ defmodule Exgit.Transport.HTTP do receive_timeout: Keyword.get(opts, :receive_timeout, defaults.receive_timeout), verify_tls: Keyword.get(opts, :verify_tls, defaults.verify_tls), connect_options: Keyword.get(opts, :connect_options, defaults.connect_options), - redirect: Keyword.get(opts, :redirect, defaults.redirect) + redirect: Keyword.get(opts, :redirect, defaults.redirect), + max_refs: Keyword.get(opts, :max_refs, defaults.max_refs) ) end @@ -141,6 +175,20 @@ defmodule Exgit.Transport.HTTP do end end + @doc """ + Discover the server's protocol-v2 capability advertisement. + + Performs the `info/refs?service=git-upload-pack` discovery GET and + parses the advertisement into a map keyed by capability name + (string keys; well-known entries like `:version` get structured + values). Returns `{:error, :server_does_not_support_v2}` for + protocol-v1-only servers, `{:error, {:malformed_response, reason}}` + when the body is not valid pkt-line framing. + + A struct carrying a memoized `capabilities_cache` (see + `capabilities_cached/1`) returns the cached result without HTTP. + """ + @spec capabilities(t()) :: {:ok, map()} | {:error, term()} def capabilities(%__MODULE__{capabilities_cache: {:ok, _} = cached}), do: cached def capabilities(%__MODULE__{capabilities_cache: {:error, _} = cached}), do: cached @@ -148,10 +196,7 @@ defmodule Exgit.Transport.HTTP do # Not cached — discover, but don't try to mutate the struct (pure # value). Callers who want memoization across many requests can # use `capabilities_cached/1` which returns an updated struct. - case discover(t, "git-upload-pack") do - {:ok, caps} -> {:ok, caps} - error -> error - end + discover(t, "git-upload-pack") end @doc """ @@ -171,10 +216,33 @@ defmodule Exgit.Transport.HTTP do def capabilities_cached(%__MODULE__{capabilities_cache: cached} = t), do: {cached, t} + @doc """ + List the remote's refs via the protocol-v2 `ls-refs` command. + + Returns `{:ok, refs, meta}` where `refs` is a list of + `{ref_name, sha}` 2-tuples and `meta` carries protocol-v2 + side-channel data (`:head` symref target, `:peeled` tag targets) — + see `t:Exgit.Transport.ls_refs_meta/0`. Invalid ref names from the + server are dropped (with `[:exgit, :security, :ref_rejected]` + telemetry); more than `max_refs` refs aborts with + `{:error, {:too_many_refs, cap}}`. + + ## Options + + * `:prefix` — ref-prefix filter or list of filters + (e.g. `"refs/heads/"`); default `[]` (all refs). + * `:symrefs` — ask for symref targets, revealing where HEAD + points (default `true`). + * `:peeled` — ask for `peeled:` attributes on annotated + tags (default `true`). + """ + @spec ls_refs(t(), keyword()) :: + {:ok, [Exgit.Transport.ref_entry()], Exgit.Transport.ls_refs_meta()} + | {:error, term()} def ls_refs(%__MODULE__{} = t, opts \\ []) do Exgit.Telemetry.span( [:exgit, :transport, :ls_refs], - %{transport: :http, url: t.url}, + %{transport: :http, url: redact_url(t.url)}, fn -> case do_ls_refs(t, opts) do {:ok, refs, meta} = result -> @@ -213,15 +281,23 @@ defmodule Exgit.Transport.HTTP do # without ever materializing the full response body or the list of # decoded packets. For repos with tens of thousands of refs (esp-idf, # linux), this keeps the transport's memory bound flat in ref count. - init_acc = {[], %{peeled: %{}}} + # Map-shaped accumulator (not a tuple) so the streaming loop's + # `%{error: e}` halt check fires the moment the ref cap trips. + init_acc = %{refs: [], meta: %{peeled: %{}}, count: 0, error: nil, max_refs: t.max_refs} + # Redact before it reaches `keep_ref?`'s security telemetry, which + # echoes the source URL. + source_url = redact_url(t.url) handle_packet = fn - {:data, line}, acc -> parse_ls_refs_line(line, t.url, acc) + {:data, line}, acc -> parse_ls_refs_line(line, source_url, acc) _, acc -> acc end case stream_upload_pack(t, body, init_acc, handle_packet) do - {:ok, {refs_rev, meta}} -> + {:ok, %{error: reason}} when not is_nil(reason) -> + {:error, reason} + + {:ok, %{refs: refs_rev, meta: meta}} -> meta = if map_size(meta.peeled) == 0, do: Map.delete(meta, :peeled), else: meta @@ -244,29 +320,51 @@ defmodule Exgit.Transport.HTTP do # Hostile ref names are rejected here via `Exgit.RefName.valid?/1`; # rejections emit `[:exgit, :security, :ref_rejected]` telemetry # and drop the entry entirely. - defp parse_ls_refs_line(line, source_url, {refs, meta}) do + # Once the ref cap trips, short-circuit: the stream loop halts on + # `acc.error`, but packets already decoded from the current chunk still + # flow through this fold, so ignore them. + defp parse_ls_refs_line(_line, _source_url, %{error: reason} = acc) when not is_nil(reason), + do: acc + + defp parse_ls_refs_line(line, source_url, acc) do line = String.trim_trailing(line, "\n") case String.split(line, " ", parts: 3) do [hex_sha, ref, attrs] when byte_size(hex_sha) == 40 -> with {:ok, sha} <- Base.decode16(hex_sha, case: :mixed), true <- keep_ref?(ref, source_url) do - attrs_map = parse_ls_refs_attrs(attrs) - add_ref(refs, meta, ref, sha, attrs_map) + add_ref(acc, ref, sha, parse_ls_refs_attrs(attrs)) else - _ -> {refs, meta} + _ -> acc end [hex_sha, ref] when byte_size(hex_sha) == 40 -> with {:ok, sha} <- Base.decode16(hex_sha, case: :mixed), true <- keep_ref?(ref, source_url) do - add_ref(refs, meta, ref, sha, %{}) + add_ref(acc, ref, sha, %{}) else - _ -> {refs, meta} + _ -> acc end _ -> - {refs, meta} + acc + end + end + + # Strip credentials from a URL before it enters telemetry metadata or a + # security event. exgit's own API carries auth in the `:auth` field, but + # callers commonly embed a token in the URL (`https://token@host/…`) the + # way git does — and that must never reach a telemetry exporter, log + # aggregator, or OTel span. + # The `is_binary/1` guard is load-bearing for Dialyzer: `URI.parse/1` + # accepts `URI.t() | binary()`, so without it Dialyzer widens `url`'s + # inferred type to include `%URI{}`, and the `userinfo == nil` branch + # (which returns `url` unchanged) then appears to return a struct. + @spec redact_url(String.t()) :: String.t() + defp redact_url(url) when is_binary(url) do + case URI.parse(url).userinfo do + nil -> url + userinfo -> String.replace(url, userinfo <> "@", "***@", global: false) end end @@ -284,10 +382,12 @@ defmodule Exgit.Transport.HTTP do end end - # Add an entry to `refs` and lift any attributes into `meta`. - # HEAD's `symref-target` becomes `meta.head`; annotated tags' - # `peeled` target becomes `meta.peeled[tag_name]`. - defp add_ref(refs, meta, ref, sha, attrs) do + # Add an entry to the accumulator and lift any attributes into `meta`. + # HEAD's `symref-target` becomes `meta.head`; annotated tags' `peeled` + # target becomes `meta.peeled[tag_name]`. Past `acc.max_refs` we stop + # appending and set `acc.error` so the transport stream loop halts — + # a hostile server can't grow the ref list without bound. + defp add_ref(%{refs: refs, meta: meta, count: count, max_refs: max_refs} = acc, ref, sha, attrs) do meta = case Map.get(attrs, :symref_target) do nil -> meta @@ -304,7 +404,11 @@ defmodule Exgit.Transport.HTTP do peeled_sha -> put_in(meta, [:peeled, ref], peeled_sha) end - {[{ref, sha} | refs], meta} + if count + 1 > max_refs do + %{acc | meta: meta, error: {:too_many_refs, max_refs}} + else + %{acc | refs: [{ref, sha} | refs], meta: meta, count: count + 1} + end end defp parse_ls_refs_attrs(attrs) do @@ -327,10 +431,33 @@ defmodule Exgit.Transport.HTTP do end) end + @doc """ + Fetch a pack for the given `wants` (20-byte SHAs) via the + protocol-v2 `fetch` command. + + Returns `{:ok, pack_bytes, summary}`. With `:object_store` the + pack is stream-parsed straight into the store — `pack_bytes` is + `<<>>` and `summary` carries `:objects` and the updated `:store`. + Without it, `pack_bytes` is the raw pack for the caller to parse. + Sideband channel-3 server messages surface as + `{:error, {:server_error, msg}}`. + + ## Options + + * `:haves` — SHAs the client already has, for negotiation. + * `:depth` — shallow-clone depth (`deepen`). + * `:filter` — partial-clone filter spec (e.g. `"blob:none"`). + * `:object_store` — an `Exgit.ObjectStore` to stream objects + into as they arrive (bounded memory on multi-GB packs). + * `:sideband`, `:thin_pack`, `:ofs_delta` — explicit feature + overrides; by default they're negotiated from the server's + advertised capabilities. + """ + @spec fetch(t(), [binary()], keyword()) :: {:ok, binary(), map()} | {:error, term()} def fetch(%__MODULE__{} = t, wants, opts \\ []) do Exgit.Telemetry.span( [:exgit, :transport, :fetch], - %{transport: :http, url: t.url, wants_count: length(wants)}, + %{transport: :http, url: redact_url(t.url), wants_count: length(wants)}, fn -> case do_fetch(t, wants, opts) do {:ok, pack_bytes, summary} = result -> @@ -432,9 +559,15 @@ defmodule Exgit.Transport.HTTP do init = init_fetch_state(Keyword.get(opts, :sideband), object_store) case stream_upload_pack(t, body, init, &handle_fetch_packet/2) do + # Binary errors are sideband channel-3 server messages. {:ok, %{error: msg}} when is_binary(msg) -> {:error, {:server_error, msg}} + # Anything else non-nil is a structured client-side reason + # (e.g. a StreamParser rejection) — pass it through as-is. + {:ok, %{error: reason}} when not is_nil(reason) -> + {:error, reason} + # Streaming path: finalise the parser and return the updated store. {:ok, %{parser: %StreamParser{} = parser}} -> finalize_stream_parse(parser) @@ -576,12 +709,25 @@ defmodule Exgit.Transport.HTTP do end end + @doc """ + Push ref `updates` and the accompanying `pack_bytes` via + `git-receive-pack`. + + Each update is a `{ref, old_sha, new_sha}` 3-tuple (20-byte SHAs; + `nil` means the all-zero SHA, i.e. create/delete). Returns + `{:ok, %{ref_results: [{ref, :ok | :error}]}}` parsed from the + server's report-status response, or + `{:error, {:malformed_response, reason}}` when the report is not + valid pkt-line framing. + """ + @spec push(t(), [Exgit.Transport.ref_update()], binary(), keyword()) :: + {:ok, %{ref_results: [{String.t(), :ok | :error}]}} | {:error, term()} def push(%__MODULE__{} = t, updates, pack_bytes, opts \\ []) do Exgit.Telemetry.span( [:exgit, :transport, :push], %{ transport: :http, - url: t.url, + url: redact_url(t.url), update_count: length(updates), pack_bytes: byte_size(pack_bytes) }, @@ -610,7 +756,7 @@ defmodule Exgit.Transport.HTTP do case do_request(:post, url, headers, body, t) do {:ok, %{status: status, body: resp_body}} when status in 200..299 -> - {:ok, parse_push_report(resp_body)} + parse_push_report(resp_body) {:ok, %{status: status, body: resp_body}} -> {:error, {:http_error, status, resp_body}} @@ -646,8 +792,13 @@ defmodule Exgit.Transport.HTTP do # untrusted server input. Well-known capabilities additionally expose # structured values (e.g. :version -> integer). defp parse_capabilities(body) do - packets = PktLine.decode_all(body) + case PktLine.decode_all(body) do + {:error, reason} -> {:error, {:malformed_response, reason}} + packets -> parse_capability_packets(packets) + end + end + defp parse_capability_packets(packets) do caps = packets |> Enum.flat_map(fn @@ -681,18 +832,21 @@ defmodule Exgit.Transport.HTTP do # --- Push response parsing --- defp parse_push_report(body) do - packets = PktLine.decode_all(body) - - ref_results = - packets - |> Enum.flat_map(fn - {:data, "unpack ok\n"} -> [] - {:data, <<"ok ", ref::binary>>} -> [{String.trim(ref), :ok}] - {:data, <<"ng ", rest::binary>>} -> [{String.trim(rest), :error}] - _ -> [] - end) - - %{ref_results: ref_results} + case PktLine.decode_all(body) do + {:error, reason} -> + {:error, {:malformed_response, reason}} + + packets -> + ref_results = + Enum.flat_map(packets, fn + {:data, "unpack ok\n"} -> [] + {:data, <<"ok ", ref::binary>>} -> [{String.trim(ref), :ok}] + {:data, <<"ng ", rest::binary>>} -> [{String.trim(rest), :error}] + _ -> [] + end) + + {:ok, %{ref_results: ref_results}} + end end # --- HTTP helpers --- @@ -754,9 +908,7 @@ defmodule Exgit.Transport.HTTP do case Req.request(req_opts) do {:ok, %{status: status, private: %{exgit_stream: state}}} when status in 200..299 -> - with :ok <- Decoder.finalize(state.decoder) do - if state.error, do: {:error, state.error}, else: {:ok, state.handler_acc} - end + finalize_stream_state(state) {:ok, %{status: status, private: %{exgit_stream: state}}} -> {:error, {:http_error, status, state.error_body}} @@ -769,6 +921,35 @@ defmodule Exgit.Transport.HTTP do end end + # Resolve a finished 2xx stream into the caller's result. An error + # accumulated mid-stream — a decode failure on `state.error`, or a + # handler/domain error on the accumulator (sideband channel-3 + # message, the ls-refs cap, a StreamParser rejection) — halts the + # body early, which legitimately leaves partial bytes in the + # decoder. So the accumulated error must win over the finalize + # truncation artifact, or the real reason gets masked by + # `{:truncated, _}`. + defp finalize_stream_state(state) do + cond do + state.error != nil -> + {:error, state.error} + + acc_error(state.handler_acc) != nil -> + {:ok, state.handler_acc} + + true -> + case Decoder.finalize(state.decoder) do + :ok -> {:ok, state.handler_acc} + {:error, _} = err -> err + end + end + end + + # Handler accumulators are caller-shaped; both ls-refs and fetch use + # a map with an `:error` slot. Anything else has no domain error. + defp acc_error(%{error: e}), do: e + defp acc_error(_), do: nil + # One streaming step: feed `chunk` through either the error buffer # (non-2xx) or the pkt-line decoder + handler (2xx), and stash the # updated state in `resp.private.exgit_stream`. @@ -780,7 +961,7 @@ defmodule Exgit.Transport.HTTP do captured = if room > 0 do take = min(room, byte_size(chunk)) - <> = chunk + <> = chunk <> else state.error_body @@ -969,8 +1150,6 @@ defmodule Exgit.Transport.HTTP do end defimpl Inspect, for: Exgit.Transport.HTTP do - import Inspect.Algebra - # Credentials live on %HTTP{}. Default Inspect would dump them into any # crash log, SASL report, or IEx session. Always redact. def inspect(%Exgit.Transport.HTTP{} = t, opts) do @@ -984,10 +1163,6 @@ defimpl Inspect, for: Exgit.Transport.HTTP do defp redact({:header, name, _}), do: {:header, name, "***"} defp redact({:callback, _}), do: {:callback, :fun} defp redact(_), do: :redacted - - # Suppress unused-import warning from concat/2 et al — we keep the - # import in case future formatting needs it. - _ = &concat/2 end defimpl Exgit.Transport, for: Exgit.Transport.HTTP do diff --git a/lib/exgit/workspace.ex b/lib/exgit/workspace.ex index 74bfb3c..f5ed179 100644 --- a/lib/exgit/workspace.ex +++ b/lib/exgit/workspace.ex @@ -144,7 +144,7 @@ defmodule Exgit.Workspace do end @doc """ - Stat the entry at `path`. Returns `%{type: :blob | :tree, mode:, size:}`. + Stat the entry at `path`. Returns `%{type: :blob | :tree | :submodule, mode:, size:}`. """ @spec stat(t(), String.t()) :: {:ok, FS.stat(), t()} | {:error, term()} def stat(%__MODULE__{} = ws, path) do diff --git a/lib/exgit/workspace/vfs.ex b/lib/exgit/workspace/vfs.ex index 122acc5..9986168 100644 --- a/lib/exgit/workspace/vfs.ex +++ b/lib/exgit/workspace/vfs.ex @@ -64,6 +64,11 @@ if Code.ensure_loaded?(VFS.Mountable) do {:ok, %{type: :tree}, ws} -> {:ok, %Stat{type: :directory, size: 0, mtime: @epoch}, ws} + # A submodule materializes as an (empty) directory, matching + # how `git checkout` presents gitlink entries. + {:ok, %{type: :submodule}, ws} -> + {:ok, %Stat{type: :directory, size: 0, mtime: @epoch}, ws} + {:error, reason} -> {:error, error_for(reason, path)} end @@ -279,7 +284,7 @@ if Code.ensure_loaded?(VFS.Mountable) do {rest, ""} rest -> - <> = rest + <> = rest {chunk, more} end) end diff --git a/mix.exs b/mix.exs index 3347296..5a23d8c 100644 --- a/mix.exs +++ b/mix.exs @@ -9,10 +9,10 @@ defmodule Exgit.MixProject do app: :exgit, version: @version, elixir: "~> 1.17", - # The library doesn't use any 1.19-only features; pinning the + # The library doesn't use any post-1.17 features; pinning the # lower bound at 1.17 keeps it installable for older consumers. - # CI matrix tests both 1.17 (minimum-supported) and 1.19 - # (primary + stricter type checks). + # CI matrix tests 1.20/OTP 29 (primary, all quality gates), + # 1.19/OTP 28, and 1.17/OTP 27 (minimum-supported). start_permanent: Mix.env() == :prod, deps: deps(), elixirc_paths: elixirc_paths(Mix.env()), @@ -53,14 +53,15 @@ defmodule Exgit.MixProject do links: %{ "GitHub" => @source_url }, - files: ~w(lib .formatter.exs mix.exs README.md LICENSE CHANGELOG.md SECURITY.md) + files: + ~w(lib docs/PERFORMANCE.md .formatter.exs mix.exs README.md LICENSE CHANGELOG.md SECURITY.md) ] end defp docs do [ main: "readme", - extras: ["README.md", "CHANGELOG.md", "SECURITY.md"], + extras: ["README.md", "CHANGELOG.md", "SECURITY.md", "docs/PERFORMANCE.md"], source_url: @source_url, source_ref: "v#{@version}" ] @@ -91,8 +92,7 @@ defmodule Exgit.MixProject do # consumers can attach to. Zero cost when no handler is attached. {:telemetry, "~> 1.0"}, # Optional vfs integration: `Exgit.Workspace` ships a - # `VFS.Mountable` defimpl when `:vfs` is loaded. Pinned to a SHA - # because vfs has no hex release yet. + # `VFS.Mountable` defimpl when `:vfs` is loaded. # # `optional: true` means downstream consumers don't have to install # :vfs to use exgit; if they DO add :vfs, Mix orders our build after @@ -118,11 +118,7 @@ defmodule Exgit.MixProject do # remove vfs from our dep graph in :prod, breaking compile-ordering # guarantees in downstream consumer builds. Requires Elixir ~> 1.18; # the 1.17 CI tier skips the integration via `Code.ensure_loaded?`. - {:vfs, - github: "ivarvong/vfs", - ref: "32d2ab618ec12c16fe4f675b5ee8b563c660dd69", - optional: true, - runtime: false}, + {:vfs, "~> 0.1.0", optional: true, runtime: false}, {:stream_data, "~> 1.0", only: [:test, :dev]}, # Test-only: localhost HTTP server for stubbing the Cloudflare # Artifacts REST API in `test/exgit/cloudflare_artifacts_test.exs`. @@ -136,7 +132,9 @@ defmodule Exgit.MixProject do {:opentelemetry_exporter, "~> 1.8", only: [:dev, :test]}, # Dev-only static analysis. Not runtime deps. {:dialyxir, "~> 1.4", only: [:dev], runtime: false}, - {:credo, "~> 1.7", only: [:dev], runtime: false} + {:credo, "~> 1.7", only: [:dev], runtime: false}, + # Doc generation for HexDocs (`mix docs` / `mix hex.publish`). + {:ex_doc, "~> 0.34", only: [:dev], runtime: false} ] end diff --git a/mix.lock b/mix.lock index 14f7f5c..4cd620a 100644 --- a/mix.lock +++ b/mix.lock @@ -3,36 +3,42 @@ "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, "bypass": {:hex, :bypass, "2.1.0", "909782781bf8e20ee86a9cabde36b259d44af8b9f38756173e8f5e2e1fabb9b1", [:mix], [{:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:ranch, "~> 1.3", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "d9b5df8fa5b7a6efa08384e9bbecfe4ce61c77d28a4282f79e02f1ef78d96b80"}, "chatterbox": {:hex, :ts_chatterbox, "0.15.1", "5cac4d15dd7ad61fc3c4415ce4826fc563d4643dee897a558ec4ea0b1c835c9c", [:rebar3], [{:hpack, "~> 0.3.0", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "4f75b91451338bc0da5f52f3480fa6ef6e3a2aeecfc33686d6b3d0a0948f31aa"}, - "cowboy": {:hex, :cowboy, "2.14.2", "4008be1df6ade45e4f2a4e9e2d22b36d0b5aba4e20b0a0d7049e28d124e34847", [:make, :rebar3], [{:cowlib, ">= 2.16.0 and < 3.0.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, ">= 1.8.0 and < 3.0.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "569081da046e7b41b5df36aa359be71a0c8874e5b9cff6f747073fc57baf1ab9"}, + "cowboy": {:hex, :cowboy, "2.16.1", "fa04080b602ff25c40a7700f2dc0152dbc1ba26b42093ae0fa9bb7a337d5a242", [:make, :rebar3], [{:cowlib, ">= 2.16.0 and < 3.0.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, ">= 1.8.0 and < 3.0.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "b8ea4dd317a043e3177ec840cfa3bcb47cfb41035d3abb24d954dc7d51def399"}, "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, - "cowlib": {:hex, :cowlib, "2.16.0", "54592074ebbbb92ee4746c8a8846e5605052f29309d3a873468d76cdf932076f", [:make, :rebar3], [], "hexpm", "7f478d80d66b747344f0ea7708c187645cfcc08b11aa424632f78e25bf05db51"}, - "credo": {:hex, :credo, "1.7.18", "5c5596bf7aedf9c8c227f13272ac499fe8eae6237bd326f2f07dfc173786f042", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "a189d164685fd945809e862fe76a7420c4398fa288d76257662aecb909d6b3e5"}, + "cowlib": {:hex, :cowlib, "2.17.1", "3e6053016d1ab245730f0af688755476dcedb1c25ed8fb5751f59a2bfdc0c9af", [:make, :rebar3], [], "hexpm", "ff08bd17e6dd931445b18af77315b9b5fe052407110964ad2588c686b57b5e3f"}, + "credo": {:hex, :credo, "1.7.19", "cc52129665fc7c15143d47838fda0f9cd6dac9ceced7bf4da6f85fcbfe64b12a", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "2d8bc95d5a7bb99dd2613621d4f08c6a3575c3fd4b62e6a2b48a100352a557b8"}, "ctx": {:hex, :ctx, "0.6.0", "8ff88b70e6400c4df90142e7f130625b82086077a45364a78d208ed3ed53c7fe", [:rebar3], [], "hexpm", "a14ed2d1b67723dbebbe423b28d7615eb0bdcba6ff28f2d1f1b0a7e1d4aa5fc2"}, "dialyxir": {:hex, :dialyxir, "1.4.7", "dda948fcee52962e4b6c5b4b16b2d8fa7d50d8645bbae8b8685c3f9ecb7f5f4d", [:mix], [{:erlex, ">= 0.2.8", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b34527202e6eb8cee198efec110996c25c5898f43a4094df157f8d28f27d9efe"}, - "erlex": {:hex, :erlex, "0.2.8", "cd8116f20f3c0afe376d1e8d1f0ae2452337729f68be016ea544a72f767d9c12", [:mix], [], "hexpm", "9d66ff9fedf69e49dc3fd12831e12a8a37b76f8651dd21cd45fcf5561a8a7590"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.45", "cba8369ab2a1342e419bc2760eec731b17be828941dcf494045d44766227e1d5", [:mix], [], "hexpm", "d3ec045bf122965db20c0bdb420e19ee1415843135327124918473feb4b328e8"}, + "erlex": {:hex, :erlex, "0.2.9", "7debbbaa9f4f368b8cd648983e0f1d7963028508e9c59e9d4ed504e94ef52a55", [:mix], [], "hexpm", "8cfffc0ec7159e6d73de2ab28a588064de80f88b2798d5cbe4482cbbc200178b"}, + "ex_doc": {:hex, :ex_doc, "0.40.3", "4a972ffe64bc07dc605af487e98fc19b72a4185f55ca031b94c0552d6071c1d9", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "2756e357742fecd9749b489b85d67c9ce99c465f2e75728d9e6dc8d704b973de"}, "file_system": {:hex, :file_system, "1.1.1", "31864f4685b0148f25bd3fbef2b1228457c0c89024ad67f7a81a3ffbc0bbad3a", [:mix], [], "hexpm", "7a15ff97dfe526aeefb090a7a9d3d03aa907e100e262a0f8f7746b78f8f87a5d"}, - "finch": {:hex, :finch, "0.21.0", "b1c3b2d48af02d0c66d2a9ebfb5622be5c5ecd62937cf79a88a7f98d48a8290c", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "87dc6e169794cb2570f75841a19da99cfde834249568f2a5b121b809588a4377"}, + "finch": {:hex, :finch, "0.23.0", "e3f9287ac25a8832f848b144c2b57346aac65b205e2e0629a52adfe6507fd837", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.8", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "80e58d3f936f57e3fdf404f83a3642897ae6d9fb642934e46da4d8fe761b99d5"}, "gproc": {:hex, :gproc, "0.9.1", "f1df0364423539cf0b80e8201c8b1839e229e5f9b3ccb944c5834626998f5b8c", [:rebar3], [], "hexpm", "905088e32e72127ed9466f0bac0d8e65704ca5e73ee5a62cb073c3117916d507"}, "grpcbox": {:hex, :grpcbox, "0.17.1", "6e040ab3ef16fe699ffb513b0ef8e2e896da7b18931a1ef817143037c454bcce", [:rebar3], [{:acceptor_pool, "~> 1.0.0", [hex: :acceptor_pool, repo: "hexpm", optional: false]}, {:chatterbox, "~> 0.15.1", [hex: :ts_chatterbox, repo: "hexpm", optional: false]}, {:ctx, "~> 0.6.0", [hex: :ctx, repo: "hexpm", optional: false]}, {:gproc, "~> 0.9.1", [hex: :gproc, repo: "hexpm", optional: false]}], "hexpm", "4a3b5d7111daabc569dc9cbd9b202a3237d81c80bf97212fbc676832cb0ceb17"}, "hpack": {:hex, :hpack_erl, "0.3.0", "2461899cc4ab6a0ef8e970c1661c5fc6a52d3c25580bc6dd204f84ce94669926", [:rebar3], [], "hexpm", "d6137d7079169d8c485c6962dfe261af5b9ef60fbc557344511c1e65e3d95fb0"}, "hpax": {:hex, :hpax, "1.0.3", "ed67ef51ad4df91e75cc6a1494f851850c0bd98ebc0be6e81b026e765ee535aa", [:mix], [], "hexpm", "8eab6e1cfa8d5918c2ce4ba43588e894af35dbd8e91e6e55c817bca5847df34a"}, - "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, + "jason": {:hex, :jason, "1.4.5", "2e3a008590b0b8d7388c20293e9dcc9cf3e5d642fd2a114e4cbbb52e595d940a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b0c823996102bcd0239b3c2444eb00409b72f6a140c1950bc8b457d836b30684"}, + "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, + "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, + "makeup_erlang": {:hex, :makeup_erlang, "1.1.0", "835f7e60792e08824cda445639555d7bf1bbbddb1b60b306e33cb6f6db24dc74", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "1cd6780fb1dd1a03979abaed0fe82712b0625118fd5257d3ebbf73f960c73c3c"}, "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, - "mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"}, + "mint": {:hex, :mint, "1.9.0", "d6f534c2a3e98b2a8cc749b4796eb77e9e3af79a76f96e4c74035a827de0d318", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "007154c7d8c43916aed3c93afd1f11aebbaa9c5ff4b7ba55ebe0d17ee0296042"}, "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, + "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, "opentelemetry": {:hex, :opentelemetry, "1.7.0", "20d0f12d3d1c398d3670fd44fd1a7c495dd748ab3e5b692a7906662e2fb1a38a", [:rebar3], [{:opentelemetry_api, "~> 1.5.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}], "hexpm", "a9173b058c4549bf824cbc2f1d2fa2adc5cdedc22aa3f0f826951187bbd53131"}, "opentelemetry_api": {:hex, :opentelemetry_api, "1.5.0", "1a676f3e3340cab81c763e939a42e11a70c22863f645aa06aafefc689b5550cf", [:mix, :rebar3], [], "hexpm", "f53ec8a1337ae4a487d43ac89da4bd3a3c99ddf576655d071deed8b56a2d5dda"}, "opentelemetry_exporter": {:hex, :opentelemetry_exporter, "1.10.0", "972e142392dbfa679ec959914664adefea38399e4f56ceba5c473e1cabdbad79", [:rebar3], [{:grpcbox, ">= 0.0.0", [hex: :grpcbox, repo: "hexpm", optional: false]}, {:opentelemetry, "~> 1.7.0", [hex: :opentelemetry, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.5.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:tls_certificate_check, "~> 1.18", [hex: :tls_certificate_check, repo: "hexpm", optional: false]}], "hexpm", "33a116ed7304cb91783f779dec02478f887c87988077bfd72840f760b8d4b952"}, "opentelemetry_telemetry": {:hex, :opentelemetry_telemetry, "1.1.2", "410ab4d76b0921f42dbccbe5a7c831b8125282850be649ee1f70050d3961118a", [:mix, :rebar3], [{:opentelemetry_api, "~> 1.3", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.1", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "641ab469deb181957ac6d59bce6e1321d5fe2a56df444fc9c19afcad623ab253"}, - "plug": {:hex, :plug, "1.19.1", "09bac17ae7a001a68ae393658aa23c7e38782be5c5c00c80be82901262c394c0", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "560a0017a8f6d5d30146916862aaf9300b7280063651dd7e532b8be168511e62"}, - "plug_cowboy": {:hex, :plug_cowboy, "2.8.1", "5aa391a5e8d1ac3192e36a3bcaff12b5fd6ef6c7e29b53a38e63a860783e77d0", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "4c200288673d5bc86a0ab7dc6a2a069176a74e5d573ef62740a1c517458a5f26"}, + "plug": {:hex, :plug, "1.20.2", "adbee2441232412e37fbb357fd5e4cd533fdd253b29f2e1992262b0f1fb01462", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b16baf55877d60891002ffc1ce0b3ff7d6f30a38a23e02e4d4293c4ac266f136"}, + "plug_cowboy": {:hex, :plug_cowboy, "2.9.0", "87e21e0d9054ced99c36d128f49e3ea2cd8b745fffb97de50bff99706087af4f", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.18", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "2002bafba4f3a45b55a58e68d70211b153a7ed18d37edb1ceb6e96e7a92c422e"}, "plug_crypto": {:hex, :plug_crypto, "2.1.1", "19bda8184399cb24afa10be734f84a16ea0a2bc65054e23a62bb10f06bc89491", [:mix], [], "hexpm", "6470bce6ffe41c8bd497612ffde1a7e4af67f36a15eea5f921af71cf3e11247c"}, "ranch": {:hex, :ranch, "1.8.1", "208169e65292ac5d333d6cdbad49388c1ae198136e4697ae2f474697140f201c", [:make, :rebar3], [], "hexpm", "aed58910f4e21deea992a67bf51632b6d60114895eb03bb392bb733064594dd0"}, - "req": {:hex, :req, "0.5.17", "0096ddd5b0ed6f576a03dde4b158a0c727215b15d2795e59e0916c6971066ede", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "0b8bc6ffdfebbc07968e59d3ff96d52f2202d0536f10fef4dc11dc02a2a43e39"}, + "req": {:hex, :req, "0.6.2", "b9b2024f35bcf60a92cc8cad2eaaf9d4e7aace463ff74be1afe5986830184413", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.21", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "cc9cd30a2ddd04989929b887178e1610c940456d962c6c3a52df6146d2eef9bf"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, "stream_data": {:hex, :stream_data, "1.3.0", "bde37905530aff386dea1ddd86ecbf00e6642dc074ceffc10b7d4e41dfd6aac9", [:mix], [], "hexpm", "3cc552e286e817dca43c98044c706eec9318083a1480c52ae2688b08e2936e3c"}, - "telemetry": {:hex, :telemetry, "1.4.1", "ab6de178e2b29b58e8256b92b382ea3f590a47152ca3651ea857a6cae05ac423", [:rebar3], [], "hexpm", "2172e05a27531d3d31dd9782841065c50dd5c3c7699d95266b2edd54c2dafa1c"}, - "tls_certificate_check": {:hex, :tls_certificate_check, "1.32.1", "f90f9647668f7af804ec63c4e67f3799931166caf8264b81f37c1d863a36ae1d", [:rebar3], [{:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "e78a157966456b500a87a2fc29cffcd6dcfb5a26348c8372a2c5c0a8e5797f51"}, - "vfs": {:git, "https://github.com/ivarvong/vfs.git", "32d2ab618ec12c16fe4f675b5ee8b563c660dd69", [ref: "32d2ab618ec12c16fe4f675b5ee8b563c660dd69"]}, + "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"}, + "tls_certificate_check": {:hex, :tls_certificate_check, "1.33.0", "01e0822a2ebb0b207c4964e8d32ad8b1b8ce1caf58f446e53f816d06db953de4", [:rebar3], [{:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "cab9a7439e2dbfe91b38104f2d8a4b6d61dbc4d3a5ad59ac364713a88c6cfd9b"}, + "vfs": {:hex, :vfs, "0.1.0", "7bac0a4cda3e4fc237d9c798116c2b23d7ece1f9f87651062b0f8f071f0b448d", [:mix], [{:telemetry, "~> 1.3", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "72f372bb60b4b4cf8fe67098c8559f007f115cb2fe3f94a05cdf9f18c71b0dd0"}, } diff --git a/scripts/consumer_smoke.sh b/scripts/consumer_smoke.sh new file mode 100755 index 0000000..90ddeb3 --- /dev/null +++ b/scripts/consumer_smoke.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# +# Consumer-install smoke test. +# +# Verifies that a *downstream* project can depend on the exgit package +# AS IT SHIPS TO HEX and actually use the public API end-to-end. +# +# Why this exists: the in-repo `mix test` suite runs with every dev/test +# dependency present, `test/support` on the load path, and the full +# source tree visible. It therefore CANNOT catch the two failure modes +# that only bite a real adopter: +# +# 1. A file the code needs is missing from the `files:` glob in mix.exs +# (the package tarball is incomplete). +# 2. Code under `lib/` has a compile-time reference to a dev/test-only +# or optional dependency (e.g. the optional `:vfs` defimpl, Bypass, +# StreamData) that isn't present in a clean consumer build. +# +# To reproduce a true consumer, we build the package the way Hex does, +# extract ONLY the shipped files, and depend on that extracted directory +# from a throwaway project compiled in :prod — so no dev/test deps and no +# test/support reach the build. Then we exercise the advertised workflow: +# clone a public repo and read a file by path. +# +# Network: clones https://github.com/elixir-ai-tools/just_bash (public, +# unauthenticated — per the "no auth on public repos" invariant). + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +echo "==> Building the Hex package (files: glob only)" +rm -f exgit-*.tar +mix hex.build +PKG_TAR="$(ls exgit-*.tar | head -1)" + +echo "==> Unpacking package contents ($PKG_TAR)" +# A Hex package tar contains: metadata.config, contents.tar.gz, CHECKSUM, +# VERSION. The shipped source files live inside contents.tar.gz. +tar -xf "$PKG_TAR" -C "$WORK" +mkdir -p "$WORK/pkg" +tar -xzf "$WORK/contents.tar.gz" -C "$WORK/pkg" +rm -f "$PKG_TAR" + +echo "==> Files the consumer will see:" +(cd "$WORK/pkg" && find . -type f | sort | sed 's/^/ /') + +echo "==> Scaffolding a throwaway consumer project" +CONSUMER="$WORK/consumer" +mkdir -p "$CONSUMER/lib" + +cat > "$CONSUMER/mix.exs" < 1.17", + # Depend on the EXTRACTED package, not the source tree — so only + # the shipped files and prod deps are visible. This is the whole + # point of the smoke test. + deps: [{:exgit, path: "${WORK}/pkg"}] + ] + end + + def application, do: [extra_applications: [:logger]] +end +EOF + +cat > "$CONSUMER/lib/smoke.ex" <<'EOF' +defmodule ExgitConsumerSmoke do + @moduledoc "Clone a public repo with exgit and read a file by path." + + @url "https://github.com/elixir-ai-tools/just_bash" + + def run do + # Eager, unauthenticated full clone of a public repo. + {:ok, repo} = Exgit.clone(@url) + + # Path-oriented read: returns {:ok, {mode, %Exgit.Object.Blob{}}, repo}. + {:ok, {_mode, blob}, _repo} = Exgit.FS.read_path(repo, "HEAD", "README.md") + data = blob.data + + if byte_size(data) == 0 do + raise "README.md came back empty" + end + + unless String.contains?(String.downcase(data), "bash") do + raise "README.md content did not look like just_bash (no 'bash' found)" + end + + IO.puts("OK: cloned just_bash, read README.md (#{byte_size(data)} bytes)") + end +end +EOF + +echo "==> Resolving deps + compiling consumer in :prod (warnings as errors)" +cd "$CONSUMER" +export MIX_ENV=prod +mix local.hex --force >/dev/null +mix local.rebar --force >/dev/null +mix deps.get +mix compile --warnings-as-errors + +echo "==> Running the smoke (clones just_bash over the network)" +mix run -e 'ExgitConsumerSmoke.run()' diff --git a/test/exgit/clone_test.exs b/test/exgit/clone_test.exs index 9caa87b..a435786 100644 --- a/test/exgit/clone_test.exs +++ b/test/exgit/clone_test.exs @@ -112,5 +112,107 @@ defmodule Exgit.CloneTest do assert {:ok, ^commit_sha} = RefStore.Disk.read_ref(RefStore.Disk.new(origin), "refs/heads/main") end + + test "defaults to pushing HEAD's branch when :refspecs is omitted", + %{origin_path: origin, clone_path: clone} do + seed_origin(origin) + transport = Exgit.Transport.File.new(origin) + {:ok, repo} = Exgit.clone(transport, path: clone) + + store = repo.object_store + blob = Blob.new("default refspec\n") + {:ok, blob_sha} = ObjectStore.Disk.put_object(store, blob) + tree = Tree.new([{"100644", "default.txt", blob_sha}]) + {:ok, tree_sha} = ObjectStore.Disk.put_object(store, tree) + + {:ok, parent_sha} = RefStore.Disk.resolve_ref(repo.ref_store, "refs/heads/main") + + commit = + Commit.new( + tree: tree_sha, + parents: [parent_sha], + author: "Clone 2000000000 +0000", + committer: "Clone 2000000000 +0000", + message: "default refspec\n" + ) + + {:ok, commit_sha} = ObjectStore.Disk.put_object(store, commit) + :ok = RefStore.Disk.write_ref(repo.ref_store, "refs/heads/main", commit_sha) + + # HEAD is symbolic → refs/heads/main (set by clone), so a bare + # push must push that branch. + assert {:ok, %{ref_results: _}} = Exgit.push(repo, transport) + + assert {:ok, ^commit_sha} = + RefStore.Disk.read_ref(RefStore.Disk.new(origin), "refs/heads/main") + end + + test "returns :no_refspecs when :refspecs is omitted and HEAD is not a branch", + %{origin_path: origin} do + seed_origin(origin) + transport = Exgit.Transport.File.new(origin) + + # A fresh memory repo has no HEAD at all — nothing to push. + {:ok, repo} = Exgit.init() + assert {:error, :no_refspecs} = Exgit.push(repo, transport) + end + + test "fails the whole push when a named refspec has no local ref", + %{origin_path: origin, clone_path: clone} do + seed_origin(origin) + transport = Exgit.Transport.File.new(origin) + {:ok, repo} = Exgit.clone(transport, path: clone) + + assert {:error, {:local_ref_not_found, "refs/heads/missing"}} = + Exgit.push(repo, transport, refspecs: ["refs/heads/main", "refs/heads/missing"]) + end + + test "deleting a ref the remote doesn't have stays a silent no-op", + %{origin_path: origin, clone_path: clone} do + seed_origin(origin) + transport = Exgit.Transport.File.new(origin) + {:ok, repo} = Exgit.clone(transport, path: clone) + + assert {:ok, %{ref_results: []}} = + Exgit.push(repo, transport, refspecs: [{:delete, "refs/heads/gone"}]) + end + end + + describe "remote-tracking refs" do + # Regression: `String.trim_leading/2` strips *repeated* prefixes, so a + # branch literally named "refs/heads/foo" (full ref + # "refs/heads/refs/heads/foo") used to be recorded as + # "refs/remotes/origin/foo" instead of + # "refs/remotes/origin/refs/heads/foo". + test "branch named refs/heads/foo maps to the right tracking ref", + %{origin_path: origin} do + commit_sha = seed_origin(origin) + + :ok = + RefStore.Disk.write_ref( + RefStore.Disk.new(origin), + "refs/heads/refs/heads/foo", + commit_sha + ) + + transport = Exgit.Transport.File.new(origin) + assert {:ok, repo} = Exgit.clone(transport) + + assert {:ok, ^commit_sha} = + Exgit.RefStore.read(repo.ref_store, "refs/remotes/origin/refs/heads/foo") + + assert {:error, _} = Exgit.RefStore.read(repo.ref_store, "refs/remotes/origin/foo") + end + end + + describe "file:// URL strings" do + test "clones from a file:// URL", %{origin_path: origin} do + commit_sha = seed_origin(origin) + + assert {:ok, repo} = Exgit.clone("file://" <> origin) + + assert {:ok, %Commit{message: "initial\n"}} = + Exgit.ObjectStore.get(repo.object_store, commit_sha) + end end end diff --git a/test/exgit/cloudflare_artifacts_perf_test.exs b/test/exgit/cloudflare_artifacts_perf_test.exs new file mode 100644 index 0000000..00d9869 --- /dev/null +++ b/test/exgit/cloudflare_artifacts_perf_test.exs @@ -0,0 +1,430 @@ +defmodule Exgit.CloudflareArtifactsPerfTest do + @moduledoc """ + Performance benchmark for Cloudflare Artifacts: bootstrap clone time + and per-blob first-touch latency across exgit's three clone modes. + + Modes: + + * `eager` — full clone, all objects fetched up front + * `filter:{:blob, :none}` — refs + commits + trees, blobs on demand + * `lazy: true` — refs only, commits + trees + blobs on demand + + Seeds a fresh repo with `@file_count × @file_bytes` random blobs in a + single commit, then runs `@clone_iterations` clones × `@reads_per_clone` + first-touch reads per partial mode. Reports compressed pack bytes + per clone and p50/p95/p99 latency for clone time and per-read time. + + Begins with a capability sniff: a single `filter:{:blob, :none}` + clone. Exgit returns `{:error, {:filter_unsupported, _}}` if the + server doesn't advertise the protocol-v2 `filter` capability — that + itself is the headline finding for partial-clone support. + + Tagged `:cloudflare_perf`. Run with `mix test --include cloudflare_perf`. + Requires `CF_ACCOUNT_ID` and `CF_API_TOKEN`; auto-skipped via + `test_helper.exs` when those aren't set. + """ + + use ExUnit.Case, async: false + + @moduletag :cloudflare_perf + @moduletag timeout: 1_800_000 + + alias Exgit.CloudflareArtifacts + alias Exgit.CloudflareArtifacts.{Repo, Token} + alias Exgit.Credentials.Artifacts, as: ArtifactsCreds + alias Exgit.Object.{Blob, Commit, Tree} + alias Exgit.ObjectStore + alias Exgit.ObjectStore.Promisor + alias Exgit.{RefStore, Repository, Transport} + alias Exgit.Test.CloudflareArtifacts, as: CFEnv + + @file_count 1_000 + @file_bytes 1_024 + @clone_iterations 10 + @reads_per_clone 50 + + setup_all do + client = + CloudflareArtifacts.new( + account_id: CFEnv.account_id(), + namespace: CFEnv.namespace(), + api_token: CFEnv.api_token() + ) + + repo_name = "exgit-perf-#{System.system_time(:millisecond)}-#{rand_hex(4)}" + + on_exit(fn -> _ = CloudflareArtifacts.delete_repo(client, repo_name) end) + + {:ok, %Repo{remote: remote}} = + CloudflareArtifacts.create_repo(client, + name: repo_name, + default_branch: "main", + description: "exgit perf benchmark" + ) + + {:ok, %Token{plaintext: write_token}} = + CloudflareArtifacts.create_token(client, + repo: repo_name, + scope: "write", + ttl: 7200 + ) + + {:ok, %Token{plaintext: read_token}} = + CloudflareArtifacts.create_token(client, + repo: repo_name, + scope: "read", + ttl: 7200 + ) + + branch = "refs/heads/main" + + IO.puts("\n[perf] building seed: #{@file_count} × #{@file_bytes}B random blobs ...") + + {build_us, {seed_repo, _commit_sha, filenames}} = + :timer.tc(fn -> build_seed_commit(branch, @file_count, @file_bytes) end) + + IO.puts("[perf] seed built locally in #{us_to_ms(build_us)} ms") + + write_transport = Transport.HTTP.new(remote, auth: ArtifactsCreds.auth(write_token)) + + IO.puts("[perf] pushing seed to CF ...") + + {push_us, {:ok, %{ref_results: ref_results}}} = + :timer.tc(fn -> Exgit.push(seed_repo, write_transport, refspecs: [branch]) end) + + assert Enum.any?(ref_results, &match?({^branch, :ok}, &1)) + IO.puts("[perf] pushed in #{us_to_ms(push_us)} ms") + + {:ok, + %{ + branch: branch, + filenames: filenames, + remote: remote, + read_token: read_token + }} + end + + test "perf: clone modes against CF Artifacts", ctx do + IO.puts("\n========== CF Artifacts Clone-Mode Benchmark ==========") + + IO.puts( + "repo size: #{@file_count} files × #{@file_bytes} bytes = " <> + "#{@file_count * @file_bytes} bytes raw" + ) + + IO.puts("clones per mode: #{@clone_iterations}") + IO.puts("first-touch reads per partial-mode clone: #{@reads_per_clone}") + + IO.puts("\n--- Capability check ---") + {filter_supported?, sniff_msg} = check_filter_capability(ctx) + IO.puts(sniff_msg) + + if not filter_supported? do + IO.puts(check_lazy_capability(ctx)) + end + + eager_stats = bench_eager(ctx, @clone_iterations) + + filter_stats = + if filter_supported? do + bench_partial(ctx, :filter_blob_none, @clone_iterations, @reads_per_clone) + else + IO.puts("\n[perf] skipping filter:{:blob, :none} benchmark (capability not advertised).") + nil + end + + # Run lazy regardless: even when the server doesn't advertise + # `filter`, exgit's on-demand fetch path still sends `filter: + # blob:none` per fetch — the server may silently ignore it (and + # ship the full pack each time, which would make lazy strictly + # worse than eager) or honor it (in which case lazy works and + # the capability advertisement is just a docs lie). + lazy_stats = bench_partial(ctx, :lazy, @clone_iterations, @reads_per_clone) + + print_summary(eager_stats, filter_stats, lazy_stats) + end + + # --- Capability sniff --- + + defp check_filter_capability(ctx) do + transport = make_transport(ctx) + + case Exgit.clone(transport, filter: {:blob, :none}) do + {:ok, %Repository{object_store: %Promisor{cache_bytes: bytes} = store}} -> + n = count_cached_objects(store) + + msg = + "[perf] CF advertises `filter` capability — partial clone succeeded.\n" <> + "[perf] bootstrap fetched #{n} objects, #{bytes} compressed bytes\n" <> + "[perf] (eager would fetch #{@file_count + 2} objects: #{@file_count} blobs + 1 tree + 1 commit)" + + {true, msg} + + {:error, {:filter_unsupported, _} = err} -> + msg = + "[perf] CF does NOT advertise `filter` capability.\n" <> + "[perf] exgit error: #{inspect(err)}\n" <> + "[perf] Implication: partial-clone agent workloads pay the full pack on every clone." + + {false, msg} + + other -> + {false, "[perf] Unexpected result from filter clone: #{inspect(other)}"} + end + end + + # If filter isn't supported, lazy mode will also fail somewhere — + # but where? Document the exact failure shape so we know whether + # exgit's lazy path is dead at clone time or at first-read time + # against this server. + defp check_lazy_capability(ctx) do + transport = make_transport(ctx) + + case Exgit.clone(transport, lazy: true) do + {:error, reason} -> + "[perf] `lazy: true` clone fails immediately: #{inspect(reason)}" + + {:ok, repo} -> + # Clone succeeded (refs only). First read will trigger an + # on-demand fetch which itself uses `filter: blob:none` + # (fs.ex:446) — that's where it should die. + sample = Enum.take_random(ctx.filenames, 1) |> hd() + + case Exgit.FS.read_path(repo, ctx.branch, sample) do + {:error, reason} -> + "[perf] `lazy: true` clone OK; first read fails: #{inspect(reason)}" + + {:ok, _, _} -> + "[perf] `lazy: true` clone + first read OK — server tolerates partial fetch?" + end + end + end + + # --- Mode benchmarks --- + + defp bench_eager(ctx, n) do + IO.puts("\n--- eager mode (#{n} iterations) ---") + + samples = + for _ <- 1..n do + transport = make_transport(ctx) + {us, {:ok, repo}} = :timer.tc(fn -> Exgit.clone(transport) end) + {us, eager_object_count(repo)} + end + + {clone_times, obj_counts} = Enum.unzip(samples) + + IO.puts("[perf] eager clone time: #{summarize(clone_times)}") + + IO.puts( + "[perf] eager object count: #{format_count(obj_counts)} (expected #{@file_count + 2})" + ) + + %{mode: :eager, clone_us: clone_times, objects: obj_counts} + end + + defp bench_partial(ctx, mode, clone_n, read_n) do + label = mode_label(mode) + IO.puts("\n--- #{label} mode (#{clone_n} clones × #{read_n} reads each) ---") + + iterations = + for _ <- 1..clone_n do + transport = make_transport(ctx) + clone_opts = clone_opts_for(mode) + + {clone_us, {:ok, repo}} = :timer.tc(fn -> Exgit.clone(transport, clone_opts) end) + bootstrap_bytes = repo.object_store.cache_bytes + + sample = Enum.take_random(ctx.filenames, read_n) + + {read_times_rev, repo} = + Enum.reduce(sample, {[], repo}, fn filename, {acc, repo} -> + {us, {:ok, {_mode, _blob}, repo}} = + :timer.tc(fn -> Exgit.FS.read_path(repo, ctx.branch, filename) end) + + {[us | acc], repo} + end) + + read_us = Enum.reverse(read_times_rev) + + %{ + clone_us: clone_us, + bootstrap_bytes: bootstrap_bytes, + read_us: read_us, + final_bytes: repo.object_store.cache_bytes + } + end + + clone_times = Enum.map(iterations, & &1.clone_us) + bootstrap_bytes_list = Enum.map(iterations, & &1.bootstrap_bytes) + final_bytes_list = Enum.map(iterations, & &1.final_bytes) + first_reads = Enum.map(iterations, &hd(&1.read_us)) + warm_reads = Enum.flat_map(iterations, &tl(&1.read_us)) + all_reads = Enum.flat_map(iterations, & &1.read_us) + + IO.puts("[perf] clone time: #{summarize(clone_times)}") + IO.puts("[perf] bootstrap bytes: #{format_bytes(bootstrap_bytes_list)}") + IO.puts("[perf] final bytes: #{format_bytes(final_bytes_list)} (after #{read_n} reads)") + IO.puts("[perf] read #1 (cold): #{summarize(first_reads)}") + IO.puts("[perf] reads 2..#{read_n} (warm): #{summarize(warm_reads)}") + IO.puts("[perf] all reads: #{summarize(all_reads)}") + + %{ + mode: mode, + clone_us: clone_times, + bootstrap_bytes: bootstrap_bytes_list, + final_bytes: final_bytes_list, + first_read_us: first_reads, + warm_read_us: warm_reads, + all_read_us: all_reads + } + end + + defp clone_opts_for(:filter_blob_none), do: [filter: {:blob, :none}] + defp clone_opts_for(:lazy), do: [lazy: true] + + defp mode_label(:filter_blob_none), do: "filter:{:blob, :none}" + defp mode_label(:lazy), do: "lazy: true" + + # --- Summary --- + + defp print_summary(eager, filter, lazy) do + IO.puts("\n========== Summary ==========") + + header = pad_row(["mode", "clone p50", "clone p95", "warm read p50", "warm read p95"]) + IO.puts(header) + IO.puts(String.duplicate("-", String.length(header))) + + print_mode_row("eager", eager) + if filter, do: print_mode_row("filter:{:blob, :none}", filter) + if lazy, do: print_mode_row("lazy: true", lazy) + + IO.puts("") + end + + defp print_mode_row(label, stats) do + clone_p50 = ms_str(percentile(stats.clone_us, 50)) + clone_p95 = ms_str(percentile(stats.clone_us, 95)) + + {read_p50, read_p95} = + case Map.get(stats, :warm_read_us) do + nil -> {"-", "-"} + [] -> {"-", "-"} + warm -> {ms_str(percentile(warm, 50)), ms_str(percentile(warm, 95))} + end + + IO.puts(pad_row([label, clone_p50, clone_p95, read_p50, read_p95])) + end + + defp pad_row(cols) do + widths = [22, 12, 12, 16, 16] + + cols + |> Enum.zip(widths) + |> Enum.map_join(" ", fn {col, w} -> String.pad_trailing(col, w) end) + end + + # --- Stats helpers --- + + defp summarize([]), do: "n=0" + + defp summarize(samples) do + n = length(samples) + p50 = ms_str(percentile(samples, 50)) + p95 = ms_str(percentile(samples, 95)) + p99 = ms_str(percentile(samples, 99)) + "n=#{n} p50=#{p50} p95=#{p95} p99=#{p99}" + end + + defp percentile([_ | _] = samples, p) do + sorted = Enum.sort(samples) + n = length(sorted) + idx = trunc(:math.ceil(p / 100 * n)) - 1 + idx = max(0, min(n - 1, idx)) + Enum.at(sorted, idx) + end + + defp ms_str(us) when is_integer(us), do: "#{us_to_ms(us)} ms" + defp ms_str(other), do: to_string(other) + + defp us_to_ms(us) when is_integer(us), do: Float.round(us / 1000, 1) + + defp format_bytes(byte_list) do + nums = Enum.reject(byte_list, &is_nil/1) + + case nums do + [] -> "n/a" + [single] -> "#{single}" + _ -> "min=#{Enum.min(nums)} max=#{Enum.max(nums)} mean=#{mean(nums) |> round()}" + end + end + + defp format_count(count_list) do + nums = Enum.reject(count_list, &is_nil/1) + + case nums do + [] -> "n/a" + [n] -> "#{n}" + _ -> "min=#{Enum.min(nums)} max=#{Enum.max(nums)}" + end + end + + # Only called from `format_bytes/1`'s `_` clause, i.e. with 2+ elements + # (the empty and single-element cases are handled before the call). + defp mean(xs), do: Enum.sum(xs) / length(xs) + + # --- exgit helpers --- + + defp eager_object_count(%Repository{object_store: %ObjectStore.Memory{objects: objs}}), + do: map_size(objs) + + defp eager_object_count(_), do: nil + + defp count_cached_objects(%Promisor{cache: %ObjectStore.Memory{objects: objs}}), + do: map_size(objs) + + defp count_cached_objects(_), do: :unknown + + defp make_transport(ctx) do + Transport.HTTP.new(ctx.remote, auth: ArtifactsCreds.auth(ctx.read_token)) + end + + defp build_seed_commit(branch, file_count, file_bytes) do + store = ObjectStore.Memory.new() + + {entries, store} = + Enum.reduce(1..file_count, {[], store}, fn i, {acc, store} -> + content = :crypto.strong_rand_bytes(file_bytes) + {:ok, blob_sha, store} = ObjectStore.put(store, Blob.new(content)) + filename = "file_#{String.pad_leading(Integer.to_string(i), 5, "0")}.bin" + {[{"100644", filename, blob_sha} | acc], store} + end) + + {:ok, tree_sha, store} = ObjectStore.put(store, Tree.new(entries)) + + commit = + Commit.new( + tree: tree_sha, + parents: [], + author: "Exgit Perf 1700000000 +0000", + committer: "Exgit Perf 1700000000 +0000", + message: "perf seed: #{file_count} x #{file_bytes}B\n" + ) + + {:ok, commit_sha, store} = ObjectStore.put(store, commit) + {:ok, ref_store} = RefStore.write(RefStore.Memory.new(), branch, commit_sha, []) + + repo = %Repository{ + object_store: store, + ref_store: ref_store, + config: Exgit.Config.new(), + path: nil + } + + filenames = entries |> Enum.map(&elem(&1, 1)) |> Enum.sort() + + {repo, commit_sha, filenames} + end + + defp rand_hex(n), do: n |> :crypto.strong_rand_bytes() |> Base.encode16(case: :lower) +end diff --git a/test/exgit/fs_prefetch_async_test.exs b/test/exgit/fs_prefetch_async_test.exs index 7737e4a..8853fb3 100644 --- a/test/exgit/fs_prefetch_async_test.exs +++ b/test/exgit/fs_prefetch_async_test.exs @@ -61,10 +61,10 @@ defmodule Exgit.FSPrefetchAsyncTest do end test "handle is updated after task completes", %{handle: handle} do - before = RepoHandle.get(handle) + before = RepoHandle.fetch!(handle) {:ok, task} = FS.prefetch_async(handle) {:ok, :prefetched} = FS.await_prefetch(task) - after_ = RepoHandle.get(handle) + after_ = RepoHandle.fetch!(handle) # For Memory-backed store, the update is a no-op but the # call still runs (which is what we're testing — the task @@ -81,7 +81,7 @@ defmodule Exgit.FSPrefetchAsyncTest do # 100 reads while the task is (was) running. for _ <- 1..100 do - assert %Repository{} = RepoHandle.get(handle) + assert %Repository{} = RepoHandle.fetch!(handle) end {:ok, :prefetched} = FS.await_prefetch(task) diff --git a/test/exgit/fs_size_test.exs b/test/exgit/fs_size_test.exs new file mode 100644 index 0000000..6658baa --- /dev/null +++ b/test/exgit/fs_size_test.exs @@ -0,0 +1,217 @@ +defmodule Exgit.FsSizeTest do + use ExUnit.Case, async: true + + alias Exgit.{FS, ObjectStore, RefStore, Repository} + alias Exgit.Object.{Blob, Commit, Tree} + alias Exgit.ObjectStore.{Disk, Memory, Promisor} + + describe "Memory.object_size/2" do + test "matches byte_size of the content across sizes, regardless of compressibility" do + for data <- [ + "", + "hello\n", + String.duplicate("a", 100_000), + :crypto.strong_rand_bytes(1_000_000) + ] do + store = Memory.new() + {:ok, sha, store} = ObjectStore.put(store, Blob.new(data)) + assert ObjectStore.object_size(store, sha) == {:ok, byte_size(data)} + end + end + + test "returns :not_found for an absent sha" do + assert ObjectStore.object_size(Memory.new(), :crypto.strong_rand_bytes(20)) == + {:error, :not_found} + end + + test "import_objects populates the size index" do + data = "imported\n" + {sha, content} = {raw_blob_sha(data), data} + {:ok, store} = ObjectStore.import_objects(Memory.new(), [{:blob, sha, content}]) + assert ObjectStore.object_size(store, sha) == {:ok, byte_size(data)} + end + + test "streaming write records the actual (summed) size, not the declared one" do + store = Memory.new() + # Declare 0 on purpose: the index must reflect bytes actually written. + {:ok, h} = ObjectStore.open_write(store, :blob, 0) + {:ok, h} = ObjectStore.write_chunk(store, h, "abc") + {:ok, h} = ObjectStore.write_chunk(store, h, "de") + {:ok, sha, store} = ObjectStore.close_write(store, h) + + assert ObjectStore.object_size(store, sha) == {:ok, 5} + end + end + + describe "Promisor.object_size/2 (never fetches)" do + test "returns the size for a locally cached object" do + blob = Blob.new("cached body\n") + # `:no_transport` is never used: object_size only reads the cache. + promisor = Promisor.new(:no_transport, initial_objects: [blob]) + sha = Blob.sha(blob) + + assert ObjectStore.object_size(promisor, sha) == {:ok, byte_size("cached body\n")} + end + + test "returns :not_local for an un-fetched object and does NOT touch the transport" do + # The transport is a bare atom with no Transport impl: if object_size + # tried to fetch, the call would crash. A clean :not_local proves it didn't. + promisor = Promisor.new(:boom_if_used, initial_objects: []) + absent = :crypto.strong_rand_bytes(20) + + assert ObjectStore.object_size(promisor, absent) == {:error, :not_local} + end + end + + describe "Disk.object_size/2" do + test "reads loose-object size from the header" do + dir = Path.join(System.tmp_dir!(), "exgit_size_#{System.unique_integer([:positive])}") + File.mkdir_p!(dir) + on_exit(fn -> File.rm_rf(dir) end) + + data = String.duplicate("x", 250_000) + store = Disk.new(dir) + {:ok, sha, store} = ObjectStore.put(store, Blob.new(data)) + + assert ObjectStore.object_size(store, sha) == {:ok, byte_size(data)} + end + end + + describe "FS.size/3" do + setup do + store = Memory.new() + + readme = Blob.new("hello world\n") + {:ok, readme_sha, store} = ObjectStore.put(store, readme) + + nested = Blob.new("deep\n") + {:ok, nested_sha, store} = ObjectStore.put(store, nested) + + nested_tree = Tree.new([{"100644", "c.txt", nested_sha}]) + {:ok, nested_tree_sha, store} = ObjectStore.put(store, nested_tree) + + src_tree = Tree.new([{"40000", "deep", nested_tree_sha}]) + {:ok, src_sha, store} = ObjectStore.put(store, src_tree) + + # Gitlink (submodule) entry: the SHA names a commit in the + # submodule's OWN repository, so it is never present in this + # object store — exactly the situation size/3 must not treat + # as a missing local object. + submodule_commit_sha = :crypto.strong_rand_bytes(20) + + root = + Tree.new([ + {"100644", "README.md", readme_sha}, + {"160000", "vendored", submodule_commit_sha}, + {"40000", "src", src_sha} + ]) + + {:ok, root_sha, store} = ObjectStore.put(store, root) + + commit = + Commit.new( + tree: root_sha, + parents: [], + author: "T 1700000000 +0000", + committer: "T 1700000000 +0000", + message: "init\n" + ) + + {:ok, commit_sha, store} = ObjectStore.put(store, commit) + {:ok, refs} = RefStore.write(RefStore.Memory.new(), "refs/heads/main", commit_sha, []) + {:ok, refs} = RefStore.write(refs, "HEAD", {:symbolic, "refs/heads/main"}, []) + + repo = %Repository{ + object_store: store, + ref_store: refs, + config: Exgit.Config.new(), + path: nil + } + + {:ok, repo: repo} + end + + test "returns the blob size for a top-level file", %{repo: repo} do + assert {:ok, size, %Repository{}} = FS.size(repo, "HEAD", "README.md") + assert size == byte_size("hello world\n") + end + + test "returns the blob size for a nested file", %{repo: repo} do + assert {:ok, 5, %Repository{}} = FS.size(repo, "HEAD", "src/deep/c.txt") + end + + test "matches read_path's blob byte_size", %{repo: repo} do + {:ok, {_mode, %Blob{data: data}}, _repo} = FS.read_path(repo, "HEAD", "README.md") + assert {:ok, byte_size(data), repo} == FS.size(repo, "HEAD", "README.md") + end + + test "rejects directories with :not_a_blob", %{repo: repo} do + assert FS.size(repo, "HEAD", "src") == {:error, :not_a_blob} + end + + test "returns :submodule for a gitlink entry instead of a doomed lookup", %{repo: repo} do + # Before the short-circuit, this fell through to an + # object_size lookup of the submodule's commit SHA and + # surfaced a misleading :not_found / :not_local. + assert FS.size(repo, "HEAD", "vendored") == {:error, :submodule} + end + + test "read_path and stat agree on gitlink handling", %{repo: repo} do + # read_path must fail the same way as size/3 — before any + # object lookup of the submodule's commit SHA... + assert FS.read_path(repo, "HEAD", "vendored") == {:error, :submodule} + + # ...while stat reports the entry without fetching anything. + assert {:ok, %{type: :submodule, mode: "160000", size: nil}, %Repository{}} = + FS.stat(repo, "HEAD", "vendored") + end + + test "returns :not_found for a missing path", %{repo: repo} do + assert FS.size(repo, "HEAD", "nope.txt") == {:error, :not_found} + end + end + + describe "FS.size/3 on a lazy clone" do + test "returns :not_local without fetching the blob" do + # Seed a Promisor cache with the commit + trees, but NOT the blob. + # `walk_path` resolves the path through cached trees; the blob's size + # is reported as :not_local rather than triggering a transport fetch. + store = Memory.new() + blob = Blob.new("would be huge\n") + blob_sha = Blob.sha(blob) + root = Tree.new([{"100644", "big.bin", blob_sha}]) + {:ok, root_sha, store} = ObjectStore.put(store, root) + + commit = + Commit.new( + tree: root_sha, + parents: [], + author: "T 1700000000 +0000", + committer: "T 1700000000 +0000", + message: "init\n" + ) + + {:ok, commit_sha, store} = ObjectStore.put(store, commit) + + # Cache holds commit + root tree only — the blob is deliberately absent. + {:ok, commit_obj} = ObjectStore.get(store, commit_sha) + {:ok, root_obj} = ObjectStore.get(store, root_sha) + promisor = Promisor.new(:boom_if_used, initial_objects: [commit_obj, root_obj]) + + {:ok, refs} = RefStore.write(RefStore.Memory.new(), "refs/heads/main", commit_sha, []) + {:ok, refs} = RefStore.write(refs, "HEAD", {:symbolic, "refs/heads/main"}, []) + + repo = %Repository{ + object_store: promisor, + ref_store: refs, + config: Exgit.Config.new(), + path: nil + } + + assert FS.size(repo, "HEAD", "big.bin") == {:error, :not_local} + end + end + + # Compute a blob sha the same way the store does, for import_objects tests. + defp raw_blob_sha(data), do: Blob.sha(Blob.new(data)) +end diff --git a/test/exgit/fs_test.exs b/test/exgit/fs_test.exs index 6063a4f..a0779ae 100644 --- a/test/exgit/fs_test.exs +++ b/test/exgit/fs_test.exs @@ -119,7 +119,7 @@ defmodule Exgit.FsTest do end end - describe "read_path/4 with resolve_lfs_pointers" do + describe "read_path/4 with detect_lfs_pointers" do # Build a repo containing: # /real.bin (200 bytes random — looks like a real binary) # /pointed.bin (a valid LFS pointer file, as if `git lfs` @@ -128,7 +128,7 @@ defmodule Exgit.FsTest do # # and verify: # 1. Default behavior (no flag) returns %Blob{} in all cases. - # 2. With resolve_lfs_pointers: true, pointed.bin surfaces as + # 2. With detect_lfs_pointers: true, pointed.bin surfaces as # {:lfs_pointer, info}; real.bin and README.md stay as blobs. setup do store = ObjectStore.Memory.new() @@ -202,7 +202,7 @@ defmodule Exgit.FsTest do test "with flag, pointer file surfaces as {:lfs_pointer, info}", %{repo: repo} do assert {:ok, {"100644", {:lfs_pointer, info}}, _} = - FS.read_path(repo, "HEAD", "pointed.bin", resolve_lfs_pointers: true) + FS.read_path(repo, "HEAD", "pointed.bin", detect_lfs_pointers: true) assert info.oid == "sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393" @@ -214,14 +214,14 @@ defmodule Exgit.FsTest do test "with flag, normal binary blob stays as %Blob{}", %{repo: repo} do # A 200-byte random blob must NOT be mistaken for a pointer. assert {:ok, {"100644", %Blob{} = blob}, _} = - FS.read_path(repo, "HEAD", "real.bin", resolve_lfs_pointers: true) + FS.read_path(repo, "HEAD", "real.bin", detect_lfs_pointers: true) assert byte_size(blob.data) == 200 end test "with flag, plain text blob stays as %Blob{}", %{repo: repo} do assert {:ok, {"100644", %Blob{data: "# Project\n"}}, _} = - FS.read_path(repo, "HEAD", "README.md", resolve_lfs_pointers: true) + FS.read_path(repo, "HEAD", "README.md", detect_lfs_pointers: true) end end @@ -293,43 +293,33 @@ defmodule Exgit.FsTest do describe "glob/3" do test "matches *.md in the root", %{repo: repo} do - assert {:ok, ["README.md"]} = FS.glob(repo, "HEAD", "*.md") + assert FS.glob(repo, "HEAD", "*.md") == ["README.md"] end - test "matches **/*.ex across subdirs", %{repo: repo} do - {:ok, paths} = FS.glob(repo, "HEAD", "**/*.ex") - - assert Enum.sort(paths) == ["src/a.ex", "src/b.ex", "src/nested/c.ex"] + test "matches **/*.ex across subdirs, sorted", %{repo: repo} do + assert FS.glob(repo, "HEAD", "**/*.ex") == ["src/a.ex", "src/b.ex", "src/nested/c.ex"] end test "matches src/*.ex (one level)", %{repo: repo} do - {:ok, paths} = FS.glob(repo, "HEAD", "src/*.ex") - assert Enum.sort(paths) == ["src/a.ex", "src/b.ex"] + assert FS.glob(repo, "HEAD", "src/*.ex") == ["src/a.ex", "src/b.ex"] end test "brace expansion: **/*.{md,ex} matches both extensions", %{repo: repo} do - {:ok, paths} = FS.glob(repo, "HEAD", "**/*.{md,ex}") - - assert Enum.sort(paths) == + assert FS.glob(repo, "HEAD", "**/*.{md,ex}") == ["README.md", "src/a.ex", "src/b.ex", "src/nested/c.ex"] end test "brace with a single option is equivalent to the literal", %{repo: repo} do - {:ok, a} = FS.glob(repo, "HEAD", "**/*.{ex}") - {:ok, b} = FS.glob(repo, "HEAD", "**/*.ex") - assert Enum.sort(a) == Enum.sort(b) + assert FS.glob(repo, "HEAD", "**/*.{ex}") == FS.glob(repo, "HEAD", "**/*.ex") end test "brace expansion at different positions in the pattern", %{repo: repo} do # {src,lib}/*.ex — no `lib/` exists, so only src hits match. - {:ok, paths} = FS.glob(repo, "HEAD", "{src,lib}/*.ex") - assert Enum.sort(paths) == ["src/a.ex", "src/b.ex"] + assert FS.glob(repo, "HEAD", "{src,lib}/*.ex") == ["src/a.ex", "src/b.ex"] end test "brace expansion with three options", %{repo: repo} do - {:ok, paths} = FS.glob(repo, "HEAD", "**/*.{md,ex,missing}") - - assert Enum.sort(paths) == + assert FS.glob(repo, "HEAD", "**/*.{md,ex,missing}") == ["README.md", "src/a.ex", "src/b.ex", "src/nested/c.ex"] end @@ -337,15 +327,13 @@ defmodule Exgit.FsTest do # `foo{,bar}` matches either `foo` or `foobar`. We exercise the # parse path; the fixture has no files matching either literal, # so we expect an empty result — but no crash. - {:ok, paths} = FS.glob(repo, "HEAD", "README{,.bak}.md") - assert paths == ["README.md"] + assert FS.glob(repo, "HEAD", "README{,.bak}.md") == ["README.md"] end test "unmatched opening brace is treated as a literal character", %{repo: repo} do # `{` without a closing `}` isn't a valid alternation — we treat # it as a literal (matches against a nonexistent file). - {:ok, paths} = FS.glob(repo, "HEAD", "README{.md") - assert paths == [] + assert FS.glob(repo, "HEAD", "README{.md") == [] end end diff --git a/test/exgit/object/tag_test.exs b/test/exgit/object/tag_test.exs index e1e6acd..d3c5276 100644 --- a/test/exgit/object/tag_test.exs +++ b/test/exgit/object/tag_test.exs @@ -21,10 +21,11 @@ defmodule Exgit.Object.TagTest do encoded = tag |> Tag.encode() |> IO.iodata_to_binary() assert {:ok, decoded} = Tag.decode(encoded) assert decoded.object == obj_sha - assert decoded.type == "commit" - assert decoded.tag == "v1.0" - assert decoded.tagger == @tagger + assert Tag.type(decoded) == "commit" + assert Tag.tag(decoded) == "v1.0" + assert Tag.tagger(decoded) == @tagger assert decoded.message == "release v1.0\n" + assert decoded == tag end test "round-trips a tag without tagger" do @@ -39,8 +40,52 @@ defmodule Exgit.Object.TagTest do encoded = tag |> Tag.encode() |> IO.iodata_to_binary() assert {:ok, decoded} = Tag.decode(encoded) - assert decoded.tagger == nil - assert decoded.tag == "v0.1" + assert Tag.tagger(decoded) == nil + assert Tag.tag(decoded) == "v0.1" + end + end + + describe "byte-exact fidelity" do + # Pinned against `git hash-object -t tag --literally`. + @exotic_sha "738c8cfdfce431cfe7add4760aa61ebd5e1964d1" + + test "decode |> encode is byte-exact for a tag with unknown and multi-line headers" do + raw = + "object #{String.duplicate("a", 40)}\n" <> + "type commit\n" <> + "tag v9.9\n" <> + "tagger #{@tagger}\n" <> + "x-custom something unusual\n" <> + "signature -----BEGIN PGP SIGNATURE-----\n" <> + " abc123\n" <> + " -----END PGP SIGNATURE-----\n" <> + "\n" <> + "Release with exotic headers\n" + + assert {:ok, tag} = Tag.decode(raw) + + # Unknown headers are preserved verbatim and in order; continuation + # lines fold into the previous header's value with newlines. + assert {"x-custom", "something unusual"} in tag.headers + + assert {"signature", "-----BEGIN PGP SIGNATURE-----\nabc123\n-----END PGP SIGNATURE-----"} in tag.headers + + assert Enum.map(tag.headers, &elem(&1, 0)) == + ["object", "type", "tag", "tagger", "x-custom", "signature"] + + # Re-encoding must reproduce the original bytes, so the SHA is stable. + assert IO.iodata_to_binary(Tag.encode(tag)) == raw + assert Tag.sha_hex(tag) == @exotic_sha + end + end + + describe "decode/1 validation" do + test "missing required headers return errors" do + assert {:error, {:missing_header, "object"}} = Tag.decode("type commit\ntag v1\n\nm\n") + + hex = String.duplicate("a", 40) + assert {:error, {:missing_header, "type"}} = Tag.decode("object #{hex}\ntag v1\n\nm\n") + assert {:error, {:missing_header, "tag"}} = Tag.decode("object #{hex}\ntype commit\n\nm\n") end end end diff --git a/test/exgit/object/tree_roundtrip_test.exs b/test/exgit/object/tree_roundtrip_test.exs index 3af089c..ad8822b 100644 --- a/test/exgit/object/tree_roundtrip_test.exs +++ b/test/exgit/object/tree_roundtrip_test.exs @@ -80,7 +80,7 @@ defmodule Exgit.Object.TreeRoundtripTest do # Strip the "tree \0" header. {pos, 1} = :binary.match(inflated, <<0>>) - <<_header::binary-size(pos), 0, content::binary>> = inflated + <<_header::binary-size(^pos), 0, content::binary>> = inflated {:ok, tree} = Exgit.Object.Tree.decode(content) assert IO.iodata_to_binary(Exgit.Object.Tree.encode(tree)) == content diff --git a/test/exgit/object/tree_test.exs b/test/exgit/object/tree_test.exs index 189af43..e40c2e9 100644 --- a/test/exgit/object/tree_test.exs +++ b/test/exgit/object/tree_test.exs @@ -58,6 +58,51 @@ defmodule Exgit.Object.TreeTest do end end + describe "canonical_mode/1 with zero-padded modes" do + test "padded modes canonicalize to their file type, not to a blob mode" do + # Regression: these used to fall through to the exec-bit coercion + # and come back as "100644"/"100755", turning dirs into files. + assert Tree.canonical_mode("040000") == "40000" + assert Tree.canonical_mode("0120000") == "120000" + assert Tree.canonical_mode("0160000") == "160000" + assert Tree.canonical_mode("0100644") == "100644" + assert Tree.canonical_mode("0100755") == "100755" + end + + test "strict mode still raises on zero-padded modes" do + assert_raise ArgumentError, fn -> Tree.canonical_mode("040000", true) end + end + + test "new/1 with padded modes matches canonical modes, round-trips, stable SHA" do + sha = :binary.copy(<<0xAB>>, 20) + + padded = + Tree.new([ + {"040000", "dir", sha}, + {"0120000", "link", sha}, + {"0160000", "sub", sha}, + {"0100644", "file", sha} + ]) + + canonical = + Tree.new([ + {"40000", "dir", sha}, + {"120000", "link", sha}, + {"160000", "sub", sha}, + {"100644", "file", sha} + ]) + + assert padded.entries == canonical.entries + + encoded = padded |> Tree.encode() |> IO.iodata_to_binary() + assert {:ok, decoded} = Tree.decode(encoded) + assert decoded.entries == padded.entries + + # Pinned against `git mktree --missing` fed the same four entries. + assert Tree.sha_hex(padded) == "93a9f977c0cdaecda57e0ec64d9a5adba8f92e1c" + end + end + describe "sha/1" do @tag :git_cross_check test "matches git mktree" do diff --git a/test/exgit/object_store/disk_object_size_test.exs b/test/exgit/object_store/disk_object_size_test.exs new file mode 100644 index 0000000..8c6f7e2 --- /dev/null +++ b/test/exgit/object_store/disk_object_size_test.exs @@ -0,0 +1,92 @@ +defmodule Exgit.ObjectStore.DiskObjectSizeTest do + @moduledoc """ + Edge cases for `Disk.object_size/2`'s bounded header streaming: + truncated zlib streams, non-zlib bytes, headers that exceed the + scan budget, and the packed-object fallback. The happy loose path + is covered in `Exgit.FsSizeTest`. + """ + + use ExUnit.Case, async: false + + alias Exgit.Object.Blob + alias Exgit.ObjectStore + alias Exgit.Pack.Index + alias Exgit.Test.PackBuilder + + setup do + root = + Path.join( + System.tmp_dir!(), + "exgit_disk_size_#{System.unique_integer([:positive, :monotonic])}" + ) + + File.mkdir_p!(Path.join(root, "objects")) + on_exit(fn -> File.rm_rf!(root) end) + %{root: root, store: ObjectStore.Disk.new(root)} + end + + describe "loose-object error paths" do + test "truncated loose object (valid zlib prefix cut mid-stream) returns a clean error", + %{root: root, store: store} do + sha = :crypto.strong_rand_bytes(20) + compressed = :zlib.compress("blob 5\0hello") + # Keep only the 2-byte zlib stream header: a valid prefix that + # inflates to nothing, so the NUL hunt runs out of input at EOF. + write_loose(root, sha, binary_part(compressed, 0, 2)) + + assert ObjectStore.object_size(store, sha) == {:error, :malformed_object_header} + end + + test "corrupt non-zlib bytes return a clean error", %{root: root, store: store} do + sha = :crypto.strong_rand_bytes(20) + write_loose(root, sha, "definitely not a zlib stream") + + assert ObjectStore.object_size(store, sha) == {:error, :zlib_error} + end + + test "header longer than the scan budget returns a clean error", + %{root: root, store: store} do + sha = :crypto.strong_rand_bytes(20) + # Valid zlib, but the "header" never terminates: 200 bytes with + # no NUL blows the @max_header_bytes budget instead of inflating + # the whole object hunting for one. + write_loose(root, sha, :zlib.compress("blob " <> String.duplicate("9", 200))) + + assert ObjectStore.object_size(store, sha) == {:error, :malformed_object_header} + end + end + + describe "packed-object fallback" do + test "reports the exact content byte size for a packed blob", + %{root: root, store: store} do + content = "packed blob body " <> :crypto.strong_rand_bytes(1_000) + sha = Blob.sha(Blob.new(content)) + + pack = PackBuilder.build([{:full, :blob, content}]) + # Single-entry pack: the object spans from the 12-byte header to + # the 20-byte checksum trailer. + entry_bytes = binary_part(pack, 12, byte_size(pack) - 32) + checksum = binary_part(pack, byte_size(pack) - 20, 20) + idx = Index.write([{sha, :erlang.crc32(entry_bytes), 12}], checksum) + + pack_dir = Path.join(root, "objects/pack") + File.mkdir_p!(pack_dir) + File.write!(Path.join(pack_dir, "pack-size.pack"), pack) + File.write!(Path.join(pack_dir, "pack-size.idx"), idx) + + # No loose object on disk, so this exercises the pack fallback. + assert ObjectStore.object_size(store, sha) == {:ok, byte_size(content)} + end + end + + # Place raw bytes at the loose path for `sha`. object_size/2 only + # parses the header — it never verifies the sha — so a random + # 20-byte sha addresses whatever bytes we plant. + defp write_loose(root, sha, bytes) do + hex = Base.encode16(sha, case: :lower) + <> = hex + dir = Path.join([root, "objects", prefix]) + File.mkdir_p!(dir) + File.write!(Path.join(dir, rest), bytes) + end +end diff --git a/test/exgit/object_store/disk_pack_lookup_test.exs b/test/exgit/object_store/disk_pack_lookup_test.exs index a6a0cb0..02c9ecc 100644 --- a/test/exgit/object_store/disk_pack_lookup_test.exs +++ b/test/exgit/object_store/disk_pack_lookup_test.exs @@ -39,6 +39,26 @@ defmodule Exgit.ObjectStore.DiskPackLookupTest do end end + test "object_size/2 reports the inflated size of a packed object", %{ + root: root, + store: store + } do + blobs = [Blob.new("a"), Blob.new("bb"), Blob.new("ccc")] + pack = Writer.build(blobs) + + pack_path = Path.join(root, "objects/pack/pack-size.pack") + idx_path = Path.join(root, "objects/pack/pack-size.idx") + File.write!(pack_path, pack) + + entries = pack_entries(pack, blobs) + pack_checksum = binary_part(pack, byte_size(pack) - 20, 20) + File.write!(idx_path, Index.write(entries, pack_checksum)) + + for blob <- blobs do + assert ObjectStore.object_size(store, Blob.sha(blob)) == {:ok, byte_size(blob.data)} + end + end + @tag :slow test "repeated lookups into a single pack scale sub-linearly in pack size", %{ root: root, @@ -116,7 +136,7 @@ defmodule Exgit.ObjectStore.DiskPackLookupTest do end defp describe_entry(pack, offset, blob) do - <<_::binary-size(offset), from_here::binary>> = pack + <<_::binary-size(^offset), from_here::binary>> = pack {_type_code, _size, after_header} = Exgit.Pack.Common.decode_type_size_varint(from_here) header_len = byte_size(from_here) - byte_size(after_header) diff --git a/test/exgit/object_store/disk_pread_test.exs b/test/exgit/object_store/disk_pread_test.exs index 781da3f..3c46b84 100644 --- a/test/exgit/object_store/disk_pread_test.exs +++ b/test/exgit/object_store/disk_pread_test.exs @@ -144,7 +144,7 @@ defmodule Exgit.ObjectStore.DiskPreadTest do end defp describe_entry(pack, offset, blob) do - <<_::binary-size(offset), from_here::binary>> = pack + <<_::binary-size(^offset), from_here::binary>> = pack {_type_code, _size, after_header} = Exgit.Pack.Common.decode_type_size_varint(from_here) header_len = byte_size(from_here) - byte_size(after_header) diff --git a/test/exgit/object_store/disk_test.exs b/test/exgit/object_store/disk_test.exs index 4694476..196ca2a 100644 --- a/test/exgit/object_store/disk_test.exs +++ b/test/exgit/object_store/disk_test.exs @@ -70,6 +70,53 @@ defmodule Exgit.ObjectStore.DiskTest do end end + describe "object_size" do + test "reads the size from the loose header without materializing", %{store: store} do + data = String.duplicate("z", 300_000) + {:ok, sha} = Disk.put_object(store, Blob.new(data)) + + assert Disk.object_size(store, sha) == {:ok, byte_size(data)} + end + + test "returns not_found for a missing object", %{store: store} do + assert {:error, :not_found} = Disk.object_size(store, :crypto.hash(:sha, "absent")) + end + + test "returns zlib_error for a corrupt loose object", %{store: store, path: path} do + sha = :crypto.hash(:sha, "corrupt") + write_loose_bytes(path, sha, "definitely not zlib data") + + assert {:error, :zlib_error} = Disk.object_size(store, sha) + end + + test "returns malformed_object_header for a truncated loose object", %{ + store: store, + path: path + } do + # Valid deflate stream cut off before the header's NUL appears. + sha = :crypto.hash(:sha, "truncated") + compressed = :zlib.compress("blob 1234\0" <> String.duplicate("y", 1234)) + write_loose_bytes(path, sha, binary_part(compressed, 0, 2)) + + assert {:error, :malformed_object_header} = Disk.object_size(store, sha) + end + + test "rejects a header with no NUL inside the scan budget", %{store: store, path: path} do + sha = :crypto.hash(:sha, "no-nul") + write_loose_bytes(path, sha, :zlib.compress(String.duplicate("a", 4096))) + + assert {:error, :malformed_object_header} = Disk.object_size(store, sha) + end + end + + defp write_loose_bytes(path, sha, bytes) do + hex = Base.encode16(sha, case: :lower) + <> = hex + dir = Path.join([path, "objects", prefix]) + File.mkdir_p!(dir) + File.write!(Path.join(dir, rest), bytes) + end + describe "delete" do test "removes the object", %{store: store} do blob = Blob.new("to delete") diff --git a/test/exgit/object_store/memory_test.exs b/test/exgit/object_store/memory_test.exs index 0b23851..4b3ca39 100644 --- a/test/exgit/object_store/memory_test.exs +++ b/test/exgit/object_store/memory_test.exs @@ -27,6 +27,26 @@ defmodule Exgit.ObjectStore.MemoryTest do refute Memory.has_object?(store, :crypto.hash(:sha, "other")) end + test "delete_object removes the object AND its size index entry" do + store = Memory.new() + {:ok, sha, store} = Memory.put_object(store, Blob.new("hello")) + assert {:ok, 5} = Memory.object_size(store, sha) + + assert {:ok, freed, store} = Memory.delete_object(store, sha) + assert freed > 0 + + refute Memory.has_object?(store, sha) + assert {:error, :not_found} = Memory.get_object(store, sha) + # The sizes index must be kept in lockstep — no stale size for a + # deleted object. + assert {:error, :not_found} = Memory.object_size(store, sha) + end + + test "delete_object returns not_found for a missing sha" do + assert {:error, :not_found} = + Memory.delete_object(Memory.new(), :crypto.hash(:sha, "nope")) + end + test "import_objects" do content = "imported" sha = Exgit.Object.sha(Blob.new(content)) diff --git a/test/exgit/object_store/promisor_eviction_test.exs b/test/exgit/object_store/promisor_eviction_test.exs index 7c2e5cb..4c5840d 100644 --- a/test/exgit/object_store/promisor_eviction_test.exs +++ b/test/exgit/object_store/promisor_eviction_test.exs @@ -57,6 +57,21 @@ defmodule Exgit.ObjectStore.PromisorEvictionTest do :ok end + test "eviction drops the size index too — object_size never returns a stale size" do + # cap=1 byte: the commit is evicted by the same put that inserted it. + p = Promisor.new(%Stub{}, max_cache_bytes: 1) + {:ok, sha, p} = Promisor.put(p, make_commit("evict me\n")) + + refute Promisor.has_object?(p, sha) + assert {:error, :not_local} = Promisor.object_size(p, sha) + assert {:error, :not_local} = Exgit.ObjectStore.object_size(p, sha) + + # The desync lived in the underlying Memory cache: eviction removed + # the object from `objects` but left its `sizes` entry behind, so a + # direct size lookup returned the stale size for an evicted object. + assert {:error, :not_found} = Exgit.ObjectStore.Memory.object_size(p.cache, sha) + end + test "large cap doesn't evict" do p = Promisor.new(%Stub{}, max_cache_bytes: 10_000) diff --git a/test/exgit/pack/stream_parser_adversarial_test.exs b/test/exgit/pack/stream_parser_adversarial_test.exs index 2c26ac3..ea0191f 100644 --- a/test/exgit/pack/stream_parser_adversarial_test.exs +++ b/test/exgit/pack/stream_parser_adversarial_test.exs @@ -256,7 +256,7 @@ defmodule Exgit.Pack.StreamParserAdversarialTest do pack = Writer.build([blob]) # Flip one bit only in the trailing 20-byte checksum (last byte). last = byte_size(pack) - 1 - <> = pack + <> = pack corrupted = <> assert {:error, :checksum_mismatch} = parse_all(corrupted) end diff --git a/test/exgit/pkt_line/decoder_test.exs b/test/exgit/pkt_line/decoder_test.exs index 0e96d02..28eba61 100644 --- a/test/exgit/pkt_line/decoder_test.exs +++ b/test/exgit/pkt_line/decoder_test.exs @@ -84,13 +84,13 @@ defmodule Exgit.PktLine.DecoderTest do end test "malformed length returns error" do - assert {:error, {:malformed_length, "ZZZZ"}} = + assert {:error, {:malformed_pkt_line, "ZZZZ"}} = Decoder.feed(Decoder.new(), "ZZZZpayload") end test "length below header size returns error" do # 0003 claims 3 bytes — less than the 4-byte header. - assert {:error, {:malformed_length, "0003"}} = + assert {:error, {:malformed_pkt_line, "0003"}} = Decoder.feed(Decoder.new(), "0003") end end @@ -140,7 +140,7 @@ defmodule Exgit.PktLine.DecoderTest do defp chunk_and_feed(bytes, [size | rest], original, decoder, acc) do n = min(size, byte_size(bytes)) - <> = bytes + <> = bytes {:ok, decoder, pkts} = Decoder.feed(decoder, chunk) chunk_and_feed(tail, rest, original, decoder, Enum.reverse(pkts) ++ acc) end diff --git a/test/exgit/pkt_line_test.exs b/test/exgit/pkt_line_test.exs index 723744d..ddbb559 100644 --- a/test/exgit/pkt_line_test.exs +++ b/test/exgit/pkt_line_test.exs @@ -12,6 +12,20 @@ defmodule Exgit.PktLineTest do test "encodes an empty payload" do assert IO.iodata_to_binary(PktLine.encode("")) == "0004" end + + test "encodes a payload at exactly the 65516-byte max" do + payload = :binary.copy("a", 65_516) + # 65516 + 4 = 65520 = 0xFFF0, git's LARGE_PACKET_MAX. + assert IO.iodata_to_binary(PktLine.encode(payload)) == "FFF0" <> payload + end + + test "raises ArgumentError for a payload over the 65516-byte max" do + payload = :binary.copy("a", 65_517) + + assert_raise ArgumentError, ~r/65517 bytes; max is 65516/, fn -> + PktLine.encode(payload) + end + end end describe "special packets" do @@ -62,6 +76,40 @@ defmodule Exgit.PktLineTest do end end + describe "malformed input" do + test "non-hex length header returns an error tuple" do + assert {:error, {:malformed_pkt_line, "ZZZZgarbage"}} = + PktLine.decode_all("ZZZZgarbage") + end + + test "truncated header returns an error tuple" do + assert {:error, {:malformed_pkt_line, "00"}} = PktLine.decode_all("00") + end + + test "truncated payload returns an error tuple" do + # Header claims 9 bytes total but only 3 payload bytes follow. + assert {:error, {:malformed_pkt_line, "0009hel"}} = PktLine.decode_all("0009hel") + end + + test "a malformed tail rejects the whole stream" do + encoded = IO.iodata_to_binary([PktLine.encode("good\n"), "ZZZZ"]) + assert {:error, {:malformed_pkt_line, "ZZZZ"}} = PktLine.decode_all(encoded) + end + + test "decode_stream yields packets, then the error token, then halts" do + encoded = IO.iodata_to_binary([PktLine.encode("good\n"), "ZZZZ"]) + + assert [{:data, "good\n"}, {:error, {:malformed_pkt_line, "ZZZZ"}}] = + Enum.to_list(PktLine.decode_stream(encoded)) + end + + test "error snippet is capped at 40 bytes" do + garbage = "Z" <> :binary.copy("y", 100) + assert {:error, {:malformed_pkt_line, snippet}} = PktLine.decode_all(garbage) + assert byte_size(snippet) == 40 + end + end + describe "round-trip" do property "encode then decode preserves payload" do check all( diff --git a/test/exgit/real_git_integration_test.exs b/test/exgit/real_git_integration_test.exs index c4219b6..f414e7b 100644 --- a/test/exgit/real_git_integration_test.exs +++ b/test/exgit/real_git_integration_test.exs @@ -68,7 +68,7 @@ defmodule Exgit.RealGitIntegrationTest do inflated = :zlib.uncompress(raw) {pos, 1} = :binary.match(inflated, <<0>>) - <<_header::binary-size(pos), 0, content::binary>> = inflated + <<_header::binary-size(^pos), 0, content::binary>> = inflated {:ok, tree} = Tree.decode(content) assert IO.iodata_to_binary(Tree.encode(tree)) == content diff --git a/test/exgit/repo_handle_test.exs b/test/exgit/repo_handle_test.exs index a3fcc3a..ee66661 100644 --- a/test/exgit/repo_handle_test.exs +++ b/test/exgit/repo_handle_test.exs @@ -38,9 +38,9 @@ defmodule Exgit.RepoHandleTest do end describe "lifecycle" do - test "start_link + get", %{repo: repo} do + test "start_link + fetch!", %{repo: repo} do {:ok, handle} = RepoHandle.start_link(repo) - assert %Repository{} = RepoHandle.get(handle) + assert %Repository{} = RepoHandle.fetch!(handle) RepoHandle.stop(handle) end @@ -50,7 +50,7 @@ defmodule Exgit.RepoHandleTest do RepoHandle.stop(handle) end - test "get/1 on dead handle raises" do + test "fetch!/1 on dead handle raises" do # Spawn a handle and kill it. {:ok, handle} = RepoHandle.start_link(%Repository{ @@ -63,7 +63,7 @@ defmodule Exgit.RepoHandleTest do RepoHandle.stop(handle) refute Process.alive?(handle) - assert_raise ArgumentError, fn -> RepoHandle.get(handle) end + assert_raise ArgumentError, fn -> RepoHandle.fetch!(handle) end end test "fetch/1 on dead handle returns {:error, _}" do @@ -83,8 +83,8 @@ defmodule Exgit.RepoHandleTest do name = :"test_named_handle_#{System.unique_integer([:positive])}" {:ok, pid} = RepoHandle.start_link(repo, name: name) - assert %Repository{} = RepoHandle.get(name) - assert RepoHandle.get(name) == RepoHandle.get(pid) + assert %Repository{} = RepoHandle.fetch!(name) + assert RepoHandle.fetch!(name) == RepoHandle.fetch!(pid) RepoHandle.stop(name) end @@ -117,7 +117,7 @@ defmodule Exgit.RepoHandleTest do %{r | object_store: store} end) - new_repo = RepoHandle.get(handle) + new_repo = RepoHandle.fetch!(handle) # Store has 5 objects now (blob + tree + commit + new blob = 4, plus... wait) # Originally 3 objects (blob, tree, commit). After update, 4. assert map_size(new_repo.object_store.objects) == 4 @@ -134,18 +134,18 @@ defmodule Exgit.RepoHandleTest do {:ok, %{r | object_store: store}} end) - assert map_size(RepoHandle.get(handle).object_store.objects) == 4 + assert map_size(RepoHandle.fetch!(handle).object_store.objects) == 4 RepoHandle.stop(handle) end test "fun returning {:error, _} leaves handle unchanged", %{repo: repo} do {:ok, handle} = RepoHandle.start_link(repo) - snapshot_before = RepoHandle.get(handle) + snapshot_before = RepoHandle.fetch!(handle) result = RepoHandle.update(handle, fn _r -> {:error, :test_error} end) assert result == {:error, :test_error} - assert RepoHandle.get(handle) == snapshot_before + assert RepoHandle.fetch!(handle) == snapshot_before RepoHandle.stop(handle) end @@ -170,7 +170,7 @@ defmodule Exgit.RepoHandleTest do Enum.each(tasks, &Task.await/1) # Initial 3 (commit+tree+blob) + n new blobs. - assert map_size(RepoHandle.get(handle).object_store.objects) == 3 + n_tasks + assert map_size(RepoHandle.fetch!(handle).object_store.objects) == 3 + n_tasks RepoHandle.stop(handle) end end @@ -185,13 +185,13 @@ defmodule Exgit.RepoHandleTest do :ok = RepoHandle.put(handle, new_repo) - stored = RepoHandle.get(handle) + stored = RepoHandle.fetch!(handle) assert map_size(stored.object_store.objects) == 4 RepoHandle.stop(handle) end end - describe "fetch_once/3 dedup" do + describe "fetch_once/4 dedup" do test "serializes concurrent fetches for the same key", %{repo: repo} do {:ok, handle} = RepoHandle.start_link(repo) @@ -267,6 +267,44 @@ defmodule Exgit.RepoHandleTest do RepoHandle.stop(handle) end + test "fetch fn that exits errors promptly instead of hanging", %{repo: repo} do + {:ok, handle} = RepoHandle.start_link(repo) + + # The 1s call timeout makes the test fail fast if the caller + # would otherwise hang until the default 300s timeout. + assert {:error, {:fetch_crashed, _}} = + RepoHandle.fetch_once(handle, :exit_key, fn _repo -> exit(:boom) end, 1_000) + + # The in_flight entry is cleared: a subsequent fetch_once for + # the SAME key re-runs the fetch and succeeds. + assert {:ok, %Repository{}} = + RepoHandle.fetch_once(handle, :exit_key, fn r -> {:ok, r} end, 1_000) + + RepoHandle.stop(handle) + end + + test "killed fetch task errors promptly instead of hanging", %{repo: repo} do + {:ok, handle} = RepoHandle.start_link(repo) + + # A brutal kill can't be caught inside the task, so this + # exercises the monitor path: the handle sees :DOWN and + # replies to waiters instead of leaking the in_flight entry. + kill_fn = fn _repo -> Process.exit(self(), :kill) end + + # Reason is :killed, or :noproc when the task dies before the + # handle's monitor attaches — both go through the :DOWN path. + assert {:error, {:fetch_crashed, reason}} = + RepoHandle.fetch_once(handle, :kill_key, kill_fn, 1_000) + + assert reason in [:killed, :noproc] + + # The entry is cleared: the same key fetches again and succeeds. + assert {:ok, %Repository{}} = + RepoHandle.fetch_once(handle, :kill_key, fn r -> {:ok, r} end, 1_000) + + RepoHandle.stop(handle) + end + test "handle stays responsive to reads during a slow fetch", %{repo: repo} do {:ok, handle} = RepoHandle.start_link(repo) @@ -284,7 +322,7 @@ defmodule Exgit.RepoHandleTest do start = System.monotonic_time() for _ <- 1..100 do - _ = RepoHandle.get(handle) + _ = RepoHandle.fetch!(handle) end us = @@ -299,15 +337,15 @@ defmodule Exgit.RepoHandleTest do end describe "read performance" do - test "get/1 does not send a message to the handle", %{repo: repo} do + test "fetch!/1 does not send a message to the handle", %{repo: repo} do {:ok, handle} = RepoHandle.start_link(repo) # Capture the process's message queue length before and after - # many get/1 calls. If get/1 used GenServer.call, the handle - # would receive N :sys messages — but it's not the test pid's - # queue that grows, it's the handle's. Instead we check that - # get/1 returns fast even while the handle is busy in a - # slow update. + # many fetch!/1 calls. If fetch!/1 used GenServer.call, the + # handle would receive N :sys messages — but it's not the test + # pid's queue that grows, it's the handle's. Instead we check + # that fetch!/1 returns fast even while the handle is busy in + # a slow update. slow_update_task = Task.async(fn -> RepoHandle.update( @@ -326,7 +364,7 @@ defmodule Exgit.RepoHandleTest do start = System.monotonic_time() for _ <- 1..100 do - _ = RepoHandle.get(handle) + _ = RepoHandle.fetch!(handle) end reads_us = @@ -336,7 +374,7 @@ defmodule Exgit.RepoHandleTest do # If reads went through the GenServer they'd block behind the # 200ms sleep and this would be ~200ms. assert reads_us < 10_000, - "100 get/1 calls took #{reads_us}µs — likely blocked on GenServer, indicating a regression" + "100 fetch!/1 calls took #{reads_us}µs — likely blocked on GenServer, indicating a regression" Task.await(slow_update_task) RepoHandle.stop(handle) diff --git a/test/exgit/repo_registry_test.exs b/test/exgit/repo_registry_test.exs index 469f0cf..acd1e2d 100644 --- a/test/exgit/repo_registry_test.exs +++ b/test/exgit/repo_registry_test.exs @@ -161,7 +161,7 @@ defmodule Exgit.RepoRegistryTest do assert {:ok, ^handle} = RepoRegistry.lookup(url) # Can fetch through the handle. - assert %Exgit.Repository{} = RepoHandle.get(handle) + assert %Exgit.Repository{} = RepoHandle.fetch!(handle) RepoHandle.stop(handle) end diff --git a/test/exgit/repository_memory_report_test.exs b/test/exgit/repository_memory_report_test.exs index 4483cc6..71da39f 100644 --- a/test/exgit/repository_memory_report_test.exs +++ b/test/exgit/repository_memory_report_test.exs @@ -2,9 +2,10 @@ defmodule Exgit.RepositoryMemoryReportTest do @moduledoc """ Tests `Exgit.Repository.memory_report/1` across the object-store backends we actually ship: `Memory` and `Promisor`. `Disk` and - user-defined stores get a degraded report (placeholders); we - assert that the shape is consistent across all backends so - callers can depend on the keys existing. + user-defined stores are not introspected, so their counts and + `:cache_bytes` are `:unknown`; we assert that the shape is + consistent across all backends so callers can depend on the keys + existing. """ use ExUnit.Case, async: true @@ -112,6 +113,36 @@ defmodule Exgit.RepositoryMemoryReportTest do end end + describe "on a Disk-backed repo" do + # Disk is not introspected: counts and cache_bytes must be + # :unknown, never a fake 0 that monitoring could mistake for + # an empty repo. + setup do + store = ObjectStore.Disk.new("/nonexistent/objects") + repo = Repository.new(store, Exgit.RefStore.Memory.new()) + {:ok, repo: repo} + end + + test "reports :unknown counts and cache_bytes", %{repo: repo} do + report = Repository.memory_report(repo) + + assert report.object_count == :unknown + assert report.cache_bytes == :unknown + assert report.commit_count == :unknown + assert report.tree_count == :unknown + assert report.blob_count == :unknown + assert report.tag_count == :unknown + end + + test "still reports backend, mode, and max_cache_bytes", %{repo: repo} do + report = Repository.memory_report(repo) + + assert report.backend == ObjectStore.Disk + assert report.mode == :eager + assert report.max_cache_bytes == :infinity + end + end + describe "shape invariants" do test "report always has the same keys regardless of backend" do # Memory-backed @@ -127,7 +158,12 @@ defmodule Exgit.RepositoryMemoryReportTest do promisor_keys = empty_promisor |> Repository.memory_report() |> Map.keys() |> Enum.sort() + # Disk-backed (not introspected) + disk = Repository.new(ObjectStore.Disk.new("/nonexistent"), Exgit.RefStore.Memory.new()) + disk_keys = disk |> Repository.memory_report() |> Map.keys() |> Enum.sort() + assert memory_keys == promisor_keys + assert memory_keys == disk_keys end end end diff --git a/test/exgit/security/decoder_fuzz_test.exs b/test/exgit/security/decoder_fuzz_test.exs index e5e694f..0341cd2 100644 --- a/test/exgit/security/decoder_fuzz_test.exs +++ b/test/exgit/security/decoder_fuzz_test.exs @@ -66,8 +66,7 @@ defmodule Exgit.Security.DecoderFuzzTest do describe "Blob.decode/1 never raises" do property "on random bytes" do check all(bytes <- random_bytes(), max_runs: @max_runs) do - result = Blob.decode(bytes) - assert match?({:ok, %Blob{}}, result) or match?({:error, _}, result) + assert {:ok, %Blob{}} = Blob.decode(bytes) end end end diff --git a/test/exgit/transport/http_refs_cap_test.exs b/test/exgit/transport/http_refs_cap_test.exs new file mode 100644 index 0000000..be73e88 --- /dev/null +++ b/test/exgit/transport/http_refs_cap_test.exs @@ -0,0 +1,114 @@ +defmodule Exgit.Transport.HttpRefsCapTest do + @moduledoc """ + Exercises the `:max_refs` backstop on `Exgit.Transport.HTTP.ls_refs/2` + against a server that streams the ls-refs response one pkt-line per + chunk — the shape a hostile server would use to grow the client's + ref list without bound. + + The cap must surface as `{:error, {:too_many_refs, cap}}`, not as a + `{:truncated, _}` decode artifact: the client halts the body early + when the cap trips, which legitimately leaves partial bytes in the + pkt-line decoder, and the accumulated domain error has to win. + + Uses a raw `:gen_tcp` server (like `Exgit.Transport.HttpTest`) + rather than Bypass: the cap test hangs up mid-response by design, + which Bypass's plug monitoring reports as a `:shutdown` failure. + """ + + use ExUnit.Case, async: true + + alias Exgit.PktLine + alias Exgit.Transport.HTTP + + describe "ls_refs/2 with :max_refs" do + test "a server streaming 100 refs past a cap of 50 aborts with {:too_many_refs, 50}" do + serve_chunked_ls_refs(100, fn port -> + t = HTTP.new("http://127.0.0.1:#{port}", max_refs: 50) + + assert HTTP.ls_refs(t) == {:error, {:too_many_refs, 50}} + end) + end + + test "exactly cap refs succeeds with all refs present (boundary)" do + serve_chunked_ls_refs(50, fn port -> + t = HTTP.new("http://127.0.0.1:#{port}", max_refs: 50) + + assert {:ok, refs, _meta} = HTTP.ls_refs(t) + assert length(refs) == 50 + assert Enum.map(refs, &elem(&1, 0)) == for(i <- 1..50, do: ref_name(i)) + assert {ref_name(50), sha(50)} == List.last(refs) + end) + end + + test "the default cap is the documented 1_000_000 backstop" do + assert HTTP.new("http://localhost").max_refs == 1_000_000 + end + end + + # Serve an ls-refs response over chunked transfer encoding, one + # pkt-line per chunk, so the cap trips while bytes are still + # arriving rather than on a fully-buffered body. Once the client's + # cap trips it hangs up; send errors past that point are expected. + defp serve_chunked_ls_refs(ref_count, fun) do + {:ok, listener} = :gen_tcp.listen(0, [:binary, active: false, reuseaddr: true, packet: :raw]) + {:ok, port} = :inet.port(listener) + + task = + Task.async(fn -> + {:ok, sock} = :gen_tcp.accept(listener, 5_000) + _ = recv_request(sock) + + head = + "HTTP/1.1 200 OK\r\n" <> + "Content-Type: application/x-git-upload-pack-result\r\n" <> + "Transfer-Encoding: chunked\r\nConnection: close\r\n\r\n" + + :ok = :gen_tcp.send(sock, head) + + _ = + 1..ref_count + |> Enum.map(&ref_line/1) + |> Enum.concat([PktLine.flush(), :last]) + |> Enum.reduce_while(:ok, fn piece, :ok -> + data = if piece == :last, do: "0\r\n\r\n", else: chunk(piece) + + case :gen_tcp.send(sock, data) do + :ok -> {:cont, :ok} + {:error, _closed_by_capped_client} -> {:halt, :ok} + end + end) + + :gen_tcp.close(sock) + end) + + try do + fun.(port) + after + Task.await(task, 5_000) + :gen_tcp.close(listener) + end + end + + defp chunk(iodata) do + data = IO.iodata_to_binary(iodata) + Integer.to_string(byte_size(data), 16) <> "\r\n" <> data <> "\r\n" + end + + defp recv_request(sock) do + case :gen_tcp.recv(sock, 0, 2_000) do + {:ok, data} -> + if String.contains?(data, "\r\n\r\n"), do: data, else: data <> recv_request(sock) + + {:error, _} -> + <<>> + end + end + + defp ref_line(i) do + PktLine.encode("#{Base.encode16(sha(i), case: :lower)} #{ref_name(i)}\n") + end + + defp ref_name(i), do: "refs/heads/branch-#{String.pad_leading(Integer.to_string(i), 3, "0")}" + + defp sha(i), do: :binary.copy(<>, 20) +end diff --git a/test/exgit/transport/http_streaming_test.exs b/test/exgit/transport/http_streaming_test.exs index 0521012..b44f020 100644 --- a/test/exgit/transport/http_streaming_test.exs +++ b/test/exgit/transport/http_streaming_test.exs @@ -54,6 +54,26 @@ defmodule Exgit.Transport.HttpStreamingTest do :gen_tcp.close(listener) end + test "channel 3 error wins over trailing truncated bytes" do + body = + IO.iodata_to_binary([ + PktLine.encode("packfile\n"), + PktLine.encode(<<3, "fatal: boom">>), + # Trailing bytes of an incomplete pkt-line. Halting on the + # channel-3 error leaves these undecoded in the decoder — + # that truncation artifact must not mask the real error. + "00ff" + ]) + + {listener, port} = run_fetch_server(body) + t = HTTP.new("http://127.0.0.1:#{port}") + + assert {:error, {:server_error, "fatal: boom"}} = + HTTP.fetch(t, [<<0::160>>], sideband: true) + + :gen_tcp.close(listener) + end + @tag :memory test "process heap stays bounded relative to pack size (regression guard)" do # 8 MB of pack bytes split into 4 KB sideband packets. Old non-streaming diff --git a/test/exgit/transport/http_telemetry_redaction_test.exs b/test/exgit/transport/http_telemetry_redaction_test.exs new file mode 100644 index 0000000..1bd22fc --- /dev/null +++ b/test/exgit/transport/http_telemetry_redaction_test.exs @@ -0,0 +1,165 @@ +defmodule Exgit.Transport.HttpTelemetryRedactionTest do + use ExUnit.Case, async: true + + alias Exgit.PktLine + alias Exgit.Transport.HTTP + + # A token embedded in a remote URL must never reach telemetry metadata. + # `Exgit.Telemetry.span/3` (via `:telemetry.span/3`) emits a `[..., :start]` + # event carrying the metadata BEFORE it runs the network call — so we can + # capture the URL metadata without a live git server: point the transport at + # a dead port, let the request fail, and read the start event. + @token "SECRET_TOKEN_do_not_leak_9f3a" + + # The telemetry handler is global and other async tests emit the same spans, + # so filter on a unique per-test path marker (which survives redaction — only + # the userinfo is stripped) to capture exactly this test's event. + defp capture_start_meta(event, marker, run) do + handler = "redact-#{marker}" + test_pid = self() + + :telemetry.attach( + handler, + event ++ [:start], + fn _e, _measurements, meta, _cfg -> + if is_binary(meta[:url]) and String.contains?(meta.url, marker) do + send(test_pid, {:tele_meta, meta}) + end + end, + nil + ) + + try do + _ = run.() + assert_receive {:tele_meta, meta}, 2_000 + meta + after + :telemetry.detach(handler) + end + end + + defp marker, do: "probe#{System.unique_integer([:positive])}" + + test "ls_refs telemetry redacts a token embedded in the URL" do + m = marker() + t = HTTP.new("https://x-access-token:#{@token}@127.0.0.1:1/#{m}.git") + meta = capture_start_meta([:exgit, :transport, :ls_refs], m, fn -> HTTP.ls_refs(t) end) + + refute meta.url =~ @token, "token leaked into ls_refs telemetry: #{meta.url}" + assert meta.url =~ "***" + end + + test "fetch telemetry redacts a token embedded in the URL" do + m = marker() + t = HTTP.new("https://x-access-token:#{@token}@127.0.0.1:1/#{m}.git") + + meta = + capture_start_meta([:exgit, :transport, :fetch], m, fn -> + # `wants` are raw 20-byte shas; the call fails at the dead port, but + # the span's :start event (with the URL metadata) fires first. + HTTP.fetch(t, [<<0::160>>], []) + end) + + refute meta.url =~ @token, "token leaked into fetch telemetry: #{meta.url}" + assert meta.url =~ "***" + end + + test "push telemetry redacts a token embedded in the URL" do + m = marker() + t = HTTP.new("https://x-access-token:#{@token}@127.0.0.1:1/#{m}.git") + update = {"refs/heads/main", nil, <<0::160>>} + + meta = + capture_start_meta([:exgit, :transport, :push], m, fn -> + HTTP.push(t, [update], "PACK") + end) + + refute meta.url =~ @token, "token leaked into push telemetry: #{meta.url}" + assert meta.url =~ "***" + end + + test "ref_rejected security telemetry redacts a token embedded in the source URL" do + # `[:exgit, :security, :ref_rejected]` fires while parsing an ls-refs + # response, so the dead-port trick isn't enough — serve one hostile + # ref name from a local socket and let `keep_ref?` reject it. + m = marker() + hostile_ref = "refs/heads/../../escape" + sha = Base.encode16(:binary.copy(<<1>>, 20), case: :lower) + body = IO.iodata_to_binary([PktLine.encode("#{sha} #{hostile_ref}\n"), PktLine.flush()]) + + handler = "redact-#{m}" + test_pid = self() + + :telemetry.attach( + handler, + [:exgit, :security, :ref_rejected], + fn _e, _measurements, meta, _cfg -> + if is_binary(meta[:source]) and String.contains?(meta.source, m) do + send(test_pid, {:sec_meta, meta}) + end + end, + nil + ) + + try do + serve_once(body, fn port -> + t = HTTP.new("http://x-access-token:#{@token}@127.0.0.1:#{port}/#{m}.git") + _ = HTTP.ls_refs(t) + end) + + assert_receive {:sec_meta, meta}, 2_000 + assert meta.ref == hostile_ref + refute meta.source =~ @token, "token leaked into ref_rejected telemetry: #{meta.source}" + assert meta.source =~ "***" + after + :telemetry.detach(handler) + end + end + + test "a URL without credentials passes through unchanged" do + m = marker() + t = HTTP.new("https://127.0.0.1:1/#{m}.git") + meta = capture_start_meta([:exgit, :transport, :ls_refs], m, fn -> HTTP.ls_refs(t) end) + + assert meta.url == "https://127.0.0.1:1/#{m}.git" + refute meta.url =~ "***" + end + + # One-shot HTTP server (same shape as `warm_server` in HttpTest): + # accept a single request, reply 200 with `body`, close. + defp serve_once(body, fun) do + {:ok, listener} = :gen_tcp.listen(0, [:binary, active: false, reuseaddr: true, packet: :raw]) + {:ok, port} = :inet.port(listener) + + task = + Task.async(fn -> + {:ok, sock} = :gen_tcp.accept(listener, 5_000) + _ = recv_request(sock) + + resp = + "HTTP/1.1 200 OK\r\n" <> + "Content-Type: application/x-git-upload-pack-result\r\n" <> + "Content-Length: #{byte_size(body)}\r\nConnection: close\r\n\r\n" <> body + + :ok = :gen_tcp.send(sock, resp) + :gen_tcp.close(sock) + end) + + try do + fun.(port) + after + Task.await(task, 5_000) + :gen_tcp.close(listener) + end + end + + defp recv_request(sock) do + case :gen_tcp.recv(sock, 0, 2_000) do + {:ok, data} -> + if String.contains?(data, "\r\n\r\n"), do: data, else: data <> recv_request(sock) + + {:error, _} -> + <<>> + end + end +end diff --git a/test/exgit/transport/http_test.exs b/test/exgit/transport/http_test.exs index 860b8aa..e665401 100644 --- a/test/exgit/transport/http_test.exs +++ b/test/exgit/transport/http_test.exs @@ -67,6 +67,62 @@ defmodule Exgit.Transport.HttpTest do end end + describe "hostile server bytes surface as error values" do + test "capabilities/1 returns an error tuple on garbage bytes" do + garbage = "ZZZZ" <> :binary.copy(<<0xFF>>, 32) + + warm_server(garbage, fn port -> + t = HTTP.new("http://127.0.0.1:#{port}") + + assert {:error, {:malformed_response, {:malformed_pkt_line, _}}} = + HTTP.capabilities(t) + end) + end + + test "push/4 returns an error tuple when the report-status body is garbage" do + warm_server("ZZZZnot-a-pkt-line", fn port -> + t = HTTP.new("http://127.0.0.1:#{port}") + update = {"refs/heads/main", nil, :binary.copy(<<1>>, 20)} + + assert {:error, {:malformed_response, {:malformed_pkt_line, _}}} = + HTTP.push(t, [update], "PACK") + end) + end + end + + describe "ls-refs ref cap (:max_refs)" do + test "exceeding the cap surfaces as {:error, {:too_many_refs, cap}}" do + lines = + for i <- 1..5 do + sha = Base.encode16(:binary.copy(<>, 20), case: :lower) + PktLine.encode("#{sha} refs/heads/b#{i}\n") + end + + body = IO.iodata_to_binary([lines, PktLine.flush()]) + + warm_server(body, fn port -> + t = HTTP.new("http://127.0.0.1:#{port}", max_refs: 3) + assert {:error, {:too_many_refs, 3}} = HTTP.ls_refs(t) + end) + end + + test "under the cap the same response lists all refs" do + lines = + for i <- 1..5 do + sha = Base.encode16(:binary.copy(<>, 20), case: :lower) + PktLine.encode("#{sha} refs/heads/b#{i}\n") + end + + body = IO.iodata_to_binary([lines, PktLine.flush()]) + + warm_server(body, fn port -> + t = HTTP.new("http://127.0.0.1:#{port}", max_refs: 5) + assert {:ok, refs, _meta} = HTTP.ls_refs(t) + assert length(refs) == 5 + end) + end + end + defp warm_server(body, fun) do {:ok, listener} = :gen_tcp.listen(0, [:binary, active: false, reuseaddr: true, packet: :raw]) {:ok, port} = :inet.port(listener) diff --git a/test/exgit/write_path_hardening_test.exs b/test/exgit/write_path_hardening_test.exs index 4256cfe..a5e0d9f 100644 --- a/test/exgit/write_path_hardening_test.exs +++ b/test/exgit/write_path_hardening_test.exs @@ -383,7 +383,7 @@ defmodule Exgit.WritePathHardeningTest do # Format: " \0". defp strip_header(bytes) do {pos, 1} = :binary.match(bytes, <<0>>) - <<_::binary-size(pos), 0, content::binary>> = bytes + <<_::binary-size(^pos), 0, content::binary>> = bytes content end end diff --git a/test/support/pack_builder.ex b/test/support/pack_builder.ex index c3b3a5b..292bb5b 100644 --- a/test/support/pack_builder.ex +++ b/test/support/pack_builder.ex @@ -69,7 +69,7 @@ defmodule Exgit.Test.PackBuilder do defp chunk_inserts(data) do take = min(127, byte_size(data)) - <> = data + <> = data [<>, chunk | chunk_inserts(rest)] end diff --git a/test/test_helper.exs b/test/test_helper.exs index c3f1f2b..787ffd7 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -38,7 +38,7 @@ exclude = if Exgit.Test.CloudflareArtifacts.api_available?() do exclude else - [{:cloudflare_api, true} | exclude] + [{:cloudflare_api, true}, {:cloudflare_perf, true} | exclude] end # Live-network tiers are excluded by default; opt in via @@ -50,6 +50,7 @@ exclude = {:integration, true}, {:cloudflare, true}, {:cloudflare_api, true}, + {:cloudflare_perf, true}, {:github_private, true}, {:github_private_write, true} | exclude