From aa1abb8ccb622cb07cf3ac64383694f877ac49d3 Mon Sep 17 00:00:00 2001 From: Ivar Vong Date: Wed, 1 Jul 2026 09:21:19 -0400 Subject: [PATCH 01/13] chore: migrate to the Elixir 1.20 toolchain Adopt 1.20's explicit pin-operator requirement for computed bitstring sizes (`binary-size(^n)`) across the object, pack, pkt-line, and workspace modules, and bump .tool-versions to 1.20.0-rc.3. The mix.exs floor stays at `~> 1.17`: pinned bitstring sizes have been valid since 1.14, so the minimum-supported version is unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01YVcfxkfN7A7KYLmNYt9UcH --- .tool-versions | 2 +- lib/exgit/index.ex | 6 +++--- lib/exgit/object/commit.ex | 2 +- lib/exgit/object/tag.ex | 2 +- lib/exgit/object/tree.ex | 2 +- lib/exgit/pack/delta.ex | 2 +- lib/exgit/pack/index.ex | 12 ++++++------ lib/exgit/pkt_line.ex | 2 +- lib/exgit/pkt_line/decoder.ex | 2 +- lib/exgit/workspace/vfs.ex | 2 +- test/exgit/object/tree_roundtrip_test.exs | 2 +- test/exgit/object_store/disk_pack_lookup_test.exs | 2 +- test/exgit/object_store/disk_pread_test.exs | 2 +- test/exgit/pack/stream_parser_adversarial_test.exs | 2 +- test/exgit/pkt_line/decoder_test.exs | 2 +- test/exgit/real_git_integration_test.exs | 2 +- test/exgit/security/decoder_fuzz_test.exs | 3 +-- test/exgit/write_path_hardening_test.exs | 2 +- test/support/pack_builder.ex | 2 +- 19 files changed, 26 insertions(+), 27 deletions(-) diff --git a/.tool-versions b/.tool-versions index 9d1de14..c77aa85 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,2 @@ -elixir 1.19.5-otp-28 erlang 28.4 +elixir 1.20.0-rc.3-otp-28 diff --git a/lib/exgit/index.ex b/lib/exgit/index.ex index 5dce2be..b1a74e4 100644 --- a/lib/exgit/index.ex +++ b/lib/exgit/index.ex @@ -127,7 +127,7 @@ defmodule Exgit.Index do # remote-controlled. defp valid_checksum?(data) when byte_size(data) >= 20 do content_size = byte_size(data) - 20 - <> = data + <> = data :crypto.hash(:sha, content) == checksum end @@ -193,7 +193,7 @@ defmodule Exgit.Index do # the real length is unknown and we must read until the next NUL. defp read_name(data, len) when len < 0xFFF do case data do - <> -> {:ok, name, rest} + <> -> {:ok, name, rest} _ -> {:error, :truncated_name} end end @@ -222,7 +222,7 @@ defmodule Exgit.Index do _ = pad if byte_size(data) >= pad_remaining do - <<_::binary-size(pad_remaining), rest::binary>> = data + <<_::binary-size(^pad_remaining), rest::binary>> = data {:ok, rest} else {:error, :truncated_padding} diff --git a/lib/exgit/object/commit.ex b/lib/exgit/object/commit.ex index d821427..f977b66 100644 --- a/lib/exgit/object/commit.ex +++ b/lib/exgit/object/commit.ex @@ -92,7 +92,7 @@ defmodule Exgit.Object.Commit do def decode(bytes) when is_binary(bytes) do case :binary.match(bytes, "\n\n") do {pos, 2} -> - <> = bytes + <> = bytes case parse_headers(raw_headers) do {:ok, headers} -> diff --git a/lib/exgit/object/tag.ex b/lib/exgit/object/tag.ex index e48b21a..b6254be 100644 --- a/lib/exgit/object/tag.ex +++ b/lib/exgit/object/tag.ex @@ -59,7 +59,7 @@ defmodule Exgit.Object.Tag do def decode(bytes) when is_binary(bytes) do case :binary.match(bytes, "\n\n") do {pos, 2} -> - <> = bytes + <> = bytes case parse_headers(raw_headers) do {:ok, headers} -> diff --git a/lib/exgit/object/tree.ex b/lib/exgit/object/tree.ex index 7a8087f..b89e1bb 100644 --- a/lib/exgit/object/tree.ex +++ b/lib/exgit/object/tree.ex @@ -180,7 +180,7 @@ defmodule Exgit.Object.Tree do defp take_until(data, byte) do case :binary.match(data, <>) do {pos, 1} -> - <> = data + <> = data {:ok, before, rest} :nomatch -> diff --git a/lib/exgit/pack/delta.ex b/lib/exgit/pack/delta.ex index b6aae76..57098b7 100644 --- a/lib/exgit/pack/delta.ex +++ b/lib/exgit/pack/delta.ex @@ -64,7 +64,7 @@ defmodule Exgit.Pack.Delta do {:error, :insert_exceeds_result_size} true -> - <> = rest + <> = rest apply_instructions(more, base, remaining - byte, [data | acc]) end end diff --git a/lib/exgit/pack/index.ex b/lib/exgit/pack/index.ex index dc3ecd3..edcf23b 100644 --- a/lib/exgit/pack/index.ex +++ b/lib/exgit/pack/index.ex @@ -53,8 +53,8 @@ defmodule Exgit.Pack.Index do crc_size = total_objects * 4 offset_size = total_objects * 4 - <> = rest + <> = rest {large_offset_table, rest} = extract_large_offsets(offsets, total_objects, rest) @@ -63,7 +63,7 @@ defmodule Exgit.Pack.Index do # Verify index checksum idx_body_size = byte_size(data) - 20 - <> = data + <> = data if :crypto.hash(:sha, idx_body) != claimed_checksum do {:error, :index_checksum_mismatch} @@ -118,8 +118,8 @@ defmodule Exgit.Pack.Index do crc_size = total * 4 offset_size = total * 4 - <> = rest + <> = rest # Large-offset table comes next in the file. We keep the whole tail # as a subbinary; `offset_at/2` only slices into it when a lookup @@ -237,7 +237,7 @@ defmodule Exgit.Pack.Index do large_count -> large_size = large_count * 8 - <> = rest + <> = rest {large, rest} end end diff --git a/lib/exgit/pkt_line.ex b/lib/exgit/pkt_line.ex index a0cea95..bcf3b47 100644 --- a/lib/exgit/pkt_line.ex +++ b/lib/exgit/pkt_line.ex @@ -55,7 +55,7 @@ defmodule Exgit.PktLine do with {len, ""} <- Integer.parse(hex_len, 16), true <- len >= 4, payload_len = len - 4, - <> <- rest do + <> <- rest do {{:data, payload}, rest} else _ -> diff --git a/lib/exgit/pkt_line/decoder.ex b/lib/exgit/pkt_line/decoder.ex index 270a042..63a1a48 100644 --- a/lib/exgit/pkt_line/decoder.ex +++ b/lib/exgit/pkt_line/decoder.ex @@ -68,7 +68,7 @@ defmodule Exgit.PktLine.Decoder do payload_len = len - 4 case rest do - <> -> + <> -> drain(tail, [{:data, payload} | acc]) # Payload not fully arrived — preserve the WHOLE pkt-line diff --git a/lib/exgit/workspace/vfs.ex b/lib/exgit/workspace/vfs.ex index 122acc5..b32fab2 100644 --- a/lib/exgit/workspace/vfs.ex +++ b/lib/exgit/workspace/vfs.ex @@ -279,7 +279,7 @@ if Code.ensure_loaded?(VFS.Mountable) do {rest, ""} rest -> - <> = rest + <> = rest {chunk, more} end) end diff --git a/test/exgit/object/tree_roundtrip_test.exs b/test/exgit/object/tree_roundtrip_test.exs index 3af089c..ad8822b 100644 --- a/test/exgit/object/tree_roundtrip_test.exs +++ b/test/exgit/object/tree_roundtrip_test.exs @@ -80,7 +80,7 @@ defmodule Exgit.Object.TreeRoundtripTest do # Strip the "tree \0" header. {pos, 1} = :binary.match(inflated, <<0>>) - <<_header::binary-size(pos), 0, content::binary>> = inflated + <<_header::binary-size(^pos), 0, content::binary>> = inflated {:ok, tree} = Exgit.Object.Tree.decode(content) assert IO.iodata_to_binary(Exgit.Object.Tree.encode(tree)) == content diff --git a/test/exgit/object_store/disk_pack_lookup_test.exs b/test/exgit/object_store/disk_pack_lookup_test.exs index a6a0cb0..88b6922 100644 --- a/test/exgit/object_store/disk_pack_lookup_test.exs +++ b/test/exgit/object_store/disk_pack_lookup_test.exs @@ -116,7 +116,7 @@ defmodule Exgit.ObjectStore.DiskPackLookupTest do end defp describe_entry(pack, offset, blob) do - <<_::binary-size(offset), from_here::binary>> = pack + <<_::binary-size(^offset), from_here::binary>> = pack {_type_code, _size, after_header} = Exgit.Pack.Common.decode_type_size_varint(from_here) header_len = byte_size(from_here) - byte_size(after_header) diff --git a/test/exgit/object_store/disk_pread_test.exs b/test/exgit/object_store/disk_pread_test.exs index 781da3f..3c46b84 100644 --- a/test/exgit/object_store/disk_pread_test.exs +++ b/test/exgit/object_store/disk_pread_test.exs @@ -144,7 +144,7 @@ defmodule Exgit.ObjectStore.DiskPreadTest do end defp describe_entry(pack, offset, blob) do - <<_::binary-size(offset), from_here::binary>> = pack + <<_::binary-size(^offset), from_here::binary>> = pack {_type_code, _size, after_header} = Exgit.Pack.Common.decode_type_size_varint(from_here) header_len = byte_size(from_here) - byte_size(after_header) diff --git a/test/exgit/pack/stream_parser_adversarial_test.exs b/test/exgit/pack/stream_parser_adversarial_test.exs index 2c26ac3..ea0191f 100644 --- a/test/exgit/pack/stream_parser_adversarial_test.exs +++ b/test/exgit/pack/stream_parser_adversarial_test.exs @@ -256,7 +256,7 @@ defmodule Exgit.Pack.StreamParserAdversarialTest do pack = Writer.build([blob]) # Flip one bit only in the trailing 20-byte checksum (last byte). last = byte_size(pack) - 1 - <> = pack + <> = pack corrupted = <> assert {:error, :checksum_mismatch} = parse_all(corrupted) end diff --git a/test/exgit/pkt_line/decoder_test.exs b/test/exgit/pkt_line/decoder_test.exs index 0e96d02..ad7852a 100644 --- a/test/exgit/pkt_line/decoder_test.exs +++ b/test/exgit/pkt_line/decoder_test.exs @@ -140,7 +140,7 @@ defmodule Exgit.PktLine.DecoderTest do defp chunk_and_feed(bytes, [size | rest], original, decoder, acc) do n = min(size, byte_size(bytes)) - <> = bytes + <> = bytes {:ok, decoder, pkts} = Decoder.feed(decoder, chunk) chunk_and_feed(tail, rest, original, decoder, Enum.reverse(pkts) ++ acc) end diff --git a/test/exgit/real_git_integration_test.exs b/test/exgit/real_git_integration_test.exs index c4219b6..f414e7b 100644 --- a/test/exgit/real_git_integration_test.exs +++ b/test/exgit/real_git_integration_test.exs @@ -68,7 +68,7 @@ defmodule Exgit.RealGitIntegrationTest do inflated = :zlib.uncompress(raw) {pos, 1} = :binary.match(inflated, <<0>>) - <<_header::binary-size(pos), 0, content::binary>> = inflated + <<_header::binary-size(^pos), 0, content::binary>> = inflated {:ok, tree} = Tree.decode(content) assert IO.iodata_to_binary(Tree.encode(tree)) == content diff --git a/test/exgit/security/decoder_fuzz_test.exs b/test/exgit/security/decoder_fuzz_test.exs index e5e694f..0341cd2 100644 --- a/test/exgit/security/decoder_fuzz_test.exs +++ b/test/exgit/security/decoder_fuzz_test.exs @@ -66,8 +66,7 @@ defmodule Exgit.Security.DecoderFuzzTest do describe "Blob.decode/1 never raises" do property "on random bytes" do check all(bytes <- random_bytes(), max_runs: @max_runs) do - result = Blob.decode(bytes) - assert match?({:ok, %Blob{}}, result) or match?({:error, _}, result) + assert {:ok, %Blob{}} = Blob.decode(bytes) end end end diff --git a/test/exgit/write_path_hardening_test.exs b/test/exgit/write_path_hardening_test.exs index 4256cfe..a5e0d9f 100644 --- a/test/exgit/write_path_hardening_test.exs +++ b/test/exgit/write_path_hardening_test.exs @@ -383,7 +383,7 @@ defmodule Exgit.WritePathHardeningTest do # Format: " \0". defp strip_header(bytes) do {pos, 1} = :binary.match(bytes, <<0>>) - <<_::binary-size(pos), 0, content::binary>> = bytes + <<_::binary-size(^pos), 0, content::binary>> = bytes content end end diff --git a/test/support/pack_builder.ex b/test/support/pack_builder.ex index c3b3a5b..292bb5b 100644 --- a/test/support/pack_builder.ex +++ b/test/support/pack_builder.ex @@ -69,7 +69,7 @@ defmodule Exgit.Test.PackBuilder do defp chunk_inserts(data) do take = min(127, byte_size(data)) - <> = data + <> = data [<>, chunk | chunk_inserts(rest)] end From 3ae8aae464976568b2c5839868179986a70cc039 Mon Sep 17 00:00:00 2001 From: Ivar Vong Date: Wed, 1 Jul 2026 09:21:39 -0400 Subject: [PATCH 02/13] feat: size-aware reads via Exgit.FS.size/3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add `Exgit.FS.size/3` and the `Exgit.ObjectStore.object_size/2` protocol callback so callers can learn a blob's size WITHOUT materializing it — the guard an agent needs before pulling a large file into memory. Memory keeps a parallel `sha => size` index (O(1), no decompression); Disk inflates only the loose-object header; Promisor answers from cache or returns `{:error, :not_local}` without triggering a fetch. `FS.size/3` resolves the path (trees only, on a lazy clone) and never fetches the blob. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01YVcfxkfN7A7KYLmNYt9UcH --- lib/exgit/fs.ex | 38 ++++++ lib/exgit/object_store.ex | 12 ++ lib/exgit/object_store/disk.ex | 78 +++++++++++- lib/exgit/object_store/memory.ex | 67 +++++++--- lib/exgit/object_store/promisor.ex | 18 +++ test/exgit/fs_size_test.exs | 188 +++++++++++++++++++++++++++++ 6 files changed, 386 insertions(+), 15 deletions(-) create mode 100644 test/exgit/fs_size_test.exs diff --git a/lib/exgit/fs.ex b/lib/exgit/fs.ex index ca0d2a1..a41be19 100644 --- a/lib/exgit/fs.ex +++ b/lib/exgit/fs.ex @@ -269,6 +269,44 @@ defmodule Exgit.FS do end end + @doc """ + Size in bytes of the blob at `path` — WITHOUT reading its content. + + The size-aware companion to `read_path/4`: use it to decide whether a + blob is too large to pull into memory *before* you pull it. For the + in-memory store this is O(1) (the size is indexed, not recomputed); + for on-disk loose objects it inflates only the header. + + Resolving the path may fetch *tree* objects (small) on a lazy clone, + but the blob itself is never fetched. For a lazy/partial clone whose + blob has not been materialized yet, returns `{:error, :not_local}` + rather than triggering a possibly-multi-GB fetch — call `read_path/4` + when you actually want the bytes. Directories return + `{:error, :not_a_blob}`. + + {:ok, size, repo} = Exgit.FS.size(repo, "HEAD", "go.mod") + + """ + @spec size(Repository.t(), ref(), path()) :: + {:ok, non_neg_integer(), Repository.t()} | {:error, term()} + def size(%Repository{} = repo, reference, path) do + with {:ok, tree_sha, repo} <- resolve_tree(repo, reference), + {:ok, {mode, sha}, repo} <- walk_path(repo, tree_sha, normalize_path(path)) do + if dir_mode?(mode) do + {:error, :not_a_blob} + else + case ObjectStore.object_size(repo.object_store, sha) do + {:ok, size} -> {:ok, size, repo} + {:error, _} = err -> err + end + end + end + end + + defp dir_mode?("40000"), do: true + defp dir_mode?("040000"), do: true + defp dir_mode?(_), do: false + @doc """ Return true if the path exists under the given reference. diff --git a/lib/exgit/object_store.ex b/lib/exgit/object_store.ex index f87853c..342a210 100644 --- a/lib/exgit/object_store.ex +++ b/lib/exgit/object_store.ex @@ -8,6 +8,18 @@ defprotocol Exgit.ObjectStore do @spec has?(t, binary()) :: boolean() def has?(store, sha) + # Uncompressed byte size of an object WITHOUT materializing its content. + # The point is a constant-memory size check: callers can decide whether a + # blob is too large to read before paying to inflate it into the heap. + # + # `{:error, :not_local}` is a meaningful, non-fatal result for lazy stores + # (`Promisor`): it means "knowing this size requires a network fetch" — so + # the caller can opt in rather than have a multi-GB fetch triggered behind + # a size check. `{:error, :not_found}` means the object is genuinely absent. + @spec object_size(t, binary()) :: + {:ok, non_neg_integer()} | {:error, :not_found | :not_local | term()} + def object_size(store, sha) + @spec import_objects(t, [{atom(), binary(), binary()}]) :: {:ok, t} def import_objects(store, raw_objects) diff --git a/lib/exgit/object_store/disk.ex b/lib/exgit/object_store/disk.ex index e683960..eaeab6e 100644 --- a/lib/exgit/object_store/disk.ex +++ b/lib/exgit/object_store/disk.ex @@ -79,6 +79,80 @@ defmodule Exgit.ObjectStore.Disk do else: {:error, {:sha_mismatch, expected_sha}} end + @doc """ + Uncompressed object size without materializing the object. + + For a loose object this inflates only the header bytes (`" + \\0"`) — constant memory, no matter how large the blob. Packed + objects fall back to a full read (the delta chain must be resolved to + know the final size), so this is cheap for loose objects and + O(object) for packed ones. + """ + @spec object_size(t(), binary()) :: {:ok, non_neg_integer()} | {:error, term()} + def object_size(%__MODULE__{root: root}, sha) when byte_size(sha) == 20 do + hex = Base.encode16(sha, case: :lower) + <> = hex + path = Path.join([root, "objects", prefix, rest]) + + case File.read(path) do + {:ok, compressed} -> loose_object_size(compressed) + {:error, :enoent} -> packed_object_size(root, sha) + {:error, reason} -> {:error, reason} + end + end + + defp loose_object_size(compressed) do + with {:ok, header} <- inflate_until_null(compressed), + {:ok, _type, size} <- parse_loose_header(header) do + {:ok, size} + end + end + + # The git object header is tiny; cap the scan so a corrupt or hostile + # object can't make us inflate unbounded output hunting for a NUL. + @max_header_bytes 64 + + # Stream-inflate only until the header's NUL terminator, returning the + # bytes before it. Bounded memory: never inflates the full object. + defp inflate_until_null(compressed) do + z = :zlib.open() + + try do + :zlib.inflateInit(z) + drain_until_null(z, compressed, <<>>) + rescue + _ -> {:error, :zlib_error} + catch + _, _ -> {:error, :zlib_error} + after + :zlib.close(z) + end + end + + defp drain_until_null(z, input, acc) do + {status, out} = :zlib.safeInflate(z, input) + acc = acc <> IO.iodata_to_binary(out) + + case :binary.match(acc, <<0>>) do + {pos, 1} -> + {:ok, binary_part(acc, 0, pos)} + + :nomatch -> + cond do + byte_size(acc) > @max_header_bytes -> {:error, :malformed_object_header} + status == :finished -> {:error, :malformed_object_header} + true -> drain_until_null(z, [], acc) + end + end + end + + defp packed_object_size(root, sha) do + case get_from_packs(root, sha) do + {:ok, obj} -> {:ok, IO.iodata_length(Exgit.Object.encode(obj))} + {:error, _} = err -> err + end + end + @spec put_object(t(), Exgit.Object.t()) :: {:ok, binary()} | {:error, term()} def put_object(%__MODULE__{root: root}, object) do {sha, raw} = object_raw(object) @@ -199,7 +273,7 @@ defmodule Exgit.ObjectStore.Disk do defp parse_loose_object(raw) do case :binary.match(raw, <<0>>) do {pos, 1} -> - <> = raw + <> = raw with {:ok, type, size} <- parse_loose_header(header), :ok <- check_loose_size(content, size) do @@ -446,6 +520,8 @@ defimpl Exgit.ObjectStore, for: Exgit.ObjectStore.Disk do ) end + def object_size(store, sha), do: Disk.object_size(store, sha) + def import_objects(store, raw_objects) do failures = Enum.reduce(raw_objects, [], fn {type, sha, content}, errs -> diff --git a/lib/exgit/object_store/memory.ex b/lib/exgit/object_store/memory.ex index 01fa19c..345c950 100644 --- a/lib/exgit/object_store/memory.ex +++ b/lib/exgit/object_store/memory.ex @@ -3,10 +3,19 @@ defmodule Exgit.ObjectStore.Memory do # Stores objects as {type_atom, zlib_compressed_content} for memory efficiency. # Objects are decoded on demand when `get` is called. + # + # `sizes` is a parallel index of `sha => uncompressed_byte_size`, kept in + # lockstep with `objects` by every insert path (put/import/streaming close). + # It lets `object_size/2` answer the size of a blob in O(1) WITHOUT + # `:zlib.uncompress`-ing the whole object into the heap — the difference + # between a constant-memory size check and materializing a multi-GB blob. - defstruct objects: %{} + defstruct objects: %{}, sizes: %{} - @type t :: %__MODULE__{objects: %{binary() => {atom(), binary()}}} + @type t :: %__MODULE__{ + objects: %{binary() => {atom(), binary()}}, + sizes: %{binary() => non_neg_integer()} + } @spec new() :: t() def new, do: %__MODULE__{} @@ -24,26 +33,44 @@ defmodule Exgit.ObjectStore.Memory do end @spec put_object(t(), Exgit.Object.t()) :: {:ok, binary(), t()} - def put_object(%__MODULE__{objects: objects} = store, object) do + def put_object(%__MODULE__{objects: objects, sizes: sizes} = store, object) do sha = Exgit.Object.sha(object) type = Exgit.Object.type(object) content = Exgit.Object.encode(object) |> IO.iodata_to_binary() compressed = :zlib.compress(content) - {:ok, sha, %{store | objects: Map.put(objects, sha, {type, compressed})}} + + {:ok, sha, + %{ + store + | objects: Map.put(objects, sha, {type, compressed}), + sizes: Map.put(sizes, sha, byte_size(content)) + }} end @spec has_object?(t(), binary()) :: boolean() def has_object?(%__MODULE__{objects: objects}, sha), do: Map.has_key?(objects, sha) + @doc """ + Uncompressed byte size of the stored object, in O(1) — without + decompressing it. Returns `{:error, :not_found}` if absent. + """ + @spec object_size(t(), binary()) :: {:ok, non_neg_integer()} | {:error, :not_found} + def object_size(%__MODULE__{sizes: sizes}, sha) do + case Map.fetch(sizes, sha) do + {:ok, size} -> {:ok, size} + :error -> {:error, :not_found} + end + end + @spec import_objects(t(), [{atom(), binary(), binary()}]) :: {:ok, t()} - def import_objects(%__MODULE__{objects: objects} = store, raw_objects) do - new_objects = - Enum.reduce(raw_objects, objects, fn {type, sha, content}, acc -> + def import_objects(%__MODULE__{objects: objects, sizes: sizes} = store, raw_objects) do + {new_objects, new_sizes} = + Enum.reduce(raw_objects, {objects, sizes}, fn {type, sha, content}, {objs, szs} -> compressed = :zlib.compress(content) - Map.put(acc, sha, {type, compressed}) + {Map.put(objs, sha, {type, compressed}), Map.put(szs, sha, byte_size(content))} end) - {:ok, %{store | objects: new_objects}} + {:ok, %{store | objects: new_objects, sizes: new_sizes}} end end @@ -86,6 +113,8 @@ defimpl Exgit.ObjectStore, for: Exgit.ObjectStore.Memory do ) end + def object_size(store, sha), do: Memory.object_size(store, sha) + def import_objects(store, raw_objects), do: Memory.import_objects(store, raw_objects) @@ -113,27 +142,37 @@ defimpl Exgit.ObjectStore, for: Exgit.ObjectStore.Memory do store_type: type, sha_ctx: sha_ctx, deflate: z, - acc: [] + acc: [], + # Actual uncompressed bytes seen, summed across chunks. We use this + # rather than `expected_size` so a caller that over- or under-declares + # the size can't desync the `sizes` index from the stored content. + usize: 0 } {:ok, handle} end - def write_chunk(_store, %{deflate: z, sha_ctx: sha_ctx, acc: acc} = handle, chunk) + def write_chunk(_store, %{deflate: z, sha_ctx: sha_ctx, acc: acc, usize: usize} = handle, chunk) when is_binary(chunk) do sha_ctx = :crypto.hash_update(sha_ctx, chunk) compressed_chunks = :zlib.deflate(z, chunk) - {:ok, %{handle | sha_ctx: sha_ctx, acc: [acc | compressed_chunks]}} + + {:ok, + %{handle | sha_ctx: sha_ctx, acc: [acc | compressed_chunks], usize: usize + byte_size(chunk)}} end - def close_write(%Memory{objects: objects} = store, %{__type__: :memory_write} = handle) do + def close_write( + %Memory{objects: objects, sizes: sizes} = store, + %{__type__: :memory_write} = handle + ) do sha = :crypto.hash_final(handle.sha_ctx) final_chunks = :zlib.deflate(handle.deflate, <<>>, :finish) :zlib.deflateEnd(handle.deflate) :zlib.close(handle.deflate) compressed = IO.iodata_to_binary([handle.acc | final_chunks]) new_objects = Map.put(objects, sha, {handle.store_type, compressed}) - {:ok, sha, %{store | objects: new_objects}} + new_sizes = Map.put(sizes, sha, handle.usize) + {:ok, sha, %{store | objects: new_objects, sizes: new_sizes}} end def cancel_write(_store, %{__type__: :memory_write, deflate: z}) do diff --git a/lib/exgit/object_store/promisor.ex b/lib/exgit/object_store/promisor.ex index de3d3a4..22fcae9 100644 --- a/lib/exgit/object_store/promisor.ex +++ b/lib/exgit/object_store/promisor.ex @@ -325,6 +325,22 @@ defmodule Exgit.ObjectStore.Promisor do ObjectStore.Memory.has_object?(cache, sha) end + @doc """ + Uncompressed byte size of `sha` IF it is already cached locally — + without triggering a fetch. Returns `{:error, :not_local}` when the + object has not been fetched yet, so a size check can never silently + pull a multi-GB blob over the network. + """ + @spec object_size(t(), binary()) :: + {:ok, non_neg_integer()} | {:error, :not_local} + def object_size(%__MODULE__{cache: cache}, sha) do + if ObjectStore.Memory.has_object?(cache, sha) do + ObjectStore.Memory.object_size(cache, sha) + else + {:error, :not_local} + end + end + @doc "Merge `raw_objects` into the cache." @spec import_objects(t(), [{atom(), binary(), binary()}]) :: {:ok, t()} def import_objects(%__MODULE__{cache: cache} = p, raw_objects) do @@ -645,6 +661,8 @@ defimpl Exgit.ObjectStore, for: Exgit.ObjectStore.Promisor do ) end + def object_size(store, sha), do: Promisor.object_size(store, sha) + def import_objects(store, raw_objects), do: Promisor.import_objects(store, raw_objects) diff --git a/test/exgit/fs_size_test.exs b/test/exgit/fs_size_test.exs new file mode 100644 index 0000000..b23b13b --- /dev/null +++ b/test/exgit/fs_size_test.exs @@ -0,0 +1,188 @@ +defmodule Exgit.FsSizeTest do + use ExUnit.Case, async: true + + alias Exgit.{FS, ObjectStore, RefStore, Repository} + alias Exgit.Object.{Blob, Commit, Tree} + alias Exgit.ObjectStore.{Disk, Memory, Promisor} + + describe "Memory.object_size/2" do + test "matches byte_size of the content across sizes, regardless of compressibility" do + for data <- [ + "", + "hello\n", + String.duplicate("a", 100_000), + :crypto.strong_rand_bytes(1_000_000) + ] do + store = Memory.new() + {:ok, sha, store} = ObjectStore.put(store, Blob.new(data)) + assert ObjectStore.object_size(store, sha) == {:ok, byte_size(data)} + end + end + + test "returns :not_found for an absent sha" do + assert ObjectStore.object_size(Memory.new(), :crypto.strong_rand_bytes(20)) == + {:error, :not_found} + end + + test "import_objects populates the size index" do + data = "imported\n" + {sha, content} = {raw_blob_sha(data), data} + {:ok, store} = ObjectStore.import_objects(Memory.new(), [{:blob, sha, content}]) + assert ObjectStore.object_size(store, sha) == {:ok, byte_size(data)} + end + + test "streaming write records the actual (summed) size, not the declared one" do + store = Memory.new() + # Declare 0 on purpose: the index must reflect bytes actually written. + {:ok, h} = ObjectStore.open_write(store, :blob, 0) + {:ok, h} = ObjectStore.write_chunk(store, h, "abc") + {:ok, h} = ObjectStore.write_chunk(store, h, "de") + {:ok, sha, store} = ObjectStore.close_write(store, h) + + assert ObjectStore.object_size(store, sha) == {:ok, 5} + end + end + + describe "Promisor.object_size/2 (never fetches)" do + test "returns the size for a locally cached object" do + blob = Blob.new("cached body\n") + # `:no_transport` is never used: object_size only reads the cache. + promisor = Promisor.new(:no_transport, initial_objects: [blob]) + sha = Blob.sha(blob) + + assert ObjectStore.object_size(promisor, sha) == {:ok, byte_size("cached body\n")} + end + + test "returns :not_local for an un-fetched object and does NOT touch the transport" do + # The transport is a bare atom with no Transport impl: if object_size + # tried to fetch, the call would crash. A clean :not_local proves it didn't. + promisor = Promisor.new(:boom_if_used, initial_objects: []) + absent = :crypto.strong_rand_bytes(20) + + assert ObjectStore.object_size(promisor, absent) == {:error, :not_local} + end + end + + describe "Disk.object_size/2" do + test "reads loose-object size from the header" do + dir = Path.join(System.tmp_dir!(), "exgit_size_#{System.unique_integer([:positive])}") + File.mkdir_p!(dir) + on_exit(fn -> File.rm_rf(dir) end) + + data = String.duplicate("x", 250_000) + store = Disk.new(dir) + {:ok, sha, store} = ObjectStore.put(store, Blob.new(data)) + + assert ObjectStore.object_size(store, sha) == {:ok, byte_size(data)} + end + end + + describe "FS.size/3" do + setup do + store = Memory.new() + + readme = Blob.new("hello world\n") + {:ok, readme_sha, store} = ObjectStore.put(store, readme) + + nested = Blob.new("deep\n") + {:ok, nested_sha, store} = ObjectStore.put(store, nested) + + nested_tree = Tree.new([{"100644", "c.txt", nested_sha}]) + {:ok, nested_tree_sha, store} = ObjectStore.put(store, nested_tree) + + src_tree = Tree.new([{"40000", "deep", nested_tree_sha}]) + {:ok, src_sha, store} = ObjectStore.put(store, src_tree) + + root = Tree.new([{"100644", "README.md", readme_sha}, {"40000", "src", src_sha}]) + {:ok, root_sha, store} = ObjectStore.put(store, root) + + commit = + Commit.new( + tree: root_sha, + parents: [], + author: "T 1700000000 +0000", + committer: "T 1700000000 +0000", + message: "init\n" + ) + + {:ok, commit_sha, store} = ObjectStore.put(store, commit) + {:ok, refs} = RefStore.write(RefStore.Memory.new(), "refs/heads/main", commit_sha, []) + {:ok, refs} = RefStore.write(refs, "HEAD", {:symbolic, "refs/heads/main"}, []) + + repo = %Repository{ + object_store: store, + ref_store: refs, + config: Exgit.Config.new(), + path: nil + } + + {:ok, repo: repo} + end + + test "returns the blob size for a top-level file", %{repo: repo} do + assert {:ok, size, %Repository{}} = FS.size(repo, "HEAD", "README.md") + assert size == byte_size("hello world\n") + end + + test "returns the blob size for a nested file", %{repo: repo} do + assert {:ok, 5, %Repository{}} = FS.size(repo, "HEAD", "src/deep/c.txt") + end + + test "matches read_path's blob byte_size", %{repo: repo} do + {:ok, {_mode, %Blob{data: data}}, _repo} = FS.read_path(repo, "HEAD", "README.md") + assert {:ok, byte_size(data), repo} == FS.size(repo, "HEAD", "README.md") + end + + test "rejects directories with :not_a_blob", %{repo: repo} do + assert FS.size(repo, "HEAD", "src") == {:error, :not_a_blob} + end + + test "returns :not_found for a missing path", %{repo: repo} do + assert FS.size(repo, "HEAD", "nope.txt") == {:error, :not_found} + end + end + + describe "FS.size/3 on a lazy clone" do + test "returns :not_local without fetching the blob" do + # Seed a Promisor cache with the commit + trees, but NOT the blob. + # `walk_path` resolves the path through cached trees; the blob's size + # is reported as :not_local rather than triggering a transport fetch. + store = Memory.new() + blob = Blob.new("would be huge\n") + blob_sha = Blob.sha(blob) + root = Tree.new([{"100644", "big.bin", blob_sha}]) + {:ok, root_sha, store} = ObjectStore.put(store, root) + + commit = + Commit.new( + tree: root_sha, + parents: [], + author: "T 1700000000 +0000", + committer: "T 1700000000 +0000", + message: "init\n" + ) + + {:ok, commit_sha, store} = ObjectStore.put(store, commit) + + # Cache holds commit + root tree only — the blob is deliberately absent. + {:ok, commit_obj} = ObjectStore.get(store, commit_sha) + {:ok, root_obj} = ObjectStore.get(store, root_sha) + promisor = Promisor.new(:boom_if_used, initial_objects: [commit_obj, root_obj]) + + {:ok, refs} = RefStore.write(RefStore.Memory.new(), "refs/heads/main", commit_sha, []) + {:ok, refs} = RefStore.write(refs, "HEAD", {:symbolic, "refs/heads/main"}, []) + + repo = %Repository{ + object_store: promisor, + ref_store: refs, + config: Exgit.Config.new(), + path: nil + } + + assert FS.size(repo, "HEAD", "big.bin") == {:error, :not_local} + end + end + + # Compute a blob sha the same way the store does, for import_objects tests. + defp raw_blob_sha(data), do: Blob.sha(Blob.new(data)) +end From 37c417baf006f96ae875297e8304d63e741cc392 Mon Sep 17 00:00:00 2001 From: Ivar Vong Date: Wed, 1 Jul 2026 09:21:39 -0400 Subject: [PATCH 03/13] security: redact URL credentials from telemetry, cap ls-refs refs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two hardening fixes on the HTTP transport: - A token embedded in a remote URL (`https://token@host/…`) is now redacted to `***` before the URL enters any :telemetry span metadata (ls_refs/fetch/push) or the ref_rejected security event. Previously such a token could reach telemetry exporters and log aggregators. - ls-refs responses are capped at 1,000,000 refs; the transport stream halts once the cap trips, so a hostile server can no longer stream unbounded refs into client memory. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01YVcfxkfN7A7KYLmNYt9UcH --- lib/exgit/transport/http.ex | 79 ++++++++++++++----- .../http_telemetry_redaction_test.exs | 74 +++++++++++++++++ 2 files changed, 134 insertions(+), 19 deletions(-) create mode 100644 test/exgit/transport/http_telemetry_redaction_test.exs diff --git a/lib/exgit/transport/http.ex b/lib/exgit/transport/http.ex index 79cd7af..99b2723 100644 --- a/lib/exgit/transport/http.ex +++ b/lib/exgit/transport/http.ex @@ -171,10 +171,15 @@ defmodule Exgit.Transport.HTTP do def capabilities_cached(%__MODULE__{capabilities_cache: cached} = t), do: {cached, t} + # Backstop cap on refs accepted from a single ls-refs response. Real repos + # top out in the tens of thousands (linux, esp-idf); this bound exists only + # to stop a hostile server from streaming unbounded refs into client memory. + @max_refs 1_000_000 + def ls_refs(%__MODULE__{} = t, opts \\ []) do Exgit.Telemetry.span( [:exgit, :transport, :ls_refs], - %{transport: :http, url: t.url}, + %{transport: :http, url: redact_url(t.url)}, fn -> case do_ls_refs(t, opts) do {:ok, refs, meta} = result -> @@ -213,15 +218,23 @@ defmodule Exgit.Transport.HTTP do # without ever materializing the full response body or the list of # decoded packets. For repos with tens of thousands of refs (esp-idf, # linux), this keeps the transport's memory bound flat in ref count. - init_acc = {[], %{peeled: %{}}} + # Map-shaped accumulator (not a tuple) so the streaming loop's + # `%{error: e}` halt check fires the moment the ref cap trips. + init_acc = %{refs: [], meta: %{peeled: %{}}, count: 0, error: nil} + # Redact before it reaches `keep_ref?`'s security telemetry, which + # echoes the source URL. + source_url = redact_url(t.url) handle_packet = fn - {:data, line}, acc -> parse_ls_refs_line(line, t.url, acc) + {:data, line}, acc -> parse_ls_refs_line(line, source_url, acc) _, acc -> acc end case stream_upload_pack(t, body, init_acc, handle_packet) do - {:ok, {refs_rev, meta}} -> + {:ok, %{error: reason}} when not is_nil(reason) -> + {:error, reason} + + {:ok, %{refs: refs_rev, meta: meta}} -> meta = if map_size(meta.peeled) == 0, do: Map.delete(meta, :peeled), else: meta @@ -244,29 +257,51 @@ defmodule Exgit.Transport.HTTP do # Hostile ref names are rejected here via `Exgit.RefName.valid?/1`; # rejections emit `[:exgit, :security, :ref_rejected]` telemetry # and drop the entry entirely. - defp parse_ls_refs_line(line, source_url, {refs, meta}) do + # Once the ref cap trips, short-circuit: the stream loop halts on + # `acc.error`, but packets already decoded from the current chunk still + # flow through this fold, so ignore them. + defp parse_ls_refs_line(_line, _source_url, %{error: reason} = acc) when not is_nil(reason), + do: acc + + defp parse_ls_refs_line(line, source_url, acc) do line = String.trim_trailing(line, "\n") case String.split(line, " ", parts: 3) do [hex_sha, ref, attrs] when byte_size(hex_sha) == 40 -> with {:ok, sha} <- Base.decode16(hex_sha, case: :mixed), true <- keep_ref?(ref, source_url) do - attrs_map = parse_ls_refs_attrs(attrs) - add_ref(refs, meta, ref, sha, attrs_map) + add_ref(acc, ref, sha, parse_ls_refs_attrs(attrs)) else - _ -> {refs, meta} + _ -> acc end [hex_sha, ref] when byte_size(hex_sha) == 40 -> with {:ok, sha} <- Base.decode16(hex_sha, case: :mixed), true <- keep_ref?(ref, source_url) do - add_ref(refs, meta, ref, sha, %{}) + add_ref(acc, ref, sha, %{}) else - _ -> {refs, meta} + _ -> acc end _ -> - {refs, meta} + acc + end + end + + # Strip credentials from a URL before it enters telemetry metadata or a + # security event. exgit's own API carries auth in the `:auth` field, but + # callers commonly embed a token in the URL (`https://token@host/…`) the + # way git does — and that must never reach a telemetry exporter, log + # aggregator, or OTel span. + # The `is_binary/1` guard is load-bearing for Dialyzer: `URI.parse/1` + # accepts `URI.t() | binary()`, so without it Dialyzer widens `url`'s + # inferred type to include `%URI{}`, and the `userinfo == nil` branch + # (which returns `url` unchanged) then appears to return a struct. + @spec redact_url(String.t()) :: String.t() + defp redact_url(url) when is_binary(url) do + case URI.parse(url).userinfo do + nil -> url + userinfo -> String.replace(url, userinfo <> "@", "***@", global: false) end end @@ -284,10 +319,12 @@ defmodule Exgit.Transport.HTTP do end end - # Add an entry to `refs` and lift any attributes into `meta`. - # HEAD's `symref-target` becomes `meta.head`; annotated tags' - # `peeled` target becomes `meta.peeled[tag_name]`. - defp add_ref(refs, meta, ref, sha, attrs) do + # Add an entry to the accumulator and lift any attributes into `meta`. + # HEAD's `symref-target` becomes `meta.head`; annotated tags' `peeled` + # target becomes `meta.peeled[tag_name]`. Past `@max_refs` we stop + # appending and set `acc.error` so the transport stream loop halts — + # a hostile server can't grow the ref list without bound. + defp add_ref(%{refs: refs, meta: meta, count: count} = acc, ref, sha, attrs) do meta = case Map.get(attrs, :symref_target) do nil -> meta @@ -304,7 +341,11 @@ defmodule Exgit.Transport.HTTP do peeled_sha -> put_in(meta, [:peeled, ref], peeled_sha) end - {[{ref, sha} | refs], meta} + if count + 1 > @max_refs do + %{acc | meta: meta, error: {:too_many_refs, @max_refs}} + else + %{acc | refs: [{ref, sha} | refs], meta: meta, count: count + 1} + end end defp parse_ls_refs_attrs(attrs) do @@ -330,7 +371,7 @@ defmodule Exgit.Transport.HTTP do def fetch(%__MODULE__{} = t, wants, opts \\ []) do Exgit.Telemetry.span( [:exgit, :transport, :fetch], - %{transport: :http, url: t.url, wants_count: length(wants)}, + %{transport: :http, url: redact_url(t.url), wants_count: length(wants)}, fn -> case do_fetch(t, wants, opts) do {:ok, pack_bytes, summary} = result -> @@ -581,7 +622,7 @@ defmodule Exgit.Transport.HTTP do [:exgit, :transport, :push], %{ transport: :http, - url: t.url, + url: redact_url(t.url), update_count: length(updates), pack_bytes: byte_size(pack_bytes) }, @@ -780,7 +821,7 @@ defmodule Exgit.Transport.HTTP do captured = if room > 0 do take = min(room, byte_size(chunk)) - <> = chunk + <> = chunk <> else state.error_body diff --git a/test/exgit/transport/http_telemetry_redaction_test.exs b/test/exgit/transport/http_telemetry_redaction_test.exs new file mode 100644 index 0000000..6328db1 --- /dev/null +++ b/test/exgit/transport/http_telemetry_redaction_test.exs @@ -0,0 +1,74 @@ +defmodule Exgit.Transport.HttpTelemetryRedactionTest do + use ExUnit.Case, async: true + + alias Exgit.Transport.HTTP + + # A token embedded in a remote URL must never reach telemetry metadata. + # `Exgit.Telemetry.span/3` (via `:telemetry.span/3`) emits a `[..., :start]` + # event carrying the metadata BEFORE it runs the network call — so we can + # capture the URL metadata without a live git server: point the transport at + # a dead port, let the request fail, and read the start event. + @token "SECRET_TOKEN_do_not_leak_9f3a" + + # The telemetry handler is global and other async tests emit the same spans, + # so filter on a unique per-test path marker (which survives redaction — only + # the userinfo is stripped) to capture exactly this test's event. + defp capture_start_meta(event, marker, run) do + handler = "redact-#{marker}" + test_pid = self() + + :telemetry.attach( + handler, + event ++ [:start], + fn _e, _measurements, meta, _cfg -> + if is_binary(meta[:url]) and String.contains?(meta.url, marker) do + send(test_pid, {:tele_meta, meta}) + end + end, + nil + ) + + try do + _ = run.() + assert_receive {:tele_meta, meta}, 2_000 + meta + after + :telemetry.detach(handler) + end + end + + defp marker, do: "probe#{System.unique_integer([:positive])}" + + test "ls_refs telemetry redacts a token embedded in the URL" do + m = marker() + t = HTTP.new("https://x-access-token:#{@token}@127.0.0.1:1/#{m}.git") + meta = capture_start_meta([:exgit, :transport, :ls_refs], m, fn -> HTTP.ls_refs(t) end) + + refute meta.url =~ @token, "token leaked into ls_refs telemetry: #{meta.url}" + assert meta.url =~ "***" + end + + test "fetch telemetry redacts a token embedded in the URL" do + m = marker() + t = HTTP.new("https://x-access-token:#{@token}@127.0.0.1:1/#{m}.git") + + meta = + capture_start_meta([:exgit, :transport, :fetch], m, fn -> + # `wants` are raw 20-byte shas; the call fails at the dead port, but + # the span's :start event (with the URL metadata) fires first. + HTTP.fetch(t, [<<0::160>>], []) + end) + + refute meta.url =~ @token, "token leaked into fetch telemetry: #{meta.url}" + assert meta.url =~ "***" + end + + test "a URL without credentials passes through unchanged" do + m = marker() + t = HTTP.new("https://127.0.0.1:1/#{m}.git") + meta = capture_start_meta([:exgit, :transport, :ls_refs], m, fn -> HTTP.ls_refs(t) end) + + assert meta.url == "https://127.0.0.1:1/#{m}.git" + refute meta.url =~ "***" + end +end From b5438c4a242aecb3bf5c81cdadb53b6d2b4fdd06 Mon Sep 17 00:00:00 2001 From: Ivar Vong Date: Wed, 1 Jul 2026 09:21:39 -0400 Subject: [PATCH 04/13] fix: resolve Dialyzer MapSet opacity warnings Under the 1.20 toolchain, two MapSet opacity false-positives surfaced. Replace the internal `seen` membership set in the reachability walk with a plain map (not opaque, semantically identical, marginally faster), and fold the registry's EXIT cleanup with MapSet.delete instead of MapSet.difference (which trips the warning on the untyped GenServer state). Dialyzer is now clean. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01YVcfxkfN7A7KYLmNYt9UcH --- lib/exgit.ex | 18 ++++++++++++++---- lib/exgit/repo_registry.ex | 8 +++++++- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/lib/exgit.ex b/lib/exgit.ex index 8ca5bf7..d2e0bb7 100644 --- a/lib/exgit.ex +++ b/lib/exgit.ex @@ -239,7 +239,6 @@ defmodule Exgit do spec -> case Exgit.Filter.encode(spec) do - :none -> {:ok, :none} {:ok, wire} -> {:ok, wire} {:error, _} = err -> err end @@ -585,7 +584,7 @@ defmodule Exgit do end defp collect_push_objects(store, sha, remote_refs) do - remote_shas = MapSet.new(Map.values(remote_refs)) + remote_shas = Map.new(Map.values(remote_refs), &{&1, true}) collect_reachable(store, [sha], remote_shas) end @@ -593,17 +592,28 @@ defmodule Exgit do # `seen` accumulator so shared subtrees are visited exactly once, and # is bounded by O(heap) rather than O(stack) so it handles deep # histories (millions of commits) without stack overflow. + # `seen` is a plain map used as a membership set (`sha => true`) rather + # than a `MapSet`. MapSet is an opaque type, and threading it through this + # self-recursive accumulator — whose base clause binds `_seen` as a + # wildcard — trips a Dialyzer opacity false-positive. A plain map is + # semantically identical here (set of seen shas), not opaque, and a touch + # faster. + @typep seen_set :: %{optional(binary()) => true} + + @spec collect_reachable(ObjectStore.t(), [binary()], seen_set()) :: [Exgit.Object.t()] defp collect_reachable(store, initial_shas, seen) do do_collect_reachable(store, initial_shas, seen, []) end + @spec do_collect_reachable(ObjectStore.t(), [binary()], seen_set(), [Exgit.Object.t()]) :: + [Exgit.Object.t()] defp do_collect_reachable(_store, [], _seen, acc), do: Enum.reverse(acc) defp do_collect_reachable(store, [sha | rest], seen, acc) do - if MapSet.member?(seen, sha) do + if Map.has_key?(seen, sha) do do_collect_reachable(store, rest, seen, acc) else - seen = MapSet.put(seen, sha) + seen = Map.put(seen, sha, true) case ObjectStore.get(store, sha) do {:ok, obj} -> diff --git a/lib/exgit/repo_registry.ex b/lib/exgit/repo_registry.ex index b8215e7..3f06b50 100644 --- a/lib/exgit/repo_registry.ex +++ b/lib/exgit/repo_registry.ex @@ -296,7 +296,13 @@ defmodule Exgit.RepoRegistry do Registry.unregister(@registry_name, url) end - {:noreply, %{state | urls: MapSet.difference(state.urls, MapSet.new(dead_urls))}} + # Fold with `MapSet.delete/2` rather than `MapSet.difference/2`. `state` + # is an untyped GenServer map, so `state.urls` reads as `any()`, which + # trips a MapSet opacity warning through `difference` (but not through + # `delete`, as the `MapSet.delete` calls elsewhere in this module show). + # The dead set is tiny (an exit-path cleanup), so the fold is cheap. + urls = Enum.reduce(dead_urls, state.urls, fn url, acc -> MapSet.delete(acc, url) end) + {:noreply, %{state | urls: urls}} end ## Internal From 553ec5d33602a2409d85536a581c8d4cc6607a9a Mon Sep 17 00:00:00 2001 From: Ivar Vong Date: Wed, 1 Jul 2026 09:22:00 -0400 Subject: [PATCH 05/13] chore: prep for hex.pm publish + consumer-install CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ready the package for its first hex.pm release: - Depend on the published `vfs ~> 0.1.0` (was a git ref) and add ex_doc so `mix hex.publish` can build HexDocs; ship docs/PERFORMANCE.md so the README link resolves on hex. - Add a consumer-install smoke test (scripts/consumer_smoke.sh + a CI job) that builds the package as it ships, depends on it from a throwaway :prod project, and clones+reads a public repo — catching missing files() entries and dev/test compile-coupling the in-repo suite can't see. - README: flag Exgit.Index as experimental. CHANGELOG: record the size-aware and security work, and note the vfs integration is pre-release. gitignore the local scratch/ and hex.build tarball. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01YVcfxkfN7A7KYLmNYt9UcH --- .github/workflows/ci.yml | 31 +++++++++++ .gitignore | 7 +++ CHANGELOG.md | 37 +++++++++++++ README.md | 4 +- mix.exs | 16 +++--- mix.lock | 8 ++- scripts/consumer_smoke.sh | 112 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 204 insertions(+), 11 deletions(-) create mode 100755 scripts/consumer_smoke.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index de17234..46fbee8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -159,3 +159,34 @@ jobs: CF_ARTIFACT_REMOTE: ${{ secrets.CF_ARTIFACT_REMOTE }} CF_ARTIFACT_TOKEN: ${{ secrets.CF_ARTIFACT_TOKEN }} run: mix test --warnings-as-errors --only cloudflare + + consumer-smoke: + name: Consumer install smoke (package + use) + runs-on: ubuntu-24.04 + # Separate from the `test` job on purpose: this builds the package + # exactly as it ships to Hex, depends on it from a throwaway project + # compiled in :prod, and uses the public API end-to-end. It catches + # missing `files:` entries and compile-time coupling to dev/test/ + # optional deps — neither of which the in-repo suite can see. + # Live network (clones a public repo); runs on every push and PR. + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Elixir / OTP + uses: erlef/setup-beam@v1 + with: + elixir-version: "1.19" + otp-version: "28" + version-type: loose + + - name: Install local Hex & rebar + run: | + mix local.hex --force + mix local.rebar --force + + - name: Fetch dependencies + run: mix deps.get + + - name: Consumer-install smoke (clones elixir-ai-tools/just_bash) + run: bash scripts/consumer_smoke.sh diff --git a/.gitignore b/.gitignore index d2cb7f0..9209ff6 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,10 @@ ex_git-*.tar # Temporary files, for example, from tests. /tmp/ +# Local scratch: internal notes, third-party product critiques, repro +# scripts. Never published to GitHub or hex. +/scratch/ + # Environment / secrets. Never commit. .env .env.* @@ -33,3 +37,6 @@ ex_git-*.tar # mix dialyzer --plt; never commit them. priv/plts/ + +# hex.build package artifact +/exgit-*.tar diff --git a/CHANGELOG.md b/CHANGELOG.md index e69e673..d16beed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,43 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Security — credential redaction + ref bounds + +- **Telemetry no longer leaks URL-embedded credentials.** A token + embedded in a remote URL (`https://token@host/…`, as git clients + commonly accept) is now redacted to `***` before the URL enters any + `:telemetry` span metadata (`ls_refs`, `fetch`, `push`) or the + `[:exgit, :security, :ref_rejected]` event. Previously such a token + could reach telemetry exporters / log aggregators. Prefer the + `:auth` field regardless — it was already redacted. +- **`ls-refs` responses are capped at 1,000,000 refs.** A hostile or + broken server can no longer stream unbounded refs into client + memory; the transport stream halts once the cap trips. Real repos + (linux, esp-idf) sit far below this. + +### Notes + +- The optional `:vfs` integration depends on the pre-1.0 `vfs` + package (`~> 0.1.0`), which itself depends on exgit. The cycle is + handled with `runtime: false` + a compile-time `Code.ensure_loaded?` + guard, but treat the integration as pre-release and pin `vfs` if you + rely on it. + +### Added — size-aware reads + +- **`Exgit.FS.size/3`** — byte size of a blob at a path *without* + materializing its content. The size-aware companion to + `read_path/4`: gate on it before pulling a blob into memory. + O(1) for the in-memory store; on-disk loose objects inflate only + the header. Resolving the path may fetch trees (small) on a lazy + clone, but never the blob — an un-fetched blob returns + `{:error, :not_local}` instead of triggering a possibly-multi-GB + fetch. Directories return `{:error, :not_a_blob}`. +- **`Exgit.ObjectStore.object_size/2`** — new protocol callback + backing the above. Memory keeps a parallel `sha => size` index + (no extra decompression); `Promisor` answers from cache or + returns `{:error, :not_local}` without fetching. + ### Added — observability + workload bench - **`Exgit.Profiler`** — structured trace of `:telemetry` span diff --git a/README.md b/README.md index e61ac15..5cf766a 100644 --- a/README.md +++ b/README.md @@ -370,7 +370,9 @@ changes require a major-version bump. Functions annotated `@doc experimental: true` — currently `FS.prefetch/3` and `Repository.materialize/2` — are explicitly -exempt from SemVer guarantees until marked stable. The `:lazy` and +exempt from SemVer guarantees until marked stable. The whole +`Exgit.Index` module is likewise experimental (see its `@moduledoc`) +and may change in any release. The `:lazy` and `:filter` options on `Exgit.clone/2` are also experimental: the threading contract (`{:ok, result, repo}` shape) may evolve before v1.0. diff --git a/mix.exs b/mix.exs index 3347296..3790b67 100644 --- a/mix.exs +++ b/mix.exs @@ -53,7 +53,8 @@ defmodule Exgit.MixProject do links: %{ "GitHub" => @source_url }, - files: ~w(lib .formatter.exs mix.exs README.md LICENSE CHANGELOG.md SECURITY.md) + files: + ~w(lib docs/PERFORMANCE.md .formatter.exs mix.exs README.md LICENSE CHANGELOG.md SECURITY.md) ] end @@ -91,8 +92,7 @@ defmodule Exgit.MixProject do # consumers can attach to. Zero cost when no handler is attached. {:telemetry, "~> 1.0"}, # Optional vfs integration: `Exgit.Workspace` ships a - # `VFS.Mountable` defimpl when `:vfs` is loaded. Pinned to a SHA - # because vfs has no hex release yet. + # `VFS.Mountable` defimpl when `:vfs` is loaded. # # `optional: true` means downstream consumers don't have to install # :vfs to use exgit; if they DO add :vfs, Mix orders our build after @@ -118,11 +118,7 @@ defmodule Exgit.MixProject do # remove vfs from our dep graph in :prod, breaking compile-ordering # guarantees in downstream consumer builds. Requires Elixir ~> 1.18; # the 1.17 CI tier skips the integration via `Code.ensure_loaded?`. - {:vfs, - github: "ivarvong/vfs", - ref: "32d2ab618ec12c16fe4f675b5ee8b563c660dd69", - optional: true, - runtime: false}, + {:vfs, "~> 0.1.0", optional: true, runtime: false}, {:stream_data, "~> 1.0", only: [:test, :dev]}, # Test-only: localhost HTTP server for stubbing the Cloudflare # Artifacts REST API in `test/exgit/cloudflare_artifacts_test.exs`. @@ -136,7 +132,9 @@ defmodule Exgit.MixProject do {:opentelemetry_exporter, "~> 1.8", only: [:dev, :test]}, # Dev-only static analysis. Not runtime deps. {:dialyxir, "~> 1.4", only: [:dev], runtime: false}, - {:credo, "~> 1.7", only: [:dev], runtime: false} + {:credo, "~> 1.7", only: [:dev], runtime: false}, + # Doc generation for HexDocs (`mix docs` / `mix hex.publish`). + {:ex_doc, "~> 0.34", only: [:dev], runtime: false} ] end diff --git a/mix.lock b/mix.lock index 14f7f5c..23ef125 100644 --- a/mix.lock +++ b/mix.lock @@ -9,7 +9,9 @@ "credo": {:hex, :credo, "1.7.18", "5c5596bf7aedf9c8c227f13272ac499fe8eae6237bd326f2f07dfc173786f042", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "a189d164685fd945809e862fe76a7420c4398fa288d76257662aecb909d6b3e5"}, "ctx": {:hex, :ctx, "0.6.0", "8ff88b70e6400c4df90142e7f130625b82086077a45364a78d208ed3ed53c7fe", [:rebar3], [], "hexpm", "a14ed2d1b67723dbebbe423b28d7615eb0bdcba6ff28f2d1f1b0a7e1d4aa5fc2"}, "dialyxir": {:hex, :dialyxir, "1.4.7", "dda948fcee52962e4b6c5b4b16b2d8fa7d50d8645bbae8b8685c3f9ecb7f5f4d", [:mix], [{:erlex, ">= 0.2.8", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b34527202e6eb8cee198efec110996c25c5898f43a4094df157f8d28f27d9efe"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.45", "cba8369ab2a1342e419bc2760eec731b17be828941dcf494045d44766227e1d5", [:mix], [], "hexpm", "d3ec045bf122965db20c0bdb420e19ee1415843135327124918473feb4b328e8"}, "erlex": {:hex, :erlex, "0.2.8", "cd8116f20f3c0afe376d1e8d1f0ae2452337729f68be016ea544a72f767d9c12", [:mix], [], "hexpm", "9d66ff9fedf69e49dc3fd12831e12a8a37b76f8651dd21cd45fcf5561a8a7590"}, + "ex_doc": {:hex, :ex_doc, "0.40.3", "4a972ffe64bc07dc605af487e98fc19b72a4185f55ca031b94c0552d6071c1d9", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "2756e357742fecd9749b489b85d67c9ce99c465f2e75728d9e6dc8d704b973de"}, "file_system": {:hex, :file_system, "1.1.1", "31864f4685b0148f25bd3fbef2b1228457c0c89024ad67f7a81a3ffbc0bbad3a", [:mix], [], "hexpm", "7a15ff97dfe526aeefb090a7a9d3d03aa907e100e262a0f8f7746b78f8f87a5d"}, "finch": {:hex, :finch, "0.21.0", "b1c3b2d48af02d0c66d2a9ebfb5622be5c5ecd62937cf79a88a7f98d48a8290c", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "87dc6e169794cb2570f75841a19da99cfde834249568f2a5b121b809588a4377"}, "gproc": {:hex, :gproc, "0.9.1", "f1df0364423539cf0b80e8201c8b1839e229e5f9b3ccb944c5834626998f5b8c", [:rebar3], [], "hexpm", "905088e32e72127ed9466f0bac0d8e65704ca5e73ee5a62cb073c3117916d507"}, @@ -17,9 +19,13 @@ "hpack": {:hex, :hpack_erl, "0.3.0", "2461899cc4ab6a0ef8e970c1661c5fc6a52d3c25580bc6dd204f84ce94669926", [:rebar3], [], "hexpm", "d6137d7079169d8c485c6962dfe261af5b9ef60fbc557344511c1e65e3d95fb0"}, "hpax": {:hex, :hpax, "1.0.3", "ed67ef51ad4df91e75cc6a1494f851850c0bd98ebc0be6e81b026e765ee535aa", [:mix], [], "hexpm", "8eab6e1cfa8d5918c2ce4ba43588e894af35dbd8e91e6e55c817bca5847df34a"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, + "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, + "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, + "makeup_erlang": {:hex, :makeup_erlang, "1.1.0", "835f7e60792e08824cda445639555d7bf1bbbddb1b60b306e33cb6f6db24dc74", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "1cd6780fb1dd1a03979abaed0fe82712b0625118fd5257d3ebbf73f960c73c3c"}, "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, "mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"}, "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, + "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, "opentelemetry": {:hex, :opentelemetry, "1.7.0", "20d0f12d3d1c398d3670fd44fd1a7c495dd748ab3e5b692a7906662e2fb1a38a", [:rebar3], [{:opentelemetry_api, "~> 1.5.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}], "hexpm", "a9173b058c4549bf824cbc2f1d2fa2adc5cdedc22aa3f0f826951187bbd53131"}, "opentelemetry_api": {:hex, :opentelemetry_api, "1.5.0", "1a676f3e3340cab81c763e939a42e11a70c22863f645aa06aafefc689b5550cf", [:mix, :rebar3], [], "hexpm", "f53ec8a1337ae4a487d43ac89da4bd3a3c99ddf576655d071deed8b56a2d5dda"}, @@ -34,5 +40,5 @@ "stream_data": {:hex, :stream_data, "1.3.0", "bde37905530aff386dea1ddd86ecbf00e6642dc074ceffc10b7d4e41dfd6aac9", [:mix], [], "hexpm", "3cc552e286e817dca43c98044c706eec9318083a1480c52ae2688b08e2936e3c"}, "telemetry": {:hex, :telemetry, "1.4.1", "ab6de178e2b29b58e8256b92b382ea3f590a47152ca3651ea857a6cae05ac423", [:rebar3], [], "hexpm", "2172e05a27531d3d31dd9782841065c50dd5c3c7699d95266b2edd54c2dafa1c"}, "tls_certificate_check": {:hex, :tls_certificate_check, "1.32.1", "f90f9647668f7af804ec63c4e67f3799931166caf8264b81f37c1d863a36ae1d", [:rebar3], [{:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "e78a157966456b500a87a2fc29cffcd6dcfb5a26348c8372a2c5c0a8e5797f51"}, - "vfs": {:git, "https://github.com/ivarvong/vfs.git", "32d2ab618ec12c16fe4f675b5ee8b563c660dd69", [ref: "32d2ab618ec12c16fe4f675b5ee8b563c660dd69"]}, + "vfs": {:hex, :vfs, "0.1.0", "7bac0a4cda3e4fc237d9c798116c2b23d7ece1f9f87651062b0f8f071f0b448d", [:mix], [{:telemetry, "~> 1.3", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "72f372bb60b4b4cf8fe67098c8559f007f115cb2fe3f94a05cdf9f18c71b0dd0"}, } diff --git a/scripts/consumer_smoke.sh b/scripts/consumer_smoke.sh new file mode 100755 index 0000000..90ddeb3 --- /dev/null +++ b/scripts/consumer_smoke.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash +# +# Consumer-install smoke test. +# +# Verifies that a *downstream* project can depend on the exgit package +# AS IT SHIPS TO HEX and actually use the public API end-to-end. +# +# Why this exists: the in-repo `mix test` suite runs with every dev/test +# dependency present, `test/support` on the load path, and the full +# source tree visible. It therefore CANNOT catch the two failure modes +# that only bite a real adopter: +# +# 1. A file the code needs is missing from the `files:` glob in mix.exs +# (the package tarball is incomplete). +# 2. Code under `lib/` has a compile-time reference to a dev/test-only +# or optional dependency (e.g. the optional `:vfs` defimpl, Bypass, +# StreamData) that isn't present in a clean consumer build. +# +# To reproduce a true consumer, we build the package the way Hex does, +# extract ONLY the shipped files, and depend on that extracted directory +# from a throwaway project compiled in :prod — so no dev/test deps and no +# test/support reach the build. Then we exercise the advertised workflow: +# clone a public repo and read a file by path. +# +# Network: clones https://github.com/elixir-ai-tools/just_bash (public, +# unauthenticated — per the "no auth on public repos" invariant). + +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$REPO_ROOT" + +WORK="$(mktemp -d)" +trap 'rm -rf "$WORK"' EXIT + +echo "==> Building the Hex package (files: glob only)" +rm -f exgit-*.tar +mix hex.build +PKG_TAR="$(ls exgit-*.tar | head -1)" + +echo "==> Unpacking package contents ($PKG_TAR)" +# A Hex package tar contains: metadata.config, contents.tar.gz, CHECKSUM, +# VERSION. The shipped source files live inside contents.tar.gz. +tar -xf "$PKG_TAR" -C "$WORK" +mkdir -p "$WORK/pkg" +tar -xzf "$WORK/contents.tar.gz" -C "$WORK/pkg" +rm -f "$PKG_TAR" + +echo "==> Files the consumer will see:" +(cd "$WORK/pkg" && find . -type f | sort | sed 's/^/ /') + +echo "==> Scaffolding a throwaway consumer project" +CONSUMER="$WORK/consumer" +mkdir -p "$CONSUMER/lib" + +cat > "$CONSUMER/mix.exs" < 1.17", + # Depend on the EXTRACTED package, not the source tree — so only + # the shipped files and prod deps are visible. This is the whole + # point of the smoke test. + deps: [{:exgit, path: "${WORK}/pkg"}] + ] + end + + def application, do: [extra_applications: [:logger]] +end +EOF + +cat > "$CONSUMER/lib/smoke.ex" <<'EOF' +defmodule ExgitConsumerSmoke do + @moduledoc "Clone a public repo with exgit and read a file by path." + + @url "https://github.com/elixir-ai-tools/just_bash" + + def run do + # Eager, unauthenticated full clone of a public repo. + {:ok, repo} = Exgit.clone(@url) + + # Path-oriented read: returns {:ok, {mode, %Exgit.Object.Blob{}}, repo}. + {:ok, {_mode, blob}, _repo} = Exgit.FS.read_path(repo, "HEAD", "README.md") + data = blob.data + + if byte_size(data) == 0 do + raise "README.md came back empty" + end + + unless String.contains?(String.downcase(data), "bash") do + raise "README.md content did not look like just_bash (no 'bash' found)" + end + + IO.puts("OK: cloned just_bash, read README.md (#{byte_size(data)} bytes)") + end +end +EOF + +echo "==> Resolving deps + compiling consumer in :prod (warnings as errors)" +cd "$CONSUMER" +export MIX_ENV=prod +mix local.hex --force >/dev/null +mix local.rebar --force >/dev/null +mix deps.get +mix compile --warnings-as-errors + +echo "==> Running the smoke (clones just_bash over the network)" +mix run -e 'ExgitConsumerSmoke.run()' From 9d5380f82d2e593d9329d2e1c396963aa2944c7f Mon Sep 17 00:00:00 2001 From: Ivar Vong Date: Wed, 1 Jul 2026 09:22:00 -0400 Subject: [PATCH 06/13] test: add gated Cloudflare Artifacts perf benchmark Bootstrap-clone and first-touch-latency benchmark across exgit's clone modes against CF Artifacts. Tagged :cloudflare_perf and excluded by default (and when CF creds are absent), so it never runs in ordinary CI. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01YVcfxkfN7A7KYLmNYt9UcH --- test/exgit/cloudflare_artifacts_perf_test.exs | 429 ++++++++++++++++++ test/test_helper.exs | 3 +- 2 files changed, 431 insertions(+), 1 deletion(-) create mode 100644 test/exgit/cloudflare_artifacts_perf_test.exs diff --git a/test/exgit/cloudflare_artifacts_perf_test.exs b/test/exgit/cloudflare_artifacts_perf_test.exs new file mode 100644 index 0000000..a999ddc --- /dev/null +++ b/test/exgit/cloudflare_artifacts_perf_test.exs @@ -0,0 +1,429 @@ +defmodule Exgit.CloudflareArtifactsPerfTest do + @moduledoc """ + Performance benchmark for Cloudflare Artifacts: bootstrap clone time + and per-blob first-touch latency across exgit's three clone modes. + + Modes: + + * `eager` — full clone, all objects fetched up front + * `filter:{:blob, :none}` — refs + commits + trees, blobs on demand + * `lazy: true` — refs only, commits + trees + blobs on demand + + Seeds a fresh repo with `@file_count × @file_bytes` random blobs in a + single commit, then runs `@clone_iterations` clones × `@reads_per_clone` + first-touch reads per partial mode. Reports compressed pack bytes + per clone and p50/p95/p99 latency for clone time and per-read time. + + Begins with a capability sniff: a single `filter:{:blob, :none}` + clone. Exgit returns `{:error, {:filter_unsupported, _}}` if the + server doesn't advertise the protocol-v2 `filter` capability — that + itself is the headline finding for partial-clone support. + + Tagged `:cloudflare_perf`. Run with `mix test --include cloudflare_perf`. + Requires `CF_ACCOUNT_ID` and `CF_API_TOKEN`; auto-skipped via + `test_helper.exs` when those aren't set. + """ + + use ExUnit.Case, async: false + + @moduletag :cloudflare_perf + @moduletag timeout: 1_800_000 + + alias Exgit.CloudflareArtifacts + alias Exgit.CloudflareArtifacts.{Repo, Token} + alias Exgit.Credentials.Artifacts, as: ArtifactsCreds + alias Exgit.Object.{Blob, Commit, Tree} + alias Exgit.ObjectStore + alias Exgit.ObjectStore.Promisor + alias Exgit.{RefStore, Repository, Transport} + alias Exgit.Test.CloudflareArtifacts, as: CFEnv + + @file_count 1_000 + @file_bytes 1_024 + @clone_iterations 10 + @reads_per_clone 50 + + setup_all do + client = + CloudflareArtifacts.new( + account_id: CFEnv.account_id(), + namespace: CFEnv.namespace(), + api_token: CFEnv.api_token() + ) + + repo_name = "exgit-perf-#{System.system_time(:millisecond)}-#{rand_hex(4)}" + + on_exit(fn -> _ = CloudflareArtifacts.delete_repo(client, repo_name) end) + + {:ok, %Repo{remote: remote}} = + CloudflareArtifacts.create_repo(client, + name: repo_name, + default_branch: "main", + description: "exgit perf benchmark" + ) + + {:ok, %Token{plaintext: write_token}} = + CloudflareArtifacts.create_token(client, + repo: repo_name, + scope: "write", + ttl: 7200 + ) + + {:ok, %Token{plaintext: read_token}} = + CloudflareArtifacts.create_token(client, + repo: repo_name, + scope: "read", + ttl: 7200 + ) + + branch = "refs/heads/main" + + IO.puts("\n[perf] building seed: #{@file_count} × #{@file_bytes}B random blobs ...") + + {build_us, {seed_repo, _commit_sha, filenames}} = + :timer.tc(fn -> build_seed_commit(branch, @file_count, @file_bytes) end) + + IO.puts("[perf] seed built locally in #{us_to_ms(build_us)} ms") + + write_transport = Transport.HTTP.new(remote, auth: ArtifactsCreds.auth(write_token)) + + IO.puts("[perf] pushing seed to CF ...") + + {push_us, {:ok, %{ref_results: ref_results}}} = + :timer.tc(fn -> Exgit.push(seed_repo, write_transport, refspecs: [branch]) end) + + assert Enum.any?(ref_results, &match?({^branch, :ok}, &1)) + IO.puts("[perf] pushed in #{us_to_ms(push_us)} ms") + + {:ok, + %{ + branch: branch, + filenames: filenames, + remote: remote, + read_token: read_token + }} + end + + test "perf: clone modes against CF Artifacts", ctx do + IO.puts("\n========== CF Artifacts Clone-Mode Benchmark ==========") + + IO.puts( + "repo size: #{@file_count} files × #{@file_bytes} bytes = " <> + "#{@file_count * @file_bytes} bytes raw" + ) + + IO.puts("clones per mode: #{@clone_iterations}") + IO.puts("first-touch reads per partial-mode clone: #{@reads_per_clone}") + + IO.puts("\n--- Capability check ---") + {filter_supported?, sniff_msg} = check_filter_capability(ctx) + IO.puts(sniff_msg) + + if not filter_supported? do + IO.puts(check_lazy_capability(ctx)) + end + + eager_stats = bench_eager(ctx, @clone_iterations) + + filter_stats = + if filter_supported? do + bench_partial(ctx, :filter_blob_none, @clone_iterations, @reads_per_clone) + else + IO.puts("\n[perf] skipping filter:{:blob, :none} benchmark (capability not advertised).") + nil + end + + # Run lazy regardless: even when the server doesn't advertise + # `filter`, exgit's on-demand fetch path still sends `filter: + # blob:none` per fetch — the server may silently ignore it (and + # ship the full pack each time, which would make lazy strictly + # worse than eager) or honor it (in which case lazy works and + # the capability advertisement is just a docs lie). + lazy_stats = bench_partial(ctx, :lazy, @clone_iterations, @reads_per_clone) + + print_summary(eager_stats, filter_stats, lazy_stats) + end + + # --- Capability sniff --- + + defp check_filter_capability(ctx) do + transport = make_transport(ctx) + + case Exgit.clone(transport, filter: {:blob, :none}) do + {:ok, %Repository{object_store: %Promisor{cache_bytes: bytes} = store}} -> + n = count_cached_objects(store) + + msg = + "[perf] CF advertises `filter` capability — partial clone succeeded.\n" <> + "[perf] bootstrap fetched #{n} objects, #{bytes} compressed bytes\n" <> + "[perf] (eager would fetch #{@file_count + 2} objects: #{@file_count} blobs + 1 tree + 1 commit)" + + {true, msg} + + {:error, {:filter_unsupported, _} = err} -> + msg = + "[perf] CF does NOT advertise `filter` capability.\n" <> + "[perf] exgit error: #{inspect(err)}\n" <> + "[perf] Implication: partial-clone agent workloads pay the full pack on every clone." + + {false, msg} + + other -> + {false, "[perf] Unexpected result from filter clone: #{inspect(other)}"} + end + end + + # If filter isn't supported, lazy mode will also fail somewhere — + # but where? Document the exact failure shape so we know whether + # exgit's lazy path is dead at clone time or at first-read time + # against this server. + defp check_lazy_capability(ctx) do + transport = make_transport(ctx) + + case Exgit.clone(transport, lazy: true) do + {:error, reason} -> + "[perf] `lazy: true` clone fails immediately: #{inspect(reason)}" + + {:ok, repo} -> + # Clone succeeded (refs only). First read will trigger an + # on-demand fetch which itself uses `filter: blob:none` + # (fs.ex:446) — that's where it should die. + sample = Enum.take_random(ctx.filenames, 1) |> hd() + + case Exgit.FS.read_path(repo, ctx.branch, sample) do + {:error, reason} -> + "[perf] `lazy: true` clone OK; first read fails: #{inspect(reason)}" + + {:ok, _, _} -> + "[perf] `lazy: true` clone + first read OK — server tolerates partial fetch?" + end + end + end + + # --- Mode benchmarks --- + + defp bench_eager(ctx, n) do + IO.puts("\n--- eager mode (#{n} iterations) ---") + + samples = + for _ <- 1..n do + transport = make_transport(ctx) + {us, {:ok, repo}} = :timer.tc(fn -> Exgit.clone(transport) end) + {us, eager_object_count(repo)} + end + + {clone_times, obj_counts} = Enum.unzip(samples) + + IO.puts("[perf] eager clone time: #{summarize(clone_times)}") + + IO.puts( + "[perf] eager object count: #{format_count(obj_counts)} (expected #{@file_count + 2})" + ) + + %{mode: :eager, clone_us: clone_times, objects: obj_counts} + end + + defp bench_partial(ctx, mode, clone_n, read_n) do + label = mode_label(mode) + IO.puts("\n--- #{label} mode (#{clone_n} clones × #{read_n} reads each) ---") + + iterations = + for _ <- 1..clone_n do + transport = make_transport(ctx) + clone_opts = clone_opts_for(mode) + + {clone_us, {:ok, repo}} = :timer.tc(fn -> Exgit.clone(transport, clone_opts) end) + bootstrap_bytes = repo.object_store.cache_bytes + + sample = Enum.take_random(ctx.filenames, read_n) + + {read_times_rev, repo} = + Enum.reduce(sample, {[], repo}, fn filename, {acc, repo} -> + {us, {:ok, {_mode, _blob}, repo}} = + :timer.tc(fn -> Exgit.FS.read_path(repo, ctx.branch, filename) end) + + {[us | acc], repo} + end) + + read_us = Enum.reverse(read_times_rev) + + %{ + clone_us: clone_us, + bootstrap_bytes: bootstrap_bytes, + read_us: read_us, + final_bytes: repo.object_store.cache_bytes + } + end + + clone_times = Enum.map(iterations, & &1.clone_us) + bootstrap_bytes_list = Enum.map(iterations, & &1.bootstrap_bytes) + final_bytes_list = Enum.map(iterations, & &1.final_bytes) + first_reads = Enum.map(iterations, &hd(&1.read_us)) + warm_reads = Enum.flat_map(iterations, &tl(&1.read_us)) + all_reads = Enum.flat_map(iterations, & &1.read_us) + + IO.puts("[perf] clone time: #{summarize(clone_times)}") + IO.puts("[perf] bootstrap bytes: #{format_bytes(bootstrap_bytes_list)}") + IO.puts("[perf] final bytes: #{format_bytes(final_bytes_list)} (after #{read_n} reads)") + IO.puts("[perf] read #1 (cold): #{summarize(first_reads)}") + IO.puts("[perf] reads 2..#{read_n} (warm): #{summarize(warm_reads)}") + IO.puts("[perf] all reads: #{summarize(all_reads)}") + + %{ + mode: mode, + clone_us: clone_times, + bootstrap_bytes: bootstrap_bytes_list, + final_bytes: final_bytes_list, + first_read_us: first_reads, + warm_read_us: warm_reads, + all_read_us: all_reads + } + end + + defp clone_opts_for(:filter_blob_none), do: [filter: {:blob, :none}] + defp clone_opts_for(:lazy), do: [lazy: true] + + defp mode_label(:filter_blob_none), do: "filter:{:blob, :none}" + defp mode_label(:lazy), do: "lazy: true" + + # --- Summary --- + + defp print_summary(eager, filter, lazy) do + IO.puts("\n========== Summary ==========") + + header = pad_row(["mode", "clone p50", "clone p95", "warm read p50", "warm read p95"]) + IO.puts(header) + IO.puts(String.duplicate("-", String.length(header))) + + print_mode_row("eager", eager) + if filter, do: print_mode_row("filter:{:blob, :none}", filter) + if lazy, do: print_mode_row("lazy: true", lazy) + + IO.puts("") + end + + defp print_mode_row(label, stats) do + clone_p50 = ms_str(percentile(stats.clone_us, 50)) + clone_p95 = ms_str(percentile(stats.clone_us, 95)) + + {read_p50, read_p95} = + case Map.get(stats, :warm_read_us) do + nil -> {"-", "-"} + [] -> {"-", "-"} + warm -> {ms_str(percentile(warm, 50)), ms_str(percentile(warm, 95))} + end + + IO.puts(pad_row([label, clone_p50, clone_p95, read_p50, read_p95])) + end + + defp pad_row(cols) do + widths = [22, 12, 12, 16, 16] + + cols + |> Enum.zip(widths) + |> Enum.map_join(" ", fn {col, w} -> String.pad_trailing(col, w) end) + end + + # --- Stats helpers --- + + defp summarize([]), do: "n=0" + + defp summarize(samples) do + n = length(samples) + p50 = ms_str(percentile(samples, 50)) + p95 = ms_str(percentile(samples, 95)) + p99 = ms_str(percentile(samples, 99)) + "n=#{n} p50=#{p50} p95=#{p95} p99=#{p99}" + end + + defp percentile([_ | _] = samples, p) do + sorted = Enum.sort(samples) + n = length(sorted) + idx = trunc(:math.ceil(p / 100 * n)) - 1 + idx = max(0, min(n - 1, idx)) + Enum.at(sorted, idx) + end + + defp ms_str(us) when is_integer(us), do: "#{us_to_ms(us)} ms" + defp ms_str(other), do: to_string(other) + + defp us_to_ms(us) when is_integer(us), do: Float.round(us / 1000, 1) + + defp format_bytes(byte_list) do + nums = Enum.reject(byte_list, &is_nil/1) + + case nums do + [] -> "n/a" + [single] -> "#{single}" + _ -> "min=#{Enum.min(nums)} max=#{Enum.max(nums)} mean=#{mean(nums) |> round()}" + end + end + + defp format_count(count_list) do + nums = Enum.reject(count_list, &is_nil/1) + + case nums do + [] -> "n/a" + [n] -> "#{n}" + _ -> "min=#{Enum.min(nums)} max=#{Enum.max(nums)}" + end + end + + defp mean([]), do: 0 + defp mean(xs), do: Enum.sum(xs) / length(xs) + + # --- exgit helpers --- + + defp eager_object_count(%Repository{object_store: %ObjectStore.Memory{objects: objs}}), + do: map_size(objs) + + defp eager_object_count(_), do: nil + + defp count_cached_objects(%Promisor{cache: %ObjectStore.Memory{objects: objs}}), + do: map_size(objs) + + defp count_cached_objects(_), do: :unknown + + defp make_transport(ctx) do + Transport.HTTP.new(ctx.remote, auth: ArtifactsCreds.auth(ctx.read_token)) + end + + defp build_seed_commit(branch, file_count, file_bytes) do + store = ObjectStore.Memory.new() + + {entries, store} = + Enum.reduce(1..file_count, {[], store}, fn i, {acc, store} -> + content = :crypto.strong_rand_bytes(file_bytes) + {:ok, blob_sha, store} = ObjectStore.put(store, Blob.new(content)) + filename = "file_#{String.pad_leading(Integer.to_string(i), 5, "0")}.bin" + {[{"100644", filename, blob_sha} | acc], store} + end) + + {:ok, tree_sha, store} = ObjectStore.put(store, Tree.new(entries)) + + commit = + Commit.new( + tree: tree_sha, + parents: [], + author: "Exgit Perf 1700000000 +0000", + committer: "Exgit Perf 1700000000 +0000", + message: "perf seed: #{file_count} x #{file_bytes}B\n" + ) + + {:ok, commit_sha, store} = ObjectStore.put(store, commit) + {:ok, ref_store} = RefStore.write(RefStore.Memory.new(), branch, commit_sha, []) + + repo = %Repository{ + object_store: store, + ref_store: ref_store, + config: Exgit.Config.new(), + path: nil + } + + filenames = entries |> Enum.map(&elem(&1, 1)) |> Enum.sort() + + {repo, commit_sha, filenames} + end + + defp rand_hex(n), do: n |> :crypto.strong_rand_bytes() |> Base.encode16(case: :lower) +end diff --git a/test/test_helper.exs b/test/test_helper.exs index c3f1f2b..787ffd7 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -38,7 +38,7 @@ exclude = if Exgit.Test.CloudflareArtifacts.api_available?() do exclude else - [{:cloudflare_api, true} | exclude] + [{:cloudflare_api, true}, {:cloudflare_perf, true} | exclude] end # Live-network tiers are excluded by default; opt in via @@ -50,6 +50,7 @@ exclude = {:integration, true}, {:cloudflare, true}, {:cloudflare_api, true}, + {:cloudflare_perf, true}, {:github_private, true}, {:github_private_write, true} | exclude From aff25f41908ac6d1044951787730fb5e51d15dcf Mon Sep 17 00:00:00 2001 From: Ivar Vong Date: Wed, 1 Jul 2026 09:27:37 -0400 Subject: [PATCH 07/13] ci: add stable Elixir 1.20 / OTP 29 as the primary matrix tier Promote newest-stable Elixir 1.20 on OTP 29 to primary (owns format, credo, dialyzer, integration gates); keep 1.19/OTP-28 as a compile+test smoke and 1.17/OTP-27 as the minimum-supported tier. Elixir 1.20 supports OTP 27-29, so the pair resolves cleanly in setup-beam. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01YVcfxkfN7A7KYLmNYt9UcH --- .github/workflows/ci.yml | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 46fbee8..64b69b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,13 +20,18 @@ jobs: fail-fast: false matrix: include: - # Primary target: newest stable Elixir (1.19.x) on newest OTP (28.x). - # `erlef/setup-beam` resolves "1.19" to the latest 1.19.y.z that has - # an OTP-28 build on hex.pm. Owns the quality gates (format, - # credo, dialyzer, integration). + # Primary target: newest stable Elixir (1.20.x) on newest OTP (29.x). + # `erlef/setup-beam` resolves "1.20" to the latest 1.20.y.z that has + # an OTP-29 build. Owns the quality gates (format, credo, dialyzer, + # integration). Elixir 1.20 supports OTP 27-29. + - elixir: "1.20" + otp: "29" + primary: true + # Previous stable line — a compile + test smoke to catch regressions + # against 1.19/OTP-28. Quality gates run once, on the primary. - elixir: "1.19" otp: "28" - primary: true + primary: false # Minimum-supported version — matches the `elixir: "~> 1.17"` # constraint in mix.exs. If we ever need a 1.18+ feature, # bump both the constraint and this matrix entry. Compile + From 3cdaa49314be306232e588cdac6bb800e4af9d8e Mon Sep 17 00:00:00 2001 From: Ivar Vong Date: Wed, 1 Jul 2026 09:49:25 -0400 Subject: [PATCH 08/13] chore: pin dev toolchain to stable Elixir 1.20.2 / OTP 29 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move .tool-versions off the 1.20.0-rc.3 release candidate to stable 1.20.2 on OTP 29 — matching the CI primary tier. Verified the full gauntlet on this exact combo locally: format, credo --strict, dialyzer (0 errors, PLT rebuilt for OTP 29), and 702 tests + 52 properties all green. Also drop a dead `mean([])` clause in the CF perf test that 1.20.2's stricter unused-clause analysis flagged. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01YVcfxkfN7A7KYLmNYt9UcH --- .tool-versions | 4 ++-- test/exgit/cloudflare_artifacts_perf_test.exs | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.tool-versions b/.tool-versions index c77aa85..088edd2 100644 --- a/.tool-versions +++ b/.tool-versions @@ -1,2 +1,2 @@ -erlang 28.4 -elixir 1.20.0-rc.3-otp-28 +erlang 29.0.2 +elixir 1.20.2-otp-29 diff --git a/test/exgit/cloudflare_artifacts_perf_test.exs b/test/exgit/cloudflare_artifacts_perf_test.exs index a999ddc..00d9869 100644 --- a/test/exgit/cloudflare_artifacts_perf_test.exs +++ b/test/exgit/cloudflare_artifacts_perf_test.exs @@ -369,7 +369,8 @@ defmodule Exgit.CloudflareArtifactsPerfTest do end end - defp mean([]), do: 0 + # Only called from `format_bytes/1`'s `_` clause, i.e. with 2+ elements + # (the empty and single-element cases are handled before the call). defp mean(xs), do: Enum.sum(xs) / length(xs) # --- exgit helpers --- From af332b1b25f66f0aa2e874957137682fbaf314b2 Mon Sep 17 00:00:00 2001 From: Ivar Vong Date: Wed, 1 Jul 2026 10:50:14 -0400 Subject: [PATCH 09/13] chore(deps): bump HTTP stack to clear security advisories MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Upgrade all dependencies to latest allowed by the mix.exs constraints. Notably req 0.5.17 -> 0.6.2 and mint 1.7.1 -> 1.9.0, which resolve the HIGH decompression-bomb DoS (CVE-2026-49755), a multipart header-injection (CVE-2026-49756), and the mint HTTP/2 CONTINUATION flood (CVE-2026-49754) in the production HTTP path. req 0.6 is a 0.x-minor bump; the full suite — including the cross-origin credential-leak tests SECURITY.md calls out — passes, and dialyzer is clean. The only advisories mix hex.audit still reports are in cowlib, reached solely via the only: :test bypass dep, so they never ship in the package or reach a consumer's runtime; cowlib 2.17.1 is already the latest. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_01YVcfxkfN7A7KYLmNYt9UcH --- CHANGELOG.md | 9 +++++++++ mix.lock | 24 ++++++++++++------------ 2 files changed, 21 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d16beed..7e444c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 broken server can no longer stream unbounded refs into client memory; the transport stream halts once the cap trips. Real repos (linux, esp-idf) sit far below this. +- **Dependencies bumped to clear HTTP-stack advisories.** `req` + `0.5.17 → 0.6.2` and `mint `1.7.1 → 1.9.0` resolve the decompression- + bomb DoS (CVE-2026-49755), multipart header injection + (CVE-2026-49756), and HTTP/2 CONTINUATION flood (CVE-2026-49754). + The cross-origin credential-leak test suite was re-run against the + new `req` line. The only remaining `mix hex.audit` advisories are in + `cowlib`, reachable solely through the `only: :test` `bypass` + dependency — never part of the published package or a consumer's + runtime. ### Notes diff --git a/mix.lock b/mix.lock index 23ef125..4cd620a 100644 --- a/mix.lock +++ b/mix.lock @@ -3,27 +3,27 @@ "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, "bypass": {:hex, :bypass, "2.1.0", "909782781bf8e20ee86a9cabde36b259d44af8b9f38756173e8f5e2e1fabb9b1", [:mix], [{:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:ranch, "~> 1.3", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "d9b5df8fa5b7a6efa08384e9bbecfe4ce61c77d28a4282f79e02f1ef78d96b80"}, "chatterbox": {:hex, :ts_chatterbox, "0.15.1", "5cac4d15dd7ad61fc3c4415ce4826fc563d4643dee897a558ec4ea0b1c835c9c", [:rebar3], [{:hpack, "~> 0.3.0", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "4f75b91451338bc0da5f52f3480fa6ef6e3a2aeecfc33686d6b3d0a0948f31aa"}, - "cowboy": {:hex, :cowboy, "2.14.2", "4008be1df6ade45e4f2a4e9e2d22b36d0b5aba4e20b0a0d7049e28d124e34847", [:make, :rebar3], [{:cowlib, ">= 2.16.0 and < 3.0.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, ">= 1.8.0 and < 3.0.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "569081da046e7b41b5df36aa359be71a0c8874e5b9cff6f747073fc57baf1ab9"}, + "cowboy": {:hex, :cowboy, "2.16.1", "fa04080b602ff25c40a7700f2dc0152dbc1ba26b42093ae0fa9bb7a337d5a242", [:make, :rebar3], [{:cowlib, ">= 2.16.0 and < 3.0.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, ">= 1.8.0 and < 3.0.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "b8ea4dd317a043e3177ec840cfa3bcb47cfb41035d3abb24d954dc7d51def399"}, "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, - "cowlib": {:hex, :cowlib, "2.16.0", "54592074ebbbb92ee4746c8a8846e5605052f29309d3a873468d76cdf932076f", [:make, :rebar3], [], "hexpm", "7f478d80d66b747344f0ea7708c187645cfcc08b11aa424632f78e25bf05db51"}, - "credo": {:hex, :credo, "1.7.18", "5c5596bf7aedf9c8c227f13272ac499fe8eae6237bd326f2f07dfc173786f042", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "a189d164685fd945809e862fe76a7420c4398fa288d76257662aecb909d6b3e5"}, + "cowlib": {:hex, :cowlib, "2.17.1", "3e6053016d1ab245730f0af688755476dcedb1c25ed8fb5751f59a2bfdc0c9af", [:make, :rebar3], [], "hexpm", "ff08bd17e6dd931445b18af77315b9b5fe052407110964ad2588c686b57b5e3f"}, + "credo": {:hex, :credo, "1.7.19", "cc52129665fc7c15143d47838fda0f9cd6dac9ceced7bf4da6f85fcbfe64b12a", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "2d8bc95d5a7bb99dd2613621d4f08c6a3575c3fd4b62e6a2b48a100352a557b8"}, "ctx": {:hex, :ctx, "0.6.0", "8ff88b70e6400c4df90142e7f130625b82086077a45364a78d208ed3ed53c7fe", [:rebar3], [], "hexpm", "a14ed2d1b67723dbebbe423b28d7615eb0bdcba6ff28f2d1f1b0a7e1d4aa5fc2"}, "dialyxir": {:hex, :dialyxir, "1.4.7", "dda948fcee52962e4b6c5b4b16b2d8fa7d50d8645bbae8b8685c3f9ecb7f5f4d", [:mix], [{:erlex, ">= 0.2.8", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b34527202e6eb8cee198efec110996c25c5898f43a4094df157f8d28f27d9efe"}, "earmark_parser": {:hex, :earmark_parser, "1.4.45", "cba8369ab2a1342e419bc2760eec731b17be828941dcf494045d44766227e1d5", [:mix], [], "hexpm", "d3ec045bf122965db20c0bdb420e19ee1415843135327124918473feb4b328e8"}, - "erlex": {:hex, :erlex, "0.2.8", "cd8116f20f3c0afe376d1e8d1f0ae2452337729f68be016ea544a72f767d9c12", [:mix], [], "hexpm", "9d66ff9fedf69e49dc3fd12831e12a8a37b76f8651dd21cd45fcf5561a8a7590"}, + "erlex": {:hex, :erlex, "0.2.9", "7debbbaa9f4f368b8cd648983e0f1d7963028508e9c59e9d4ed504e94ef52a55", [:mix], [], "hexpm", "8cfffc0ec7159e6d73de2ab28a588064de80f88b2798d5cbe4482cbbc200178b"}, "ex_doc": {:hex, :ex_doc, "0.40.3", "4a972ffe64bc07dc605af487e98fc19b72a4185f55ca031b94c0552d6071c1d9", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "2756e357742fecd9749b489b85d67c9ce99c465f2e75728d9e6dc8d704b973de"}, "file_system": {:hex, :file_system, "1.1.1", "31864f4685b0148f25bd3fbef2b1228457c0c89024ad67f7a81a3ffbc0bbad3a", [:mix], [], "hexpm", "7a15ff97dfe526aeefb090a7a9d3d03aa907e100e262a0f8f7746b78f8f87a5d"}, - "finch": {:hex, :finch, "0.21.0", "b1c3b2d48af02d0c66d2a9ebfb5622be5c5ecd62937cf79a88a7f98d48a8290c", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "87dc6e169794cb2570f75841a19da99cfde834249568f2a5b121b809588a4377"}, + "finch": {:hex, :finch, "0.23.0", "e3f9287ac25a8832f848b144c2b57346aac65b205e2e0629a52adfe6507fd837", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.8", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "80e58d3f936f57e3fdf404f83a3642897ae6d9fb642934e46da4d8fe761b99d5"}, "gproc": {:hex, :gproc, "0.9.1", "f1df0364423539cf0b80e8201c8b1839e229e5f9b3ccb944c5834626998f5b8c", [:rebar3], [], "hexpm", "905088e32e72127ed9466f0bac0d8e65704ca5e73ee5a62cb073c3117916d507"}, "grpcbox": {:hex, :grpcbox, "0.17.1", "6e040ab3ef16fe699ffb513b0ef8e2e896da7b18931a1ef817143037c454bcce", [:rebar3], [{:acceptor_pool, "~> 1.0.0", [hex: :acceptor_pool, repo: "hexpm", optional: false]}, {:chatterbox, "~> 0.15.1", [hex: :ts_chatterbox, repo: "hexpm", optional: false]}, {:ctx, "~> 0.6.0", [hex: :ctx, repo: "hexpm", optional: false]}, {:gproc, "~> 0.9.1", [hex: :gproc, repo: "hexpm", optional: false]}], "hexpm", "4a3b5d7111daabc569dc9cbd9b202a3237d81c80bf97212fbc676832cb0ceb17"}, "hpack": {:hex, :hpack_erl, "0.3.0", "2461899cc4ab6a0ef8e970c1661c5fc6a52d3c25580bc6dd204f84ce94669926", [:rebar3], [], "hexpm", "d6137d7079169d8c485c6962dfe261af5b9ef60fbc557344511c1e65e3d95fb0"}, "hpax": {:hex, :hpax, "1.0.3", "ed67ef51ad4df91e75cc6a1494f851850c0bd98ebc0be6e81b026e765ee535aa", [:mix], [], "hexpm", "8eab6e1cfa8d5918c2ce4ba43588e894af35dbd8e91e6e55c817bca5847df34a"}, - "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, + "jason": {:hex, :jason, "1.4.5", "2e3a008590b0b8d7388c20293e9dcc9cf3e5d642fd2a114e4cbbb52e595d940a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "b0c823996102bcd0239b3c2444eb00409b72f6a140c1950bc8b457d836b30684"}, "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, "makeup_erlang": {:hex, :makeup_erlang, "1.1.0", "835f7e60792e08824cda445639555d7bf1bbbddb1b60b306e33cb6f6db24dc74", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "1cd6780fb1dd1a03979abaed0fe82712b0625118fd5257d3ebbf73f960c73c3c"}, "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, - "mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"}, + "mint": {:hex, :mint, "1.9.0", "d6f534c2a3e98b2a8cc749b4796eb77e9e3af79a76f96e4c74035a827de0d318", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "007154c7d8c43916aed3c93afd1f11aebbaa9c5ff4b7ba55ebe0d17ee0296042"}, "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, @@ -31,14 +31,14 @@ "opentelemetry_api": {:hex, :opentelemetry_api, "1.5.0", "1a676f3e3340cab81c763e939a42e11a70c22863f645aa06aafefc689b5550cf", [:mix, :rebar3], [], "hexpm", "f53ec8a1337ae4a487d43ac89da4bd3a3c99ddf576655d071deed8b56a2d5dda"}, "opentelemetry_exporter": {:hex, :opentelemetry_exporter, "1.10.0", "972e142392dbfa679ec959914664adefea38399e4f56ceba5c473e1cabdbad79", [:rebar3], [{:grpcbox, ">= 0.0.0", [hex: :grpcbox, repo: "hexpm", optional: false]}, {:opentelemetry, "~> 1.7.0", [hex: :opentelemetry, repo: "hexpm", optional: false]}, {:opentelemetry_api, "~> 1.5.0", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:tls_certificate_check, "~> 1.18", [hex: :tls_certificate_check, repo: "hexpm", optional: false]}], "hexpm", "33a116ed7304cb91783f779dec02478f887c87988077bfd72840f760b8d4b952"}, "opentelemetry_telemetry": {:hex, :opentelemetry_telemetry, "1.1.2", "410ab4d76b0921f42dbccbe5a7c831b8125282850be649ee1f70050d3961118a", [:mix, :rebar3], [{:opentelemetry_api, "~> 1.3", [hex: :opentelemetry_api, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.1", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "641ab469deb181957ac6d59bce6e1321d5fe2a56df444fc9c19afcad623ab253"}, - "plug": {:hex, :plug, "1.19.1", "09bac17ae7a001a68ae393658aa23c7e38782be5c5c00c80be82901262c394c0", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "560a0017a8f6d5d30146916862aaf9300b7280063651dd7e532b8be168511e62"}, - "plug_cowboy": {:hex, :plug_cowboy, "2.8.1", "5aa391a5e8d1ac3192e36a3bcaff12b5fd6ef6c7e29b53a38e63a860783e77d0", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "4c200288673d5bc86a0ab7dc6a2a069176a74e5d573ef62740a1c517458a5f26"}, + "plug": {:hex, :plug, "1.20.2", "adbee2441232412e37fbb357fd5e4cd533fdd253b29f2e1992262b0f1fb01462", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b16baf55877d60891002ffc1ce0b3ff7d6f30a38a23e02e4d4293c4ac266f136"}, + "plug_cowboy": {:hex, :plug_cowboy, "2.9.0", "87e21e0d9054ced99c36d128f49e3ea2cd8b745fffb97de50bff99706087af4f", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.18", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "2002bafba4f3a45b55a58e68d70211b153a7ed18d37edb1ceb6e96e7a92c422e"}, "plug_crypto": {:hex, :plug_crypto, "2.1.1", "19bda8184399cb24afa10be734f84a16ea0a2bc65054e23a62bb10f06bc89491", [:mix], [], "hexpm", "6470bce6ffe41c8bd497612ffde1a7e4af67f36a15eea5f921af71cf3e11247c"}, "ranch": {:hex, :ranch, "1.8.1", "208169e65292ac5d333d6cdbad49388c1ae198136e4697ae2f474697140f201c", [:make, :rebar3], [], "hexpm", "aed58910f4e21deea992a67bf51632b6d60114895eb03bb392bb733064594dd0"}, - "req": {:hex, :req, "0.5.17", "0096ddd5b0ed6f576a03dde4b158a0c727215b15d2795e59e0916c6971066ede", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "0b8bc6ffdfebbc07968e59d3ff96d52f2202d0536f10fef4dc11dc02a2a43e39"}, + "req": {:hex, :req, "0.6.2", "b9b2024f35bcf60a92cc8cad2eaaf9d4e7aace463ff74be1afe5986830184413", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.21", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "cc9cd30a2ddd04989929b887178e1610c940456d962c6c3a52df6146d2eef9bf"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, "stream_data": {:hex, :stream_data, "1.3.0", "bde37905530aff386dea1ddd86ecbf00e6642dc074ceffc10b7d4e41dfd6aac9", [:mix], [], "hexpm", "3cc552e286e817dca43c98044c706eec9318083a1480c52ae2688b08e2936e3c"}, - "telemetry": {:hex, :telemetry, "1.4.1", "ab6de178e2b29b58e8256b92b382ea3f590a47152ca3651ea857a6cae05ac423", [:rebar3], [], "hexpm", "2172e05a27531d3d31dd9782841065c50dd5c3c7699d95266b2edd54c2dafa1c"}, - "tls_certificate_check": {:hex, :tls_certificate_check, "1.32.1", "f90f9647668f7af804ec63c4e67f3799931166caf8264b81f37c1d863a36ae1d", [:rebar3], [{:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "e78a157966456b500a87a2fc29cffcd6dcfb5a26348c8372a2c5c0a8e5797f51"}, + "telemetry": {:hex, :telemetry, "1.4.2", "a0cb522801dffb1c49fe6e30561badffc7b6d0e180db1300df759faa22062855", [:rebar3], [], "hexpm", "928f6495066506077862c0d1646609eed891a4326bee3126ba54b60af61febb1"}, + "tls_certificate_check": {:hex, :tls_certificate_check, "1.33.0", "01e0822a2ebb0b207c4964e8d32ad8b1b8ce1caf58f446e53f816d06db953de4", [:rebar3], [{:ssl_verify_fun, "~> 1.1", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "cab9a7439e2dbfe91b38104f2d8a4b6d61dbc4d3a5ad59ac364713a88c6cfd9b"}, "vfs": {:hex, :vfs, "0.1.0", "7bac0a4cda3e4fc237d9c798116c2b23d7ece1f9f87651062b0f8f071f0b448d", [:mix], [{:telemetry, "~> 1.3", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "72f372bb60b4b4cf8fe67098c8559f007f115cb2fe3f94a05cdf9f18c71b0dd0"}, } From d987b7fb1950722107b52a57b7d1a349bcf4a623 Mon Sep 17 00:00:00 2001 From: Ivar Vong Date: Wed, 1 Jul 2026 16:43:48 -0400 Subject: [PATCH 10/13] fix(transport): malformed wire data as error values, not crashes Review findings for 0.1.0: - PktLine.decode_all/decode_stream return {:error, {:malformed_pkt_line, snippet}} instead of raising, so Transport.HTTP.capabilities/1 and push/4 no longer crash on hostile or broken server bytes ({:error, {:malformed_response, _}}). - PktLine.encode/1 raises ArgumentError past git's 65516-byte pkt payload max instead of silently emitting corrupt framing. - Stream finalization checks accumulated domain errors before the decoder's truncation artifact, so the ls-refs ref-cap ({:too_many_refs, cap}), sideband channel-3 errors, and StreamParser rejections surface instead of {:truncated, n}. - do_fetch propagates non-binary StreamParser errors. - New :max_refs option on Transport.HTTP.new/2 (default 1,000,000) makes the ls-refs cap tunable and testable; cap + boundary now covered by chunked-streaming regression tests. - Push-span and ref_rejected credential redaction now tested. - @doc/@spec on capabilities/ls_refs/fetch/push. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01VEo6srWr3aFwhtwaXDTCZZ --- lib/exgit/pkt_line.ex | 78 +++++-- lib/exgit/pkt_line/decoder.ex | 9 +- lib/exgit/transport/http.ex | 210 ++++++++++++++---- test/exgit/pkt_line/decoder_test.exs | 4 +- test/exgit/pkt_line_test.exs | 48 ++++ test/exgit/transport/http_refs_cap_test.exs | 114 ++++++++++ test/exgit/transport/http_streaming_test.exs | 20 ++ .../http_telemetry_redaction_test.exs | 91 ++++++++ test/exgit/transport/http_test.exs | 56 +++++ 9 files changed, 573 insertions(+), 57 deletions(-) create mode 100644 test/exgit/transport/http_refs_cap_test.exs diff --git a/lib/exgit/pkt_line.ex b/lib/exgit/pkt_line.ex index bcf3b47..6fee2e3 100644 --- a/lib/exgit/pkt_line.ex +++ b/lib/exgit/pkt_line.ex @@ -12,17 +12,44 @@ defmodule Exgit.PktLine do * `0002` — response-end `encode/1` emits the framed bytes for a data payload; - `decode_stream/1` parses a concatenated pkt-line stream into a - list of `{:data, bin} | :flush | :delim | :response_end` tokens. - Decoders never raise on truncated input; see - `Exgit.PropertiesTest` for the roundtrip property. + `decode_stream/1` parses a concatenated pkt-line stream into + `{:data, bin} | :flush | :delim | :response_end` tokens. + Decoders never raise on malformed or truncated input — bad + framing surfaces as an `{:error, {:malformed_pkt_line, snippet}}` + value; see `Exgit.PropertiesTest` for the fuzz property. """ @type packet :: {:data, binary()} | :flush | :delim | :response_end + @type decode_error :: {:error, {:malformed_pkt_line, binary()}} + # Largest payload a single pkt-line can carry: git caps the framed + # length at 65520 bytes (LARGE_PACKET_MAX), minus the 4-byte header. + @max_payload 65_516 + + # How many bytes of the offending input to include in a + # `:malformed_pkt_line` error detail. + @error_snippet_bytes 40 + + @doc """ + Frame a data payload as a pkt-line. + + Payloads are capped at #{@max_payload} bytes — git's + `LARGE_PACKET_MAX` (65520) minus the 4-byte length header. Raises + `ArgumentError` for oversized payloads: the length header is 4 hex + digits, so silently encoding anything larger would corrupt the + wire stream. Callers sending bulk data (e.g. sideband frames) must + chunk it below the limit. + """ @spec encode(iodata()) :: iolist() def encode(data) do payload = IO.iodata_to_binary(data) + + if byte_size(payload) > @max_payload do + raise ArgumentError, + "pkt-line payload is #{byte_size(payload)} bytes; " <> + "max is #{@max_payload} (LARGE_PACKET_MAX minus the 4-byte header)" + end + len = byte_size(payload) + 4 [len |> Integer.to_string(16) |> String.pad_leading(4, "0"), payload] end @@ -36,9 +63,20 @@ defmodule Exgit.PktLine do @spec response_end() :: binary() def response_end, do: "0002" + @doc """ + Lazily decode a concatenated pkt-line stream. + + Yields `t:packet/0` tokens. On malformed or truncated input the + stream yields a final `{:error, {:malformed_pkt_line, snippet}}` + token — `snippet` is up to #{@error_snippet_bytes} bytes of the + offending input — and halts. Never raises on hostile bytes. + """ @spec decode_stream(binary()) :: Enumerable.t() def decode_stream(bytes) when is_binary(bytes) do Stream.unfold(bytes, fn + :halted -> + nil + <<>> -> nil @@ -51,25 +89,39 @@ defmodule Exgit.PktLine do <<"0002", rest::binary>> -> {:response_end, rest} - <> -> + <> = buf -> with {len, ""} <- Integer.parse(hex_len, 16), true <- len >= 4, payload_len = len - 4, - <> <- rest do - {{:data, payload}, rest} + <> <- rest do + {{:data, payload}, tail} else - _ -> - raise ArgumentError, - "malformed pkt-line at: #{inspect(binary_part(hex_len <> rest, 0, min(byte_size(hex_len <> rest), 40)))}" + _ -> {{:error, {:malformed_pkt_line, snippet(buf)}}, :halted} end truncated -> - raise ArgumentError, "truncated pkt-line header: #{inspect(truncated)}" + {{:error, {:malformed_pkt_line, snippet(truncated)}}, :halted} end) end - @spec decode_all(binary()) :: [packet()] + @doc """ + Eagerly decode a concatenated pkt-line stream. + + Returns the packet list, or `{:error, {:malformed_pkt_line, snippet}}` + when the input contains bad or truncated framing. A malformed stream + is rejected whole — packets decoded before the bad framing are + discarded. Never raises on hostile bytes. + """ + @spec decode_all(binary()) :: [packet()] | decode_error() def decode_all(bytes) when is_binary(bytes) do - decode_stream(bytes) |> Enum.to_list() + tokens = bytes |> decode_stream() |> Enum.to_list() + + case List.last(tokens) do + {:error, _} = error -> error + _ -> tokens + end end + + defp snippet(bytes), + do: binary_part(bytes, 0, min(byte_size(bytes), @error_snippet_bytes)) end diff --git a/lib/exgit/pkt_line/decoder.ex b/lib/exgit/pkt_line/decoder.ex index 63a1a48..956a4cc 100644 --- a/lib/exgit/pkt_line/decoder.ex +++ b/lib/exgit/pkt_line/decoder.ex @@ -36,11 +36,12 @@ defmodule Exgit.PktLine.Decoder do Feed a chunk of bytes into the decoder. Returns the updated decoder and any complete packets that became decodable from `buffer ++ chunk`. - Returns `{:error, reason}` on malformed framing (length-prefix that - is not valid hex or claims a length below the 4-byte header). + Returns `{:error, {:malformed_pkt_line, hex}}` on malformed framing + (a length-prefix that is not valid hex or claims a length below the + 4-byte header). """ @spec feed(t(), binary()) :: - {:ok, t(), [PktLine.packet()]} | {:error, term()} + {:ok, t(), [PktLine.packet()]} | PktLine.decode_error() def feed(%__MODULE__{buffer: buf}, chunk) when is_binary(chunk) do drain(<>, []) end @@ -78,7 +79,7 @@ defmodule Exgit.PktLine.Decoder do end _ -> - {:error, {:malformed_length, hex}} + {:error, {:malformed_pkt_line, hex}} end end diff --git a/lib/exgit/transport/http.ex b/lib/exgit/transport/http.ex index 99b2723..5c726bb 100644 --- a/lib/exgit/transport/http.ex +++ b/lib/exgit/transport/http.ex @@ -33,6 +33,12 @@ defmodule Exgit.Transport.HTTP do alias Exgit.PktLine alias Exgit.PktLine.Decoder + # Default backstop cap on refs accepted from a single ls-refs response. + # Real repos top out in the tens of thousands (linux, esp-idf); this + # bound exists only to stop a hostile server from streaming unbounded + # refs into client memory. Tunable per-transport via `:max_refs`. + @max_refs 1_000_000 + @enforce_keys [:url] defstruct [ :url, @@ -81,6 +87,9 @@ defmodule Exgit.Transport.HTTP do # git hosts do redirect (canonicalization, repo renames) — set # `:same_origin` for hosts where this is common. redirect: false, + # Cap on refs accepted from a single ls-refs response before the + # transport aborts with {:error, {:too_many_refs, cap}}. + max_refs: @max_refs, # Cached server capabilities (protocol v2 advertisements). `nil` # means "not yet discovered"; an `{:ok, caps}` / `{:error, _}` # tuple means the result is memoized. A fresh struct has `nil` @@ -100,6 +109,30 @@ defmodule Exgit.Transport.HTTP do @type t :: %__MODULE__{url: String.t(), auth: auth(), user_agent: String.t()} + @doc """ + Build a transport for the git smart-HTTP remote at `url`. + + ## Options + + * `:auth` — credentials; a bare auth tuple (auto-wrapped in a + host-bound `%Exgit.Credentials{}`) or an explicit + `%Exgit.Credentials{}` struct. See the module doc. + * `:user_agent` — the `user-agent` header value. + * `:connect_timeout` — connect timeout in milliseconds + (default 10_000). + * `:receive_timeout` — response-body timeout in milliseconds + (default 300_000); `:infinity` disables. + * `:verify_tls` — TLS peer verification (default `true`). + * `:connect_options` — extra transport options merged into the + library's TLS defaults (custom CA bundles, mTLS, SNI). + * `:redirect` — redirect policy: `false` (default), + `:same_origin`, or `:follow`. + * `:max_refs` — cap on refs accepted from a single ls-refs + response (default #{@max_refs}). `ls_refs/2` aborts with + `{:error, {:too_many_refs, cap}}` when a server exceeds it — + a backstop against a hostile server streaming unbounded refs + into client memory. + """ @spec new(String.t(), keyword()) :: t() def new(url, opts \\ []) do trimmed_url = String.trim_trailing(url, "/") @@ -112,7 +145,8 @@ defmodule Exgit.Transport.HTTP do receive_timeout: Keyword.get(opts, :receive_timeout, defaults.receive_timeout), verify_tls: Keyword.get(opts, :verify_tls, defaults.verify_tls), connect_options: Keyword.get(opts, :connect_options, defaults.connect_options), - redirect: Keyword.get(opts, :redirect, defaults.redirect) + redirect: Keyword.get(opts, :redirect, defaults.redirect), + max_refs: Keyword.get(opts, :max_refs, defaults.max_refs) ) end @@ -141,6 +175,20 @@ defmodule Exgit.Transport.HTTP do end end + @doc """ + Discover the server's protocol-v2 capability advertisement. + + Performs the `info/refs?service=git-upload-pack` discovery GET and + parses the advertisement into a map keyed by capability name + (string keys; well-known entries like `:version` get structured + values). Returns `{:error, :server_does_not_support_v2}` for + protocol-v1-only servers, `{:error, {:malformed_response, reason}}` + when the body is not valid pkt-line framing. + + A struct carrying a memoized `capabilities_cache` (see + `capabilities_cached/1`) returns the cached result without HTTP. + """ + @spec capabilities(t()) :: {:ok, map()} | {:error, term()} def capabilities(%__MODULE__{capabilities_cache: {:ok, _} = cached}), do: cached def capabilities(%__MODULE__{capabilities_cache: {:error, _} = cached}), do: cached @@ -148,10 +196,7 @@ defmodule Exgit.Transport.HTTP do # Not cached — discover, but don't try to mutate the struct (pure # value). Callers who want memoization across many requests can # use `capabilities_cached/1` which returns an updated struct. - case discover(t, "git-upload-pack") do - {:ok, caps} -> {:ok, caps} - error -> error - end + discover(t, "git-upload-pack") end @doc """ @@ -171,11 +216,29 @@ defmodule Exgit.Transport.HTTP do def capabilities_cached(%__MODULE__{capabilities_cache: cached} = t), do: {cached, t} - # Backstop cap on refs accepted from a single ls-refs response. Real repos - # top out in the tens of thousands (linux, esp-idf); this bound exists only - # to stop a hostile server from streaming unbounded refs into client memory. - @max_refs 1_000_000 - + @doc """ + List the remote's refs via the protocol-v2 `ls-refs` command. + + Returns `{:ok, refs, meta}` where `refs` is a list of + `{ref_name, sha}` 2-tuples and `meta` carries protocol-v2 + side-channel data (`:head` symref target, `:peeled` tag targets) — + see `t:Exgit.Transport.ls_refs_meta/0`. Invalid ref names from the + server are dropped (with `[:exgit, :security, :ref_rejected]` + telemetry); more than `max_refs` refs aborts with + `{:error, {:too_many_refs, cap}}`. + + ## Options + + * `:prefix` — ref-prefix filter or list of filters + (e.g. `"refs/heads/"`); default `[]` (all refs). + * `:symrefs` — ask for symref targets, revealing where HEAD + points (default `true`). + * `:peeled` — ask for `peeled:` attributes on annotated + tags (default `true`). + """ + @spec ls_refs(t(), keyword()) :: + {:ok, [Exgit.Transport.ref_entry()], Exgit.Transport.ls_refs_meta()} + | {:error, term()} def ls_refs(%__MODULE__{} = t, opts \\ []) do Exgit.Telemetry.span( [:exgit, :transport, :ls_refs], @@ -220,7 +283,7 @@ defmodule Exgit.Transport.HTTP do # linux), this keeps the transport's memory bound flat in ref count. # Map-shaped accumulator (not a tuple) so the streaming loop's # `%{error: e}` halt check fires the moment the ref cap trips. - init_acc = %{refs: [], meta: %{peeled: %{}}, count: 0, error: nil} + init_acc = %{refs: [], meta: %{peeled: %{}}, count: 0, error: nil, max_refs: t.max_refs} # Redact before it reaches `keep_ref?`'s security telemetry, which # echoes the source URL. source_url = redact_url(t.url) @@ -321,10 +384,10 @@ defmodule Exgit.Transport.HTTP do # Add an entry to the accumulator and lift any attributes into `meta`. # HEAD's `symref-target` becomes `meta.head`; annotated tags' `peeled` - # target becomes `meta.peeled[tag_name]`. Past `@max_refs` we stop + # target becomes `meta.peeled[tag_name]`. Past `acc.max_refs` we stop # appending and set `acc.error` so the transport stream loop halts — # a hostile server can't grow the ref list without bound. - defp add_ref(%{refs: refs, meta: meta, count: count} = acc, ref, sha, attrs) do + defp add_ref(%{refs: refs, meta: meta, count: count, max_refs: max_refs} = acc, ref, sha, attrs) do meta = case Map.get(attrs, :symref_target) do nil -> meta @@ -341,8 +404,8 @@ defmodule Exgit.Transport.HTTP do peeled_sha -> put_in(meta, [:peeled, ref], peeled_sha) end - if count + 1 > @max_refs do - %{acc | meta: meta, error: {:too_many_refs, @max_refs}} + if count + 1 > max_refs do + %{acc | meta: meta, error: {:too_many_refs, max_refs}} else %{acc | refs: [{ref, sha} | refs], meta: meta, count: count + 1} end @@ -368,6 +431,29 @@ defmodule Exgit.Transport.HTTP do end) end + @doc """ + Fetch a pack for the given `wants` (20-byte SHAs) via the + protocol-v2 `fetch` command. + + Returns `{:ok, pack_bytes, summary}`. With `:object_store` the + pack is stream-parsed straight into the store — `pack_bytes` is + `<<>>` and `summary` carries `:objects` and the updated `:store`. + Without it, `pack_bytes` is the raw pack for the caller to parse. + Sideband channel-3 server messages surface as + `{:error, {:server_error, msg}}`. + + ## Options + + * `:haves` — SHAs the client already has, for negotiation. + * `:depth` — shallow-clone depth (`deepen`). + * `:filter` — partial-clone filter spec (e.g. `"blob:none"`). + * `:object_store` — an `Exgit.ObjectStore` to stream objects + into as they arrive (bounded memory on multi-GB packs). + * `:sideband`, `:thin_pack`, `:ofs_delta` — explicit feature + overrides; by default they're negotiated from the server's + advertised capabilities. + """ + @spec fetch(t(), [binary()], keyword()) :: {:ok, binary(), map()} | {:error, term()} def fetch(%__MODULE__{} = t, wants, opts \\ []) do Exgit.Telemetry.span( [:exgit, :transport, :fetch], @@ -473,9 +559,15 @@ defmodule Exgit.Transport.HTTP do init = init_fetch_state(Keyword.get(opts, :sideband), object_store) case stream_upload_pack(t, body, init, &handle_fetch_packet/2) do + # Binary errors are sideband channel-3 server messages. {:ok, %{error: msg}} when is_binary(msg) -> {:error, {:server_error, msg}} + # Anything else non-nil is a structured client-side reason + # (e.g. a StreamParser rejection) — pass it through as-is. + {:ok, %{error: reason}} when not is_nil(reason) -> + {:error, reason} + # Streaming path: finalise the parser and return the updated store. {:ok, %{parser: %StreamParser{} = parser}} -> finalize_stream_parse(parser) @@ -617,6 +709,19 @@ defmodule Exgit.Transport.HTTP do end end + @doc """ + Push ref `updates` and the accompanying `pack_bytes` via + `git-receive-pack`. + + Each update is a `{ref, old_sha, new_sha}` 3-tuple (20-byte SHAs; + `nil` means the all-zero SHA, i.e. create/delete). Returns + `{:ok, %{ref_results: [{ref, :ok | :error}]}}` parsed from the + server's report-status response, or + `{:error, {:malformed_response, reason}}` when the report is not + valid pkt-line framing. + """ + @spec push(t(), [Exgit.Transport.ref_update()], binary(), keyword()) :: + {:ok, %{ref_results: [{String.t(), :ok | :error}]}} | {:error, term()} def push(%__MODULE__{} = t, updates, pack_bytes, opts \\ []) do Exgit.Telemetry.span( [:exgit, :transport, :push], @@ -651,7 +756,7 @@ defmodule Exgit.Transport.HTTP do case do_request(:post, url, headers, body, t) do {:ok, %{status: status, body: resp_body}} when status in 200..299 -> - {:ok, parse_push_report(resp_body)} + parse_push_report(resp_body) {:ok, %{status: status, body: resp_body}} -> {:error, {:http_error, status, resp_body}} @@ -687,8 +792,13 @@ defmodule Exgit.Transport.HTTP do # untrusted server input. Well-known capabilities additionally expose # structured values (e.g. :version -> integer). defp parse_capabilities(body) do - packets = PktLine.decode_all(body) + case PktLine.decode_all(body) do + {:error, reason} -> {:error, {:malformed_response, reason}} + packets -> parse_capability_packets(packets) + end + end + defp parse_capability_packets(packets) do caps = packets |> Enum.flat_map(fn @@ -722,18 +832,21 @@ defmodule Exgit.Transport.HTTP do # --- Push response parsing --- defp parse_push_report(body) do - packets = PktLine.decode_all(body) - - ref_results = - packets - |> Enum.flat_map(fn - {:data, "unpack ok\n"} -> [] - {:data, <<"ok ", ref::binary>>} -> [{String.trim(ref), :ok}] - {:data, <<"ng ", rest::binary>>} -> [{String.trim(rest), :error}] - _ -> [] - end) - - %{ref_results: ref_results} + case PktLine.decode_all(body) do + {:error, reason} -> + {:error, {:malformed_response, reason}} + + packets -> + ref_results = + Enum.flat_map(packets, fn + {:data, "unpack ok\n"} -> [] + {:data, <<"ok ", ref::binary>>} -> [{String.trim(ref), :ok}] + {:data, <<"ng ", rest::binary>>} -> [{String.trim(rest), :error}] + _ -> [] + end) + + {:ok, %{ref_results: ref_results}} + end end # --- HTTP helpers --- @@ -795,9 +908,7 @@ defmodule Exgit.Transport.HTTP do case Req.request(req_opts) do {:ok, %{status: status, private: %{exgit_stream: state}}} when status in 200..299 -> - with :ok <- Decoder.finalize(state.decoder) do - if state.error, do: {:error, state.error}, else: {:ok, state.handler_acc} - end + finalize_stream_state(state) {:ok, %{status: status, private: %{exgit_stream: state}}} -> {:error, {:http_error, status, state.error_body}} @@ -810,6 +921,35 @@ defmodule Exgit.Transport.HTTP do end end + # Resolve a finished 2xx stream into the caller's result. An error + # accumulated mid-stream — a decode failure on `state.error`, or a + # handler/domain error on the accumulator (sideband channel-3 + # message, the ls-refs cap, a StreamParser rejection) — halts the + # body early, which legitimately leaves partial bytes in the + # decoder. So the accumulated error must win over the finalize + # truncation artifact, or the real reason gets masked by + # `{:truncated, _}`. + defp finalize_stream_state(state) do + cond do + state.error != nil -> + {:error, state.error} + + acc_error(state.handler_acc) != nil -> + {:ok, state.handler_acc} + + true -> + case Decoder.finalize(state.decoder) do + :ok -> {:ok, state.handler_acc} + {:error, _} = err -> err + end + end + end + + # Handler accumulators are caller-shaped; both ls-refs and fetch use + # a map with an `:error` slot. Anything else has no domain error. + defp acc_error(%{error: e}), do: e + defp acc_error(_), do: nil + # One streaming step: feed `chunk` through either the error buffer # (non-2xx) or the pkt-line decoder + handler (2xx), and stash the # updated state in `resp.private.exgit_stream`. @@ -1010,8 +1150,6 @@ defmodule Exgit.Transport.HTTP do end defimpl Inspect, for: Exgit.Transport.HTTP do - import Inspect.Algebra - # Credentials live on %HTTP{}. Default Inspect would dump them into any # crash log, SASL report, or IEx session. Always redact. def inspect(%Exgit.Transport.HTTP{} = t, opts) do @@ -1025,10 +1163,6 @@ defimpl Inspect, for: Exgit.Transport.HTTP do defp redact({:header, name, _}), do: {:header, name, "***"} defp redact({:callback, _}), do: {:callback, :fun} defp redact(_), do: :redacted - - # Suppress unused-import warning from concat/2 et al — we keep the - # import in case future formatting needs it. - _ = &concat/2 end defimpl Exgit.Transport, for: Exgit.Transport.HTTP do diff --git a/test/exgit/pkt_line/decoder_test.exs b/test/exgit/pkt_line/decoder_test.exs index ad7852a..28eba61 100644 --- a/test/exgit/pkt_line/decoder_test.exs +++ b/test/exgit/pkt_line/decoder_test.exs @@ -84,13 +84,13 @@ defmodule Exgit.PktLine.DecoderTest do end test "malformed length returns error" do - assert {:error, {:malformed_length, "ZZZZ"}} = + assert {:error, {:malformed_pkt_line, "ZZZZ"}} = Decoder.feed(Decoder.new(), "ZZZZpayload") end test "length below header size returns error" do # 0003 claims 3 bytes — less than the 4-byte header. - assert {:error, {:malformed_length, "0003"}} = + assert {:error, {:malformed_pkt_line, "0003"}} = Decoder.feed(Decoder.new(), "0003") end end diff --git a/test/exgit/pkt_line_test.exs b/test/exgit/pkt_line_test.exs index 723744d..ddbb559 100644 --- a/test/exgit/pkt_line_test.exs +++ b/test/exgit/pkt_line_test.exs @@ -12,6 +12,20 @@ defmodule Exgit.PktLineTest do test "encodes an empty payload" do assert IO.iodata_to_binary(PktLine.encode("")) == "0004" end + + test "encodes a payload at exactly the 65516-byte max" do + payload = :binary.copy("a", 65_516) + # 65516 + 4 = 65520 = 0xFFF0, git's LARGE_PACKET_MAX. + assert IO.iodata_to_binary(PktLine.encode(payload)) == "FFF0" <> payload + end + + test "raises ArgumentError for a payload over the 65516-byte max" do + payload = :binary.copy("a", 65_517) + + assert_raise ArgumentError, ~r/65517 bytes; max is 65516/, fn -> + PktLine.encode(payload) + end + end end describe "special packets" do @@ -62,6 +76,40 @@ defmodule Exgit.PktLineTest do end end + describe "malformed input" do + test "non-hex length header returns an error tuple" do + assert {:error, {:malformed_pkt_line, "ZZZZgarbage"}} = + PktLine.decode_all("ZZZZgarbage") + end + + test "truncated header returns an error tuple" do + assert {:error, {:malformed_pkt_line, "00"}} = PktLine.decode_all("00") + end + + test "truncated payload returns an error tuple" do + # Header claims 9 bytes total but only 3 payload bytes follow. + assert {:error, {:malformed_pkt_line, "0009hel"}} = PktLine.decode_all("0009hel") + end + + test "a malformed tail rejects the whole stream" do + encoded = IO.iodata_to_binary([PktLine.encode("good\n"), "ZZZZ"]) + assert {:error, {:malformed_pkt_line, "ZZZZ"}} = PktLine.decode_all(encoded) + end + + test "decode_stream yields packets, then the error token, then halts" do + encoded = IO.iodata_to_binary([PktLine.encode("good\n"), "ZZZZ"]) + + assert [{:data, "good\n"}, {:error, {:malformed_pkt_line, "ZZZZ"}}] = + Enum.to_list(PktLine.decode_stream(encoded)) + end + + test "error snippet is capped at 40 bytes" do + garbage = "Z" <> :binary.copy("y", 100) + assert {:error, {:malformed_pkt_line, snippet}} = PktLine.decode_all(garbage) + assert byte_size(snippet) == 40 + end + end + describe "round-trip" do property "encode then decode preserves payload" do check all( diff --git a/test/exgit/transport/http_refs_cap_test.exs b/test/exgit/transport/http_refs_cap_test.exs new file mode 100644 index 0000000..be73e88 --- /dev/null +++ b/test/exgit/transport/http_refs_cap_test.exs @@ -0,0 +1,114 @@ +defmodule Exgit.Transport.HttpRefsCapTest do + @moduledoc """ + Exercises the `:max_refs` backstop on `Exgit.Transport.HTTP.ls_refs/2` + against a server that streams the ls-refs response one pkt-line per + chunk — the shape a hostile server would use to grow the client's + ref list without bound. + + The cap must surface as `{:error, {:too_many_refs, cap}}`, not as a + `{:truncated, _}` decode artifact: the client halts the body early + when the cap trips, which legitimately leaves partial bytes in the + pkt-line decoder, and the accumulated domain error has to win. + + Uses a raw `:gen_tcp` server (like `Exgit.Transport.HttpTest`) + rather than Bypass: the cap test hangs up mid-response by design, + which Bypass's plug monitoring reports as a `:shutdown` failure. + """ + + use ExUnit.Case, async: true + + alias Exgit.PktLine + alias Exgit.Transport.HTTP + + describe "ls_refs/2 with :max_refs" do + test "a server streaming 100 refs past a cap of 50 aborts with {:too_many_refs, 50}" do + serve_chunked_ls_refs(100, fn port -> + t = HTTP.new("http://127.0.0.1:#{port}", max_refs: 50) + + assert HTTP.ls_refs(t) == {:error, {:too_many_refs, 50}} + end) + end + + test "exactly cap refs succeeds with all refs present (boundary)" do + serve_chunked_ls_refs(50, fn port -> + t = HTTP.new("http://127.0.0.1:#{port}", max_refs: 50) + + assert {:ok, refs, _meta} = HTTP.ls_refs(t) + assert length(refs) == 50 + assert Enum.map(refs, &elem(&1, 0)) == for(i <- 1..50, do: ref_name(i)) + assert {ref_name(50), sha(50)} == List.last(refs) + end) + end + + test "the default cap is the documented 1_000_000 backstop" do + assert HTTP.new("http://localhost").max_refs == 1_000_000 + end + end + + # Serve an ls-refs response over chunked transfer encoding, one + # pkt-line per chunk, so the cap trips while bytes are still + # arriving rather than on a fully-buffered body. Once the client's + # cap trips it hangs up; send errors past that point are expected. + defp serve_chunked_ls_refs(ref_count, fun) do + {:ok, listener} = :gen_tcp.listen(0, [:binary, active: false, reuseaddr: true, packet: :raw]) + {:ok, port} = :inet.port(listener) + + task = + Task.async(fn -> + {:ok, sock} = :gen_tcp.accept(listener, 5_000) + _ = recv_request(sock) + + head = + "HTTP/1.1 200 OK\r\n" <> + "Content-Type: application/x-git-upload-pack-result\r\n" <> + "Transfer-Encoding: chunked\r\nConnection: close\r\n\r\n" + + :ok = :gen_tcp.send(sock, head) + + _ = + 1..ref_count + |> Enum.map(&ref_line/1) + |> Enum.concat([PktLine.flush(), :last]) + |> Enum.reduce_while(:ok, fn piece, :ok -> + data = if piece == :last, do: "0\r\n\r\n", else: chunk(piece) + + case :gen_tcp.send(sock, data) do + :ok -> {:cont, :ok} + {:error, _closed_by_capped_client} -> {:halt, :ok} + end + end) + + :gen_tcp.close(sock) + end) + + try do + fun.(port) + after + Task.await(task, 5_000) + :gen_tcp.close(listener) + end + end + + defp chunk(iodata) do + data = IO.iodata_to_binary(iodata) + Integer.to_string(byte_size(data), 16) <> "\r\n" <> data <> "\r\n" + end + + defp recv_request(sock) do + case :gen_tcp.recv(sock, 0, 2_000) do + {:ok, data} -> + if String.contains?(data, "\r\n\r\n"), do: data, else: data <> recv_request(sock) + + {:error, _} -> + <<>> + end + end + + defp ref_line(i) do + PktLine.encode("#{Base.encode16(sha(i), case: :lower)} #{ref_name(i)}\n") + end + + defp ref_name(i), do: "refs/heads/branch-#{String.pad_leading(Integer.to_string(i), 3, "0")}" + + defp sha(i), do: :binary.copy(<>, 20) +end diff --git a/test/exgit/transport/http_streaming_test.exs b/test/exgit/transport/http_streaming_test.exs index 0521012..b44f020 100644 --- a/test/exgit/transport/http_streaming_test.exs +++ b/test/exgit/transport/http_streaming_test.exs @@ -54,6 +54,26 @@ defmodule Exgit.Transport.HttpStreamingTest do :gen_tcp.close(listener) end + test "channel 3 error wins over trailing truncated bytes" do + body = + IO.iodata_to_binary([ + PktLine.encode("packfile\n"), + PktLine.encode(<<3, "fatal: boom">>), + # Trailing bytes of an incomplete pkt-line. Halting on the + # channel-3 error leaves these undecoded in the decoder — + # that truncation artifact must not mask the real error. + "00ff" + ]) + + {listener, port} = run_fetch_server(body) + t = HTTP.new("http://127.0.0.1:#{port}") + + assert {:error, {:server_error, "fatal: boom"}} = + HTTP.fetch(t, [<<0::160>>], sideband: true) + + :gen_tcp.close(listener) + end + @tag :memory test "process heap stays bounded relative to pack size (regression guard)" do # 8 MB of pack bytes split into 4 KB sideband packets. Old non-streaming diff --git a/test/exgit/transport/http_telemetry_redaction_test.exs b/test/exgit/transport/http_telemetry_redaction_test.exs index 6328db1..1bd22fc 100644 --- a/test/exgit/transport/http_telemetry_redaction_test.exs +++ b/test/exgit/transport/http_telemetry_redaction_test.exs @@ -1,6 +1,7 @@ defmodule Exgit.Transport.HttpTelemetryRedactionTest do use ExUnit.Case, async: true + alias Exgit.PktLine alias Exgit.Transport.HTTP # A token embedded in a remote URL must never reach telemetry metadata. @@ -63,6 +64,58 @@ defmodule Exgit.Transport.HttpTelemetryRedactionTest do assert meta.url =~ "***" end + test "push telemetry redacts a token embedded in the URL" do + m = marker() + t = HTTP.new("https://x-access-token:#{@token}@127.0.0.1:1/#{m}.git") + update = {"refs/heads/main", nil, <<0::160>>} + + meta = + capture_start_meta([:exgit, :transport, :push], m, fn -> + HTTP.push(t, [update], "PACK") + end) + + refute meta.url =~ @token, "token leaked into push telemetry: #{meta.url}" + assert meta.url =~ "***" + end + + test "ref_rejected security telemetry redacts a token embedded in the source URL" do + # `[:exgit, :security, :ref_rejected]` fires while parsing an ls-refs + # response, so the dead-port trick isn't enough — serve one hostile + # ref name from a local socket and let `keep_ref?` reject it. + m = marker() + hostile_ref = "refs/heads/../../escape" + sha = Base.encode16(:binary.copy(<<1>>, 20), case: :lower) + body = IO.iodata_to_binary([PktLine.encode("#{sha} #{hostile_ref}\n"), PktLine.flush()]) + + handler = "redact-#{m}" + test_pid = self() + + :telemetry.attach( + handler, + [:exgit, :security, :ref_rejected], + fn _e, _measurements, meta, _cfg -> + if is_binary(meta[:source]) and String.contains?(meta.source, m) do + send(test_pid, {:sec_meta, meta}) + end + end, + nil + ) + + try do + serve_once(body, fn port -> + t = HTTP.new("http://x-access-token:#{@token}@127.0.0.1:#{port}/#{m}.git") + _ = HTTP.ls_refs(t) + end) + + assert_receive {:sec_meta, meta}, 2_000 + assert meta.ref == hostile_ref + refute meta.source =~ @token, "token leaked into ref_rejected telemetry: #{meta.source}" + assert meta.source =~ "***" + after + :telemetry.detach(handler) + end + end + test "a URL without credentials passes through unchanged" do m = marker() t = HTTP.new("https://127.0.0.1:1/#{m}.git") @@ -71,4 +124,42 @@ defmodule Exgit.Transport.HttpTelemetryRedactionTest do assert meta.url == "https://127.0.0.1:1/#{m}.git" refute meta.url =~ "***" end + + # One-shot HTTP server (same shape as `warm_server` in HttpTest): + # accept a single request, reply 200 with `body`, close. + defp serve_once(body, fun) do + {:ok, listener} = :gen_tcp.listen(0, [:binary, active: false, reuseaddr: true, packet: :raw]) + {:ok, port} = :inet.port(listener) + + task = + Task.async(fn -> + {:ok, sock} = :gen_tcp.accept(listener, 5_000) + _ = recv_request(sock) + + resp = + "HTTP/1.1 200 OK\r\n" <> + "Content-Type: application/x-git-upload-pack-result\r\n" <> + "Content-Length: #{byte_size(body)}\r\nConnection: close\r\n\r\n" <> body + + :ok = :gen_tcp.send(sock, resp) + :gen_tcp.close(sock) + end) + + try do + fun.(port) + after + Task.await(task, 5_000) + :gen_tcp.close(listener) + end + end + + defp recv_request(sock) do + case :gen_tcp.recv(sock, 0, 2_000) do + {:ok, data} -> + if String.contains?(data, "\r\n\r\n"), do: data, else: data <> recv_request(sock) + + {:error, _} -> + <<>> + end + end end diff --git a/test/exgit/transport/http_test.exs b/test/exgit/transport/http_test.exs index 860b8aa..e665401 100644 --- a/test/exgit/transport/http_test.exs +++ b/test/exgit/transport/http_test.exs @@ -67,6 +67,62 @@ defmodule Exgit.Transport.HttpTest do end end + describe "hostile server bytes surface as error values" do + test "capabilities/1 returns an error tuple on garbage bytes" do + garbage = "ZZZZ" <> :binary.copy(<<0xFF>>, 32) + + warm_server(garbage, fn port -> + t = HTTP.new("http://127.0.0.1:#{port}") + + assert {:error, {:malformed_response, {:malformed_pkt_line, _}}} = + HTTP.capabilities(t) + end) + end + + test "push/4 returns an error tuple when the report-status body is garbage" do + warm_server("ZZZZnot-a-pkt-line", fn port -> + t = HTTP.new("http://127.0.0.1:#{port}") + update = {"refs/heads/main", nil, :binary.copy(<<1>>, 20)} + + assert {:error, {:malformed_response, {:malformed_pkt_line, _}}} = + HTTP.push(t, [update], "PACK") + end) + end + end + + describe "ls-refs ref cap (:max_refs)" do + test "exceeding the cap surfaces as {:error, {:too_many_refs, cap}}" do + lines = + for i <- 1..5 do + sha = Base.encode16(:binary.copy(<>, 20), case: :lower) + PktLine.encode("#{sha} refs/heads/b#{i}\n") + end + + body = IO.iodata_to_binary([lines, PktLine.flush()]) + + warm_server(body, fn port -> + t = HTTP.new("http://127.0.0.1:#{port}", max_refs: 3) + assert {:error, {:too_many_refs, 3}} = HTTP.ls_refs(t) + end) + end + + test "under the cap the same response lists all refs" do + lines = + for i <- 1..5 do + sha = Base.encode16(:binary.copy(<>, 20), case: :lower) + PktLine.encode("#{sha} refs/heads/b#{i}\n") + end + + body = IO.iodata_to_binary([lines, PktLine.flush()]) + + warm_server(body, fn port -> + t = HTTP.new("http://127.0.0.1:#{port}", max_refs: 5) + assert {:ok, refs, _meta} = HTTP.ls_refs(t) + assert length(refs) == 5 + end) + end + end + defp warm_server(body, fun) do {:ok, listener} = :gen_tcp.listen(0, [:binary, active: false, reuseaddr: true, packet: :raw]) {:ok, port} = :inet.port(listener) From 067999515dd23d29b5055bc324de42cd412d164d Mon Sep 17 00:00:00 2001 From: Ivar Vong Date: Wed, 1 Jul 2026 16:44:02 -0400 Subject: [PATCH 11/13] fix(objects,stores): tree-mode canonicalization, tag round-trip fidelity, size-index sync Review findings for 0.1.0: - Tree.canonical_mode/2 no longer coerces zero-padded directory/ symlink/gitlink modes (040000, 0120000, 0160000) into blob modes; canonicalization now dispatches on the file-type bits. Regression SHAs pinned against real git. - Tag.decode/1 preserves unknown headers and continuation lines in order (headers list like Commit), so decode |> encode is byte-exact and re-encoding cannot change a tag's SHA. Accessors type/1, tag/1, tagger/1 replace the removed struct fields. - Memory.delete_object/2 keeps the objects and sizes indexes in lockstep; promisor cache eviction uses it instead of reaching into Memory internals, fixing stale sizes for evicted objects. - Disk.object_size/2 streams loose-object headers in bounded chunks (constant memory AND constant I/O) instead of File.read on the whole compressed file; packed fallback measures the inflated content directly. Truncated/corrupt/oversized-header inputs now covered by tests, as is the packed path. - object_size instrumented with the [:exgit, :object_store, :object_size] telemetry span in all three defimpls; protocol contract promoted from a code comment to real @doc. - Repository.memory_report/1 reports :unknown (per its doc) for non-introspected backends instead of a misleading 0. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01VEo6srWr3aFwhtwaXDTCZZ --- lib/exgit/object/tag.ex | 173 +++++++++++------- lib/exgit/object/tree.ex | 46 +++-- lib/exgit/object_store.ex | 37 +++- lib/exgit/object_store/disk.ex | 100 ++++++++-- lib/exgit/object_store/memory.ex | 32 +++- lib/exgit/object_store/promisor.ex | 36 ++-- lib/exgit/repository.ex | 40 ++-- lib/exgit/telemetry.ex | 4 + test/exgit/object/tag_test.exs | 55 +++++- test/exgit/object/tree_test.exs | 45 +++++ .../object_store/disk_object_size_test.exs | 92 ++++++++++ .../object_store/disk_pack_lookup_test.exs | 20 ++ test/exgit/object_store/disk_test.exs | 47 +++++ test/exgit/object_store/memory_test.exs | 20 ++ .../object_store/promisor_eviction_test.exs | 15 ++ test/exgit/repository_memory_report_test.exs | 42 ++++- 16 files changed, 656 insertions(+), 148 deletions(-) create mode 100644 test/exgit/object_store/disk_object_size_test.exs diff --git a/lib/exgit/object/tag.ex b/lib/exgit/object/tag.ex index b6254be..b17e78d 100644 --- a/lib/exgit/object/tag.ex +++ b/lib/exgit/object/tag.ex @@ -2,58 +2,91 @@ defmodule Exgit.Object.Tag do @moduledoc """ Git annotated tag object. - `decode/1` validates the `object` header as a 40-char hex string - and stores it as the raw 20-byte binary. Accessors on a decoded - tag are infallible — a hostile remote cannot DoS a walk/diff by - shipping a tag with non-hex header bytes. See + Like `Exgit.Object.Commit`, a tag is represented as a message plus an + **ordered list of headers**. Unknown headers and continuation lines + (multi-line values such as embedded signatures) are preserved verbatim + and in order, so `decode |> encode` is byte-exact and re-encoding a + decoded tag never changes its SHA. + + The `:object` field caches the validated `object` header as a raw + 20-byte binary; `encode/1` reads only `:headers` and `:message`. + Convenience accessors (`type/1`, `tag/1`, `tagger/1`) extract + well-known headers. + + `decode/1` validates the `object` header as a 40-char hex string. + Accessors on a decoded tag are infallible — a hostile remote cannot + DoS a walk/diff by shipping a tag with non-hex header bytes. See `test/exgit/security/tag_malformed_hex_test.exs` for the regression. """ alias Exgit.Object.Hex - @enforce_keys [:object, :type, :tag, :message] - defstruct [:object, :type, :tag, :message, tagger: nil] + @enforce_keys [:object, :headers, :message] + defstruct [:object, :headers, :message] + @type header :: {name :: String.t(), value :: String.t()} @type t :: %__MODULE__{ object: binary(), - type: String.t(), - tag: String.t(), - tagger: String.t() | nil, + headers: [header()], message: String.t() } @spec new(keyword()) :: t() def new(opts) do - %__MODULE__{ - object: Keyword.fetch!(opts, :object), - type: Keyword.get(opts, :type, "commit"), - tag: Keyword.fetch!(opts, :tag), - tagger: Keyword.get(opts, :tagger), - message: Keyword.fetch!(opts, :message) - } + object = Keyword.fetch!(opts, :object) + type = Keyword.get(opts, :type, "commit") + tag = Keyword.fetch!(opts, :tag) + tagger = Keyword.get(opts, :tagger) + message = Keyword.fetch!(opts, :message) + + # `Hex.encode/1` accepts either a raw 20-byte sha or a 40-char hex + # string; the field always holds the raw form, the header the hex. + object_hex = Hex.encode(object) + + headers = + [{"object", object_hex}, {"type", type}, {"tag", tag}] ++ + if(tagger, do: [{"tagger", tagger}], else: []) + + %__MODULE__{object: Hex.decode!(object_hex), headers: headers, message: message} + end + + @spec type(t()) :: String.t() + def type(%__MODULE__{} = t), do: header!(t, "type") + + @spec tag(t()) :: String.t() + def tag(%__MODULE__{} = t), do: header!(t, "tag") + + @spec tagger(t()) :: String.t() | nil + def tagger(%__MODULE__{headers: hs}) do + Enum.find_value(hs, fn + {"tagger", v} -> v + _ -> nil + end) + end + + defp header!(%__MODULE__{headers: hs}, name) do + case Enum.find(hs, fn {n, _} -> n == name end) do + {_, v} -> v + nil -> raise KeyError, key: name + end end @spec encode(t()) :: iolist() - def encode(%__MODULE__{} = t) do + def encode(%__MODULE__{headers: headers, message: message}) do [ - "object ", - Hex.encode(t.object), - ?\n, - "type ", - t.type, - ?\n, - "tag ", - t.tag, - ?\n, - encode_tagger(t.tagger), + Enum.map(headers, &encode_header/1), ?\n, - t.message + message ] end - defp encode_tagger(nil), do: [] - defp encode_tagger(tagger), do: ["tagger ", tagger, ?\n] + defp encode_header({name, value}) do + case String.split(value, "\n") do + [single] -> [name, ?\s, single, ?\n] + [first | rest] -> [name, ?\s, first, ?\n, Enum.map(rest, fn l -> [?\s, l, ?\n] end)] + end + end @spec decode(binary()) :: {:ok, t()} | {:error, term()} def decode(bytes) when is_binary(bytes) do @@ -63,19 +96,14 @@ defmodule Exgit.Object.Tag do case parse_headers(raw_headers) do {:ok, headers} -> - with {:ok, object} <- Map.fetch(headers, :object), - {:ok, type} <- Map.fetch(headers, :type), - {:ok, tag} <- Map.fetch(headers, :tag) do - {:ok, - %__MODULE__{ - object: object, - type: type, - tag: tag, - tagger: Map.get(headers, :tagger), - message: message - }} - else - :error -> {:error, :missing_header} + # A tag must have object + type + tag. The `object` header + # is additionally validated as 40-char hex so that the + # `:object` field (and downstream walks) are infallible — + # we never `raise` on the wire bytes. + with :ok <- ensure_header(headers, "type"), + :ok <- ensure_header(headers, "tag"), + {:ok, object} <- fetch_object(headers) do + {:ok, %__MODULE__{object: object, headers: headers, message: message}} end {:error, _} = err -> @@ -89,33 +117,50 @@ defmodule Exgit.Object.Tag do e -> {:error, {:decode_failed, e}} end - # Parse headers into a map. If any `object` header has non-hex content, - # return an error — a hostile remote could otherwise DoS downstream - # accessors. We never `raise` on the wire bytes. - defp parse_headers(raw) do - raw - |> String.split("\n") - |> Enum.reduce_while({:ok, %{}}, fn line, {:ok, acc} -> - case String.split(line, " ", parts: 2) do - ["object", hex] -> - case Hex.decode(hex) do - {:ok, bin} -> {:cont, {:ok, Map.put(acc, :object, bin)}} - :error -> {:halt, {:error, {:invalid_hex_header, "object", hex}}} - end + defp ensure_header(headers, name) do + if Enum.any?(headers, fn {n, _} -> n == name end), + do: :ok, + else: {:error, {:missing_header, name}} + end - ["type", val] -> - {:cont, {:ok, Map.put(acc, :type, val)}} + defp fetch_object(headers) do + case Enum.find(headers, fn {n, _} -> n == "object" end) do + {_, hex} -> + case Hex.decode(hex) do + {:ok, bin} -> {:ok, bin} + :error -> {:error, {:invalid_hex_header, "object", hex}} + end - ["tag", val] -> - {:cont, {:ok, Map.put(acc, :tag, val)}} + nil -> + {:error, {:missing_header, "object"}} + end + end + + # Parse headers preserving order. Continuation lines (lines starting with + # a space) extend the previous header's value with a leading newline. + # A continuation line appearing before any header is an error. + defp parse_headers(raw) do + lines = String.split(raw, "\n") - ["tagger", val] -> - {:cont, {:ok, Map.put(acc, :tagger, val)}} + Enum.reduce_while(lines, {:ok, []}, fn line, {:ok, acc} -> + case {line, acc} do + {" " <> rest, [{name, val} | tail]} -> + {:cont, {:ok, [{name, val <> "\n" <> rest} | tail]}} - _ -> - {:cont, {:ok, acc}} + {" " <> _, []} -> + {:halt, {:error, :leading_continuation}} + + {line, acc} -> + case String.split(line, " ", parts: 2) do + [name, val] when name != "" -> {:cont, {:ok, [{name, val} | acc]}} + _ -> {:halt, {:error, :malformed_header}} + end end end) + |> case do + {:ok, rev} -> {:ok, Enum.reverse(rev)} + {:error, _} = err -> err + end end @spec sha(t()) :: Exgit.Object.sha() diff --git a/lib/exgit/object/tree.ex b/lib/exgit/object/tree.ex index b89e1bb..91f1cf6 100644 --- a/lib/exgit/object/tree.ex +++ b/lib/exgit/object/tree.ex @@ -9,10 +9,10 @@ defmodule Exgit.Object.Tree do tree's SHA and corrupt verification. `new/1` applies git's canonical ordering (dirs sort as if they had a - trailing `/`) and — by default — normalizes regular file modes via - `canonical_mode/1`. Pass `strict: true` to error on unknown modes - instead of silently coercing, or build the struct directly for a - raw, unvalidated tree. + trailing `/`) and — by default — normalizes modes via + `canonical_mode/1`. Pass `strict: true` to raise on non-canonical + modes instead of silently coercing, or build the struct directly for + a raw, unvalidated tree. > #### Round-trip note {: .warning} > @@ -44,10 +44,14 @@ defmodule Exgit.Object.Tree do Options: - * `:strict` — when `true`, reject unknown modes with a - `:invalid_mode` error. Default: `false` (unknown modes are - silently coerced to `100644` / `100755` based on the executable - bit). + * `:strict` — when `true`, raise `ArgumentError` on any mode that + is not one of the five canonical git modes (`40000`, `100644`, + `100755`, `120000`, `160000`). Default: `false`, which + canonicalizes via `canonical_mode/1`: zero-padded modes normalize + to their canonical form (`"040000"` → `"40000"`, `"0120000"` → + `"120000"`), other octal modes coerce to `100644` / `100755` + based on the executable bit, and non-octal strings pass through + unchanged. """ @spec new([entry()], keyword()) :: t() def new(entries, opts \\ []) when is_list(entries) do @@ -68,10 +72,14 @@ defmodule Exgit.Object.Tree do end @doc """ - Normalize a file mode to one of the canonical git file modes. Used by - `new/1` but NOT by `decode/1`. Unknown modes are returned unchanged - by the one-arg form; the two-arg form with `strict: true` raises - on unknown input. + Normalize a mode string to the canonical git spelling. Used by `new/1` + but NOT by `decode/1`. Octal modes canonicalize by their file-type + bits: directories (`"040000"` → `"40000"`), symlinks (`"0120000"` → + `"120000"`) and gitlinks (`"0160000"` → `"160000"`) keep their type; + everything else coerces to `100644` / `100755` based on the + executable bit. Non-octal strings are returned unchanged by the + one-arg form; the two-arg form with `strict: true` raises + `ArgumentError` on any non-canonical input. """ @spec canonical_mode(String.t()) :: String.t() def canonical_mode(mode), do: canonical_mode(mode, false) @@ -87,7 +95,7 @@ defmodule Exgit.Object.Tree do # Parse as octal (git modes are always octal strings). case Integer.parse(mode, 8) do {n, ""} when not strict -> - if band(n, 0o111) != 0, do: "100755", else: "100644" + canonicalize_octal(n) {_n, ""} when strict -> raise ArgumentError, "invalid git tree mode: #{inspect(mode)}" @@ -100,6 +108,18 @@ defmodule Exgit.Object.Tree do end end + # Canonicalize by the file-type bits (upper 4 octal digits of a + # 16-bit st_mode). A zero-padded "040000" must stay a directory — + # coercing it to a blob mode would silently corrupt the tree. + defp canonicalize_octal(n) do + case band(n, 0o170000) do + 0o040000 -> "40000" + 0o120000 -> "120000" + 0o160000 -> "160000" + _ -> if band(n, 0o111) != 0, do: "100755", else: "100644" + end + end + @spec encode(t()) :: iolist() def encode(%__MODULE__{entries: entries}) do Enum.map(entries, fn {mode, name, sha} -> diff --git a/lib/exgit/object_store.ex b/lib/exgit/object_store.ex index 342a210..272218e 100644 --- a/lib/exgit/object_store.ex +++ b/lib/exgit/object_store.ex @@ -1,4 +1,16 @@ defprotocol Exgit.ObjectStore do + @moduledoc """ + Storage protocol for git objects, keyed by binary SHA-1. + + Implemented by `Exgit.ObjectStore.Memory` (in-heap), + `Exgit.ObjectStore.Disk` (git's on-disk layout), and + `Exgit.ObjectStore.Promisor` (lazy, fetch-on-demand). + + Stores are pure values: every write (`put/2`, `import_objects/2`, + `close_write/2`) returns the updated store, which the caller must + thread forward. Reads never mutate. + """ + @spec get(t, binary()) :: {:ok, Exgit.Object.t()} | {:error, term()} def get(store, sha) @@ -8,14 +20,23 @@ defprotocol Exgit.ObjectStore do @spec has?(t, binary()) :: boolean() def has?(store, sha) - # Uncompressed byte size of an object WITHOUT materializing its content. - # The point is a constant-memory size check: callers can decide whether a - # blob is too large to read before paying to inflate it into the heap. - # - # `{:error, :not_local}` is a meaningful, non-fatal result for lazy stores - # (`Promisor`): it means "knowing this size requires a network fetch" — so - # the caller can opt in rather than have a multi-GB fetch triggered behind - # a size check. `{:error, :not_found}` means the object is genuinely absent. + @doc """ + Uncompressed byte size of an object WITHOUT materializing its content. + + The point is a constant-memory size check: callers can decide whether a + blob is too large to read before paying to inflate it into the heap. + + Returns `{:ok, byte_count}` when the store can answer, or: + + * `{:error, :not_found}` — the object is genuinely absent from the + store. + * `{:error, :not_local}` — a meaningful, non-fatal result for lazy + stores (`Promisor`): it means "knowing this size requires a network + fetch", so the caller can opt in rather than have a multi-GB fetch + triggered behind a size check. + * `{:error, term()}` — store-specific failure (e.g. a corrupt loose + object on disk). + """ @spec object_size(t, binary()) :: {:ok, non_neg_integer()} | {:error, :not_found | :not_local | term()} def object_size(store, sha) diff --git a/lib/exgit/object_store/disk.ex b/lib/exgit/object_store/disk.ex index eaeab6e..4a62522 100644 --- a/lib/exgit/object_store/disk.ex +++ b/lib/exgit/object_store/disk.ex @@ -82,11 +82,12 @@ defmodule Exgit.ObjectStore.Disk do @doc """ Uncompressed object size without materializing the object. - For a loose object this inflates only the header bytes (`" - \\0"`) — constant memory, no matter how large the blob. Packed - objects fall back to a full read (the delta chain must be resolved to - know the final size), so this is cheap for loose objects and - O(object) for packed ones. + For a loose object this reads and inflates only enough compressed + bytes to cover the header (`" \\0"`) — constant memory + AND constant I/O, no matter how large the blob. Packed objects fall + back to a full read (the delta chain must be resolved to know the + final size), so this is cheap for loose objects and O(object) for + packed ones. """ @spec object_size(t(), binary()) :: {:ok, non_neg_integer()} | {:error, term()} def object_size(%__MODULE__{root: root}, sha) when byte_size(sha) == 20 do @@ -94,17 +95,29 @@ defmodule Exgit.ObjectStore.Disk do <> = hex path = Path.join([root, "objects", prefix, rest]) - case File.read(path) do - {:ok, compressed} -> loose_object_size(compressed) + case loose_object_size(path) do + {:ok, size} -> {:ok, size} {:error, :enoent} -> packed_object_size(root, sha) {:error, reason} -> {:error, reason} end end - defp loose_object_size(compressed) do - with {:ok, header} <- inflate_until_null(compressed), - {:ok, _type, size} <- parse_loose_header(header) do - {:ok, size} + defp loose_object_size(path) do + case :file.open(path, [:read, :raw, :binary]) do + {:ok, fd} -> + try do + with {:ok, header} <- inflate_until_null(fd) do + case parse_loose_header(header) do + {:ok, _type, size} -> {:ok, size} + {:error, _} = err -> err + end + end + after + :file.close(fd) + end + + {:error, reason} -> + {:error, reason} end end @@ -112,14 +125,19 @@ defmodule Exgit.ObjectStore.Disk do # object can't make us inflate unbounded output hunting for a NUL. @max_header_bytes 64 + # Compressed bytes read from disk per iteration while hunting for the + # header's NUL. The header fits in the first chunk for any real object. + @header_read_chunk_bytes 256 + # Stream-inflate only until the header's NUL terminator, returning the - # bytes before it. Bounded memory: never inflates the full object. - defp inflate_until_null(compressed) do + # bytes before it. Bounded memory AND bounded I/O: reads small + # compressed chunks from `fd` and never inflates the full object. + defp inflate_until_null(fd) do z = :zlib.open() try do :zlib.inflateInit(z) - drain_until_null(z, compressed, <<>>) + feed_until_null(z, fd, <<>>) rescue _ -> {:error, :zlib_error} catch @@ -129,9 +147,30 @@ defmodule Exgit.ObjectStore.Disk do end end + defp feed_until_null(z, fd, acc) do + case :file.read(fd, @header_read_chunk_bytes) do + {:ok, chunk} -> + case drain_until_null(z, chunk, acc) do + {:need_input, acc} -> feed_until_null(z, fd, acc) + done -> done + end + + # Truncated file: zlib still wants input but there is none. + :eof -> + {:error, :malformed_object_header} + + {:error, reason} -> + {:error, reason} + end + end + + # Inflate `input` and any pending output, scanning for the NUL. + # `{:need_input, acc}` means zlib produced nothing more from this + # chunk — the caller must read more compressed bytes. defp drain_until_null(z, input, acc) do {status, out} = :zlib.safeInflate(z, input) - acc = acc <> IO.iodata_to_binary(out) + out = IO.iodata_to_binary(out) + acc = acc <> out case :binary.match(acc, <<0>>) do {pos, 1} -> @@ -141,14 +180,17 @@ defmodule Exgit.ObjectStore.Disk do cond do byte_size(acc) > @max_header_bytes -> {:error, :malformed_object_header} status == :finished -> {:error, :malformed_object_header} + out == <<>> -> {:need_input, acc} true -> drain_until_null(z, [], acc) end end end defp packed_object_size(root, sha) do - case get_from_packs(root, sha) do - {:ok, obj} -> {:ok, IO.iodata_length(Exgit.Object.encode(obj))} + case raw_from_packs(root, sha) do + # `content` is the already-inflated (delta-resolved) object body; + # its byte_size IS the object size. No decode→re-encode round trip. + {:ok, _type, content} -> {:ok, byte_size(content)} {:error, _} = err -> err end end @@ -324,6 +366,15 @@ defmodule Exgit.ObjectStore.Disk do defp hex_char?(_), do: false defp get_from_packs(root, sha) do + case raw_from_packs(root, sha) do + {:ok, type, content} -> Exgit.Object.decode(type, content) + {:error, _} = err -> err + end + end + + # Locate `sha` in any pack and return its inflated (delta-resolved) + # type + content, without decoding into an `Exgit.Object`. + defp raw_from_packs(root, sha) do pack_dir = Path.join([root, "objects", "pack"]) case File.ls(pack_dir) do @@ -350,7 +401,7 @@ defmodule Exgit.ObjectStore.Disk do # the header we preserved). Parse at that offset, NOT the on-disk # offset. case Exgit.Pack.Reader.parse_at(pack_slice, 12) do - {:ok, {type, ^sha, content}} -> Exgit.Object.decode(type, content) + {:ok, {type, ^sha, content}} -> {:ok, type, content} _ -> find_in_packs(dir, rest, sha) end else @@ -520,7 +571,18 @@ defimpl Exgit.ObjectStore, for: Exgit.ObjectStore.Disk do ) end - def object_size(store, sha), do: Disk.object_size(store, sha) + def object_size(store, sha) do + Telemetry.span( + [:exgit, :object_store, :object_size], + %{store: :disk, sha: sha}, + fn -> + case Disk.object_size(store, sha) do + {:ok, _} = ok -> {:span, ok, %{hit?: true}} + other -> {:span, other, %{hit?: false}} + end + end + ) + end def import_objects(store, raw_objects) do failures = diff --git a/lib/exgit/object_store/memory.ex b/lib/exgit/object_store/memory.ex index 345c950..710b32e 100644 --- a/lib/exgit/object_store/memory.ex +++ b/lib/exgit/object_store/memory.ex @@ -62,6 +62,25 @@ defmodule Exgit.ObjectStore.Memory do end end + @doc """ + Remove `sha` from the store, keeping the `objects` and `sizes` + indexes in lockstep. Returns `{:ok, freed_compressed_bytes, store}` + so callers tracking a byte budget (e.g. the Promisor's cache + accounting) know how much was reclaimed, or `{:error, :not_found}` + if the object is absent. + """ + @spec delete_object(t(), binary()) :: {:ok, non_neg_integer(), t()} | {:error, :not_found} + def delete_object(%__MODULE__{objects: objects, sizes: sizes} = store, sha) do + case Map.pop(objects, sha) do + {nil, _} -> + {:error, :not_found} + + {{_type, compressed}, new_objects} -> + {:ok, byte_size(compressed), + %{store | objects: new_objects, sizes: Map.delete(sizes, sha)}} + end + end + @spec import_objects(t(), [{atom(), binary(), binary()}]) :: {:ok, t()} def import_objects(%__MODULE__{objects: objects, sizes: sizes} = store, raw_objects) do {new_objects, new_sizes} = @@ -113,7 +132,18 @@ defimpl Exgit.ObjectStore, for: Exgit.ObjectStore.Memory do ) end - def object_size(store, sha), do: Memory.object_size(store, sha) + def object_size(store, sha) do + Telemetry.span( + [:exgit, :object_store, :object_size], + %{store: :memory, sha: sha}, + fn -> + case Memory.object_size(store, sha) do + {:ok, _} = ok -> {:span, ok, %{hit?: true}} + other -> {:span, other, %{hit?: false}} + end + end + ) + end def import_objects(store, raw_objects), do: Memory.import_objects(store, raw_objects) diff --git a/lib/exgit/object_store/promisor.ex b/lib/exgit/object_store/promisor.ex index 22fcae9..3f63396 100644 --- a/lib/exgit/object_store/promisor.ex +++ b/lib/exgit/object_store/promisor.ex @@ -541,23 +541,16 @@ defmodule Exgit.ObjectStore.Promisor do _ -> {_key, sha, q2} = :gb_trees.take_smallest(q) - # Drop the commit object from the Memory cache. Track the - # byte delta. We pattern-match `p.cache` into a - # `%ObjectStore.Memory{}` binding first so the subsequent - # struct-update is visible to Elixir 1.19's type checker - # (a struct update on a field-access expression is rejected - # under --warnings-as-errors because the type is dynamic() - # at that site). - %ObjectStore.Memory{objects: objs} = cache = p.cache - - {dropped_bytes, new_objs} = - case Map.pop(objs, sha) do - {nil, o} -> {0, o} - {{_type, compressed}, o} -> {byte_size(compressed), o} + # Drop the commit object from the Memory cache via its delete + # helper — it keeps the `objects` and `sizes` indexes in + # lockstep, so `object_size/2` can never report a stale size + # for an evicted object. + {dropped_bytes, new_cache} = + case ObjectStore.Memory.delete_object(p.cache, sha) do + {:ok, freed, cache} -> {freed, cache} + {:error, :not_found} -> {0, p.cache} end - new_cache = %ObjectStore.Memory{cache | objects: new_objs} - %{ p | cache: new_cache, @@ -661,7 +654,18 @@ defimpl Exgit.ObjectStore, for: Exgit.ObjectStore.Promisor do ) end - def object_size(store, sha), do: Promisor.object_size(store, sha) + def object_size(store, sha) do + Telemetry.span( + [:exgit, :object_store, :object_size], + %{store: :promisor, sha: sha}, + fn -> + case Promisor.object_size(store, sha) do + {:ok, _} = ok -> {:span, ok, %{hit?: true}} + other -> {:span, other, %{hit?: false}} + end + end + ) + end def import_objects(store, raw_objects), do: Promisor.import_objects(store, raw_objects) diff --git a/lib/exgit/repository.ex b/lib/exgit/repository.ex index d8a312f..9e435d4 100644 --- a/lib/exgit/repository.ex +++ b/lib/exgit/repository.ex @@ -88,14 +88,17 @@ defmodule Exgit.Repository do * `:max_cache_bytes` — configured cap (`:infinity` if unbounded) * `:mode` — repo mode (`:eager` or `:lazy`) * `:backend` — the object-store module + + Counts and `:cache_bytes` are `:unknown` for backends that can't + be introspected (`Disk`, user-defined stores). """ @type memory_report :: %{ - object_count: non_neg_integer(), - cache_bytes: non_neg_integer(), - commit_count: non_neg_integer(), - tree_count: non_neg_integer(), - blob_count: non_neg_integer(), - tag_count: non_neg_integer(), + object_count: non_neg_integer() | :unknown, + cache_bytes: non_neg_integer() | :unknown, + commit_count: non_neg_integer() | :unknown, + tree_count: non_neg_integer() | :unknown, + blob_count: non_neg_integer() | :unknown, + tag_count: non_neg_integer() | :unknown, max_cache_bytes: non_neg_integer() | :infinity, mode: mode(), backend: module() @@ -109,10 +112,9 @@ defmodule Exgit.Repository do cache growth, and alert when a configured cap is approached. Returns consistent shape across all object-store backends - (`Memory`, `Disk`, `Promisor`, `SharedPromisor`); counts for - backends without per-type bookkeeping (like `Disk`) are - `:unknown`. The `:cache_bytes` and `:max_cache_bytes` fields - are always present. + (`Memory`, `Disk`, `Promisor`, `SharedPromisor`); counts and + `:cache_bytes` for backends without per-type bookkeeping (like + `Disk`) are `:unknown`. All keys are always present. ## Examples @@ -189,18 +191,18 @@ defmodule Exgit.Repository do defp store_report(other) do # Backends we don't introspect deeply (ObjectStore.Disk, - # user-defined stores). Report a degraded shape with - # placeholders so callers can still depend on the keys - # existing. + # user-defined stores). Report `:unknown` rather than a fake 0 + # so monitoring callers can't mistake "not introspected" for + # "empty"; the keys still always exist. _ = other %{ - object_count: 0, - cache_bytes: 0, - commit_count: 0, - tree_count: 0, - blob_count: 0, - tag_count: 0, + object_count: :unknown, + cache_bytes: :unknown, + commit_count: :unknown, + tree_count: :unknown, + blob_count: :unknown, + tag_count: :unknown, max_cache_bytes: :infinity } end diff --git a/lib/exgit/telemetry.ex b/lib/exgit/telemetry.ex index cc044e5..5d557d3 100644 --- a/lib/exgit/telemetry.ex +++ b/lib/exgit/telemetry.ex @@ -43,6 +43,10 @@ defmodule Exgit.Telemetry do - `hit?: false` when it had to go to the network - omitted for stores that don't track this distinction * `[:exgit, :object_store, :put]` — `%{store, sha}` + * `[:exgit, :object_store, :object_size]` — `%{store, sha, hit?}` + - `hit?: false` covers both `{:error, :not_found}` and + `{:error, :not_local}` (a `Promisor` never fetches for a + size probe) * `[:exgit, :object_store, :has?]` — `%{store, sha, present?}` * `[:exgit, :object_store, :fetch_and_cache]` — `%{sha, object_count, cache_bytes}` (stop event only) diff --git a/test/exgit/object/tag_test.exs b/test/exgit/object/tag_test.exs index e1e6acd..d3c5276 100644 --- a/test/exgit/object/tag_test.exs +++ b/test/exgit/object/tag_test.exs @@ -21,10 +21,11 @@ defmodule Exgit.Object.TagTest do encoded = tag |> Tag.encode() |> IO.iodata_to_binary() assert {:ok, decoded} = Tag.decode(encoded) assert decoded.object == obj_sha - assert decoded.type == "commit" - assert decoded.tag == "v1.0" - assert decoded.tagger == @tagger + assert Tag.type(decoded) == "commit" + assert Tag.tag(decoded) == "v1.0" + assert Tag.tagger(decoded) == @tagger assert decoded.message == "release v1.0\n" + assert decoded == tag end test "round-trips a tag without tagger" do @@ -39,8 +40,52 @@ defmodule Exgit.Object.TagTest do encoded = tag |> Tag.encode() |> IO.iodata_to_binary() assert {:ok, decoded} = Tag.decode(encoded) - assert decoded.tagger == nil - assert decoded.tag == "v0.1" + assert Tag.tagger(decoded) == nil + assert Tag.tag(decoded) == "v0.1" + end + end + + describe "byte-exact fidelity" do + # Pinned against `git hash-object -t tag --literally`. + @exotic_sha "738c8cfdfce431cfe7add4760aa61ebd5e1964d1" + + test "decode |> encode is byte-exact for a tag with unknown and multi-line headers" do + raw = + "object #{String.duplicate("a", 40)}\n" <> + "type commit\n" <> + "tag v9.9\n" <> + "tagger #{@tagger}\n" <> + "x-custom something unusual\n" <> + "signature -----BEGIN PGP SIGNATURE-----\n" <> + " abc123\n" <> + " -----END PGP SIGNATURE-----\n" <> + "\n" <> + "Release with exotic headers\n" + + assert {:ok, tag} = Tag.decode(raw) + + # Unknown headers are preserved verbatim and in order; continuation + # lines fold into the previous header's value with newlines. + assert {"x-custom", "something unusual"} in tag.headers + + assert {"signature", "-----BEGIN PGP SIGNATURE-----\nabc123\n-----END PGP SIGNATURE-----"} in tag.headers + + assert Enum.map(tag.headers, &elem(&1, 0)) == + ["object", "type", "tag", "tagger", "x-custom", "signature"] + + # Re-encoding must reproduce the original bytes, so the SHA is stable. + assert IO.iodata_to_binary(Tag.encode(tag)) == raw + assert Tag.sha_hex(tag) == @exotic_sha + end + end + + describe "decode/1 validation" do + test "missing required headers return errors" do + assert {:error, {:missing_header, "object"}} = Tag.decode("type commit\ntag v1\n\nm\n") + + hex = String.duplicate("a", 40) + assert {:error, {:missing_header, "type"}} = Tag.decode("object #{hex}\ntag v1\n\nm\n") + assert {:error, {:missing_header, "tag"}} = Tag.decode("object #{hex}\ntype commit\n\nm\n") end end end diff --git a/test/exgit/object/tree_test.exs b/test/exgit/object/tree_test.exs index 189af43..e40c2e9 100644 --- a/test/exgit/object/tree_test.exs +++ b/test/exgit/object/tree_test.exs @@ -58,6 +58,51 @@ defmodule Exgit.Object.TreeTest do end end + describe "canonical_mode/1 with zero-padded modes" do + test "padded modes canonicalize to their file type, not to a blob mode" do + # Regression: these used to fall through to the exec-bit coercion + # and come back as "100644"/"100755", turning dirs into files. + assert Tree.canonical_mode("040000") == "40000" + assert Tree.canonical_mode("0120000") == "120000" + assert Tree.canonical_mode("0160000") == "160000" + assert Tree.canonical_mode("0100644") == "100644" + assert Tree.canonical_mode("0100755") == "100755" + end + + test "strict mode still raises on zero-padded modes" do + assert_raise ArgumentError, fn -> Tree.canonical_mode("040000", true) end + end + + test "new/1 with padded modes matches canonical modes, round-trips, stable SHA" do + sha = :binary.copy(<<0xAB>>, 20) + + padded = + Tree.new([ + {"040000", "dir", sha}, + {"0120000", "link", sha}, + {"0160000", "sub", sha}, + {"0100644", "file", sha} + ]) + + canonical = + Tree.new([ + {"40000", "dir", sha}, + {"120000", "link", sha}, + {"160000", "sub", sha}, + {"100644", "file", sha} + ]) + + assert padded.entries == canonical.entries + + encoded = padded |> Tree.encode() |> IO.iodata_to_binary() + assert {:ok, decoded} = Tree.decode(encoded) + assert decoded.entries == padded.entries + + # Pinned against `git mktree --missing` fed the same four entries. + assert Tree.sha_hex(padded) == "93a9f977c0cdaecda57e0ec64d9a5adba8f92e1c" + end + end + describe "sha/1" do @tag :git_cross_check test "matches git mktree" do diff --git a/test/exgit/object_store/disk_object_size_test.exs b/test/exgit/object_store/disk_object_size_test.exs new file mode 100644 index 0000000..8c6f7e2 --- /dev/null +++ b/test/exgit/object_store/disk_object_size_test.exs @@ -0,0 +1,92 @@ +defmodule Exgit.ObjectStore.DiskObjectSizeTest do + @moduledoc """ + Edge cases for `Disk.object_size/2`'s bounded header streaming: + truncated zlib streams, non-zlib bytes, headers that exceed the + scan budget, and the packed-object fallback. The happy loose path + is covered in `Exgit.FsSizeTest`. + """ + + use ExUnit.Case, async: false + + alias Exgit.Object.Blob + alias Exgit.ObjectStore + alias Exgit.Pack.Index + alias Exgit.Test.PackBuilder + + setup do + root = + Path.join( + System.tmp_dir!(), + "exgit_disk_size_#{System.unique_integer([:positive, :monotonic])}" + ) + + File.mkdir_p!(Path.join(root, "objects")) + on_exit(fn -> File.rm_rf!(root) end) + %{root: root, store: ObjectStore.Disk.new(root)} + end + + describe "loose-object error paths" do + test "truncated loose object (valid zlib prefix cut mid-stream) returns a clean error", + %{root: root, store: store} do + sha = :crypto.strong_rand_bytes(20) + compressed = :zlib.compress("blob 5\0hello") + # Keep only the 2-byte zlib stream header: a valid prefix that + # inflates to nothing, so the NUL hunt runs out of input at EOF. + write_loose(root, sha, binary_part(compressed, 0, 2)) + + assert ObjectStore.object_size(store, sha) == {:error, :malformed_object_header} + end + + test "corrupt non-zlib bytes return a clean error", %{root: root, store: store} do + sha = :crypto.strong_rand_bytes(20) + write_loose(root, sha, "definitely not a zlib stream") + + assert ObjectStore.object_size(store, sha) == {:error, :zlib_error} + end + + test "header longer than the scan budget returns a clean error", + %{root: root, store: store} do + sha = :crypto.strong_rand_bytes(20) + # Valid zlib, but the "header" never terminates: 200 bytes with + # no NUL blows the @max_header_bytes budget instead of inflating + # the whole object hunting for one. + write_loose(root, sha, :zlib.compress("blob " <> String.duplicate("9", 200))) + + assert ObjectStore.object_size(store, sha) == {:error, :malformed_object_header} + end + end + + describe "packed-object fallback" do + test "reports the exact content byte size for a packed blob", + %{root: root, store: store} do + content = "packed blob body " <> :crypto.strong_rand_bytes(1_000) + sha = Blob.sha(Blob.new(content)) + + pack = PackBuilder.build([{:full, :blob, content}]) + # Single-entry pack: the object spans from the 12-byte header to + # the 20-byte checksum trailer. + entry_bytes = binary_part(pack, 12, byte_size(pack) - 32) + checksum = binary_part(pack, byte_size(pack) - 20, 20) + idx = Index.write([{sha, :erlang.crc32(entry_bytes), 12}], checksum) + + pack_dir = Path.join(root, "objects/pack") + File.mkdir_p!(pack_dir) + File.write!(Path.join(pack_dir, "pack-size.pack"), pack) + File.write!(Path.join(pack_dir, "pack-size.idx"), idx) + + # No loose object on disk, so this exercises the pack fallback. + assert ObjectStore.object_size(store, sha) == {:ok, byte_size(content)} + end + end + + # Place raw bytes at the loose path for `sha`. object_size/2 only + # parses the header — it never verifies the sha — so a random + # 20-byte sha addresses whatever bytes we plant. + defp write_loose(root, sha, bytes) do + hex = Base.encode16(sha, case: :lower) + <> = hex + dir = Path.join([root, "objects", prefix]) + File.mkdir_p!(dir) + File.write!(Path.join(dir, rest), bytes) + end +end diff --git a/test/exgit/object_store/disk_pack_lookup_test.exs b/test/exgit/object_store/disk_pack_lookup_test.exs index 88b6922..02c9ecc 100644 --- a/test/exgit/object_store/disk_pack_lookup_test.exs +++ b/test/exgit/object_store/disk_pack_lookup_test.exs @@ -39,6 +39,26 @@ defmodule Exgit.ObjectStore.DiskPackLookupTest do end end + test "object_size/2 reports the inflated size of a packed object", %{ + root: root, + store: store + } do + blobs = [Blob.new("a"), Blob.new("bb"), Blob.new("ccc")] + pack = Writer.build(blobs) + + pack_path = Path.join(root, "objects/pack/pack-size.pack") + idx_path = Path.join(root, "objects/pack/pack-size.idx") + File.write!(pack_path, pack) + + entries = pack_entries(pack, blobs) + pack_checksum = binary_part(pack, byte_size(pack) - 20, 20) + File.write!(idx_path, Index.write(entries, pack_checksum)) + + for blob <- blobs do + assert ObjectStore.object_size(store, Blob.sha(blob)) == {:ok, byte_size(blob.data)} + end + end + @tag :slow test "repeated lookups into a single pack scale sub-linearly in pack size", %{ root: root, diff --git a/test/exgit/object_store/disk_test.exs b/test/exgit/object_store/disk_test.exs index 4694476..196ca2a 100644 --- a/test/exgit/object_store/disk_test.exs +++ b/test/exgit/object_store/disk_test.exs @@ -70,6 +70,53 @@ defmodule Exgit.ObjectStore.DiskTest do end end + describe "object_size" do + test "reads the size from the loose header without materializing", %{store: store} do + data = String.duplicate("z", 300_000) + {:ok, sha} = Disk.put_object(store, Blob.new(data)) + + assert Disk.object_size(store, sha) == {:ok, byte_size(data)} + end + + test "returns not_found for a missing object", %{store: store} do + assert {:error, :not_found} = Disk.object_size(store, :crypto.hash(:sha, "absent")) + end + + test "returns zlib_error for a corrupt loose object", %{store: store, path: path} do + sha = :crypto.hash(:sha, "corrupt") + write_loose_bytes(path, sha, "definitely not zlib data") + + assert {:error, :zlib_error} = Disk.object_size(store, sha) + end + + test "returns malformed_object_header for a truncated loose object", %{ + store: store, + path: path + } do + # Valid deflate stream cut off before the header's NUL appears. + sha = :crypto.hash(:sha, "truncated") + compressed = :zlib.compress("blob 1234\0" <> String.duplicate("y", 1234)) + write_loose_bytes(path, sha, binary_part(compressed, 0, 2)) + + assert {:error, :malformed_object_header} = Disk.object_size(store, sha) + end + + test "rejects a header with no NUL inside the scan budget", %{store: store, path: path} do + sha = :crypto.hash(:sha, "no-nul") + write_loose_bytes(path, sha, :zlib.compress(String.duplicate("a", 4096))) + + assert {:error, :malformed_object_header} = Disk.object_size(store, sha) + end + end + + defp write_loose_bytes(path, sha, bytes) do + hex = Base.encode16(sha, case: :lower) + <> = hex + dir = Path.join([path, "objects", prefix]) + File.mkdir_p!(dir) + File.write!(Path.join(dir, rest), bytes) + end + describe "delete" do test "removes the object", %{store: store} do blob = Blob.new("to delete") diff --git a/test/exgit/object_store/memory_test.exs b/test/exgit/object_store/memory_test.exs index 0b23851..4b3ca39 100644 --- a/test/exgit/object_store/memory_test.exs +++ b/test/exgit/object_store/memory_test.exs @@ -27,6 +27,26 @@ defmodule Exgit.ObjectStore.MemoryTest do refute Memory.has_object?(store, :crypto.hash(:sha, "other")) end + test "delete_object removes the object AND its size index entry" do + store = Memory.new() + {:ok, sha, store} = Memory.put_object(store, Blob.new("hello")) + assert {:ok, 5} = Memory.object_size(store, sha) + + assert {:ok, freed, store} = Memory.delete_object(store, sha) + assert freed > 0 + + refute Memory.has_object?(store, sha) + assert {:error, :not_found} = Memory.get_object(store, sha) + # The sizes index must be kept in lockstep — no stale size for a + # deleted object. + assert {:error, :not_found} = Memory.object_size(store, sha) + end + + test "delete_object returns not_found for a missing sha" do + assert {:error, :not_found} = + Memory.delete_object(Memory.new(), :crypto.hash(:sha, "nope")) + end + test "import_objects" do content = "imported" sha = Exgit.Object.sha(Blob.new(content)) diff --git a/test/exgit/object_store/promisor_eviction_test.exs b/test/exgit/object_store/promisor_eviction_test.exs index 7c2e5cb..4c5840d 100644 --- a/test/exgit/object_store/promisor_eviction_test.exs +++ b/test/exgit/object_store/promisor_eviction_test.exs @@ -57,6 +57,21 @@ defmodule Exgit.ObjectStore.PromisorEvictionTest do :ok end + test "eviction drops the size index too — object_size never returns a stale size" do + # cap=1 byte: the commit is evicted by the same put that inserted it. + p = Promisor.new(%Stub{}, max_cache_bytes: 1) + {:ok, sha, p} = Promisor.put(p, make_commit("evict me\n")) + + refute Promisor.has_object?(p, sha) + assert {:error, :not_local} = Promisor.object_size(p, sha) + assert {:error, :not_local} = Exgit.ObjectStore.object_size(p, sha) + + # The desync lived in the underlying Memory cache: eviction removed + # the object from `objects` but left its `sizes` entry behind, so a + # direct size lookup returned the stale size for an evicted object. + assert {:error, :not_found} = Exgit.ObjectStore.Memory.object_size(p.cache, sha) + end + test "large cap doesn't evict" do p = Promisor.new(%Stub{}, max_cache_bytes: 10_000) diff --git a/test/exgit/repository_memory_report_test.exs b/test/exgit/repository_memory_report_test.exs index 4483cc6..71da39f 100644 --- a/test/exgit/repository_memory_report_test.exs +++ b/test/exgit/repository_memory_report_test.exs @@ -2,9 +2,10 @@ defmodule Exgit.RepositoryMemoryReportTest do @moduledoc """ Tests `Exgit.Repository.memory_report/1` across the object-store backends we actually ship: `Memory` and `Promisor`. `Disk` and - user-defined stores get a degraded report (placeholders); we - assert that the shape is consistent across all backends so - callers can depend on the keys existing. + user-defined stores are not introspected, so their counts and + `:cache_bytes` are `:unknown`; we assert that the shape is + consistent across all backends so callers can depend on the keys + existing. """ use ExUnit.Case, async: true @@ -112,6 +113,36 @@ defmodule Exgit.RepositoryMemoryReportTest do end end + describe "on a Disk-backed repo" do + # Disk is not introspected: counts and cache_bytes must be + # :unknown, never a fake 0 that monitoring could mistake for + # an empty repo. + setup do + store = ObjectStore.Disk.new("/nonexistent/objects") + repo = Repository.new(store, Exgit.RefStore.Memory.new()) + {:ok, repo: repo} + end + + test "reports :unknown counts and cache_bytes", %{repo: repo} do + report = Repository.memory_report(repo) + + assert report.object_count == :unknown + assert report.cache_bytes == :unknown + assert report.commit_count == :unknown + assert report.tree_count == :unknown + assert report.blob_count == :unknown + assert report.tag_count == :unknown + end + + test "still reports backend, mode, and max_cache_bytes", %{repo: repo} do + report = Repository.memory_report(repo) + + assert report.backend == ObjectStore.Disk + assert report.mode == :eager + assert report.max_cache_bytes == :infinity + end + end + describe "shape invariants" do test "report always has the same keys regardless of backend" do # Memory-backed @@ -127,7 +158,12 @@ defmodule Exgit.RepositoryMemoryReportTest do promisor_keys = empty_promisor |> Repository.memory_report() |> Map.keys() |> Enum.sort() + # Disk-backed (not introspected) + disk = Repository.new(ObjectStore.Disk.new("/nonexistent"), Exgit.RefStore.Memory.new()) + disk_keys = disk |> Repository.memory_report() |> Map.keys() |> Enum.sort() + assert memory_keys == promisor_keys + assert memory_keys == disk_keys end end end From c7ce08122ce00bb4dcba3a2318c5c8197085e37b Mon Sep 17 00:00:00 2001 From: Ivar Vong Date: Wed, 1 Jul 2026 16:44:19 -0400 Subject: [PATCH 12/13] fix(core): push refspec safety, RepoHandle crash recovery, FS API consistency Review findings for 0.1.0: - push/3 defaults to HEAD's symref target instead of hardcoded refs/heads/main; a repo on another branch no longer silently pushes nothing ({:error, :no_refspecs} when HEAD is detached, {:error, {:local_ref_not_found, name}} for unresolvable named refspecs). - String.replace_prefix replaces misused String.trim_leading in tracking-ref mapping and file:// URL handling, so a branch literally named refs/heads/foo maps to the right remote ref. - RepoHandle.fetch_once/4 monitors its fetch task: a task that dies via throw/exit/kill now fails waiters with {:error, {:fetch_crashed, reason}} and clears the in-flight entry instead of poisoning the key forever; safe_fetch catches all three kinds. - RepoHandle.get/1 renamed to fetch!/1, restoring the get/fetch/fetch! convention. fetch_once @spec covers its timeout. - FS.glob/3 returns the plain sorted list (no error path existed); :resolve_lfs_pointers renamed to :detect_lfs_pointers (it never fetched LFS content); gitlink entries short-circuit as {:error, :submodule} in read_path/size and stat as %{type: :submodule} instead of doomed object lookups. - init/1 wraps disk failures as {:error, {:init_failed, reason}}; @doc added to init/open/fetch; shared remote() type across clone/fetch/push specs. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01VEo6srWr3aFwhtwaXDTCZZ --- lib/exgit.ex | 143 ++++++++++++++++++++++---- lib/exgit/error.ex | 4 +- lib/exgit/fs.ex | 113 ++++++++++++++------ lib/exgit/lfs.ex | 6 +- lib/exgit/repo_handle.ex | 117 ++++++++++++++------- lib/exgit/workspace.ex | 2 +- lib/exgit/workspace/vfs.ex | 5 + test/exgit/clone_test.exs | 102 ++++++++++++++++++ test/exgit/fs_prefetch_async_test.exs | 6 +- test/exgit/fs_size_test.exs | 31 +++++- test/exgit/fs_test.exs | 42 +++----- test/exgit/repo_handle_test.exs | 82 +++++++++++---- test/exgit/repo_registry_test.exs | 2 +- 13 files changed, 506 insertions(+), 149 deletions(-) diff --git a/lib/exgit.ex b/lib/exgit.ex index d2e0bb7..7cde28e 100644 --- a/lib/exgit.ex +++ b/lib/exgit.ex @@ -21,6 +21,34 @@ defmodule Exgit do alias Exgit.{Config, ObjectStore, Pack, RefStore, Repository, Transport} + @typedoc """ + A remote endpoint for `clone/2`, `fetch/3`, and `push/3`. + + Either a URL string (`"https://..."` or `"file://..."`), one of the + built-in transport structs (`Exgit.Transport.File`, + `Exgit.Transport.HTTP`), or any struct implementing the + `Exgit.Transport` protocol. + """ + @type remote :: String.t() | Transport.File.t() | Transport.HTTP.t() | struct() + + @doc """ + Create an empty bare repository. + + {:ok, repo} = Exgit.init() + {:ok, repo} = Exgit.init(path: "/tmp/repo.git") + + ## Options + + * `:path` — create the repo on disk at this path, backed by + `ObjectStore.Disk`/`RefStore.Disk`. Default: in-memory + (lost on process exit). + + ## Returns + + `{:ok, %Repository{}}` with `HEAD` pointing at `refs/heads/main`. + When `:path` is set and any directory or file write fails, returns + `{:error, {:init_failed, reason}}` with the underlying posix reason. + """ @spec init(keyword()) :: {:ok, Repository.t()} | {:error, term()} def init(opts \\ []) do case Keyword.get(opts, :path) do @@ -29,6 +57,22 @@ defmodule Exgit do end end + @doc """ + Open an existing on-disk repository. + + {:ok, repo} = Exgit.open("/tmp/repo.git") + + `path` must point at a git directory — a bare repo or a `.git` + directory — containing a `HEAD` file and an `objects` directory. + The repo's `config` file is read when present; a missing or + unparseable config falls back to an empty `Exgit.Config`. + + ## Returns + + `{:ok, %Repository{}}` backed by `ObjectStore.Disk`/`RefStore.Disk`, + or `{:error, {:not_a_repository, reason}}` when the path doesn't + look like a git directory. + """ @spec open(Path.t(), keyword()) :: {:ok, Repository.t()} | {:error, term()} def open(path, _opts \\ []) do head_path = Path.join(path, "HEAD") @@ -114,8 +158,7 @@ defmodule Exgit do `Exgit.Repository.materialize/2` to convert `:lazy` → `:eager` after reading what you need. """ - @spec clone(String.t() | Transport.File.t() | Transport.HTTP.t(), keyword()) :: - {:ok, Repository.t()} | {:error, term()} + @spec clone(remote(), keyword()) :: {:ok, Repository.t()} | {:error, term()} def clone(source, opts \\ []) do cond do disk_partial_clone?(opts) -> @@ -313,7 +356,30 @@ defmodule Exgit do end end - @spec fetch(Repository.t(), String.t() | term(), keyword()) :: + @doc """ + Fetch updates from a remote into an existing repository. + + {:ok, repo} = Exgit.fetch(repo, "https://github.com/user/repo") + + Lists the remote's refs, fetches any objects reachable from them, + and records remote-tracking refs under `refs/remotes//` + (for branches) and `refs/tags/` (for tags). Local branches and + `HEAD` are never moved. + + ## Options + + * `:remote` — remote name to record tracking refs under. + Default: `"origin"`. + + * `:prefix` — ref prefixes to list on the remote. + Default: `["refs/heads/", "refs/tags/"]`. + + ## Returns + + `{:ok, %Repository{}}` with the updated object and ref stores, or + `{:error, reason}` when listing refs or fetching the pack fails. + """ + @spec fetch(Repository.t(), remote(), keyword()) :: {:ok, Repository.t()} | {:error, term()} def fetch(%Repository{} = repo, source, opts \\ []) do transport = to_transport(source, opts) @@ -334,23 +400,57 @@ defmodule Exgit do ref to the same name on the remote (creating it if absent). * `{:delete, "refs/heads/branch"}` — delete the ref on the remote. + When `:refspecs` is omitted, the repo's current branch — `HEAD`'s + symref target — is pushed. If `HEAD` is not a symbolic ref (e.g. + detached or unset), the push fails with `{:error, :no_refspecs}`. + + ## Errors + + * `{:error, :no_refspecs}` — `:refspecs` omitted and `HEAD` + doesn't point at a branch. + * `{:error, {:local_ref_not_found, name}}` — a named refspec + doesn't resolve in the local ref store. The whole push fails; + nothing is sent to the remote. (Deleting a ref the remote + doesn't have remains a silent no-op.) + Returns `{:ok, %{ref_results: [...]}}` on success. """ - @spec push(Repository.t(), String.t() | term(), keyword()) :: + @spec push(Repository.t(), remote(), keyword()) :: {:ok, map()} | {:error, term()} def push(%Repository{} = repo, dest, opts \\ []) do transport = to_transport(dest, opts) - refspecs = Keyword.get(opts, :refspecs, ["refs/heads/main"]) - ref_names = Enum.map(refspecs, &refspec_ref_name/1) - remote_refs = load_remote_refs_for_push(transport, ref_names) - {updates, objects} = plan_push_updates(refspecs, repo, remote_refs) + with {:ok, refspecs} <- push_refspecs(repo, opts) do + ref_names = Enum.map(refspecs, &refspec_ref_name/1) + remote_refs = load_remote_refs_for_push(transport, ref_names) - if updates == [] do - {:ok, %{ref_results: []}} - else - pack = build_push_pack(updates, objects) - Transport.push(transport, Enum.reverse(updates), pack, []) + case plan_push_updates(refspecs, repo, remote_refs) do + {:ok, [], _objects} -> + {:ok, %{ref_results: []}} + + {:ok, updates, objects} -> + pack = build_push_pack(updates, objects) + Transport.push(transport, Enum.reverse(updates), pack, []) + + {:error, _} = err -> + err + end + end + end + + # When the caller doesn't name refspecs, push the current branch — + # HEAD's symref target. A detached or unset HEAD leaves nothing + # sensible to push, so the caller must name refspecs explicitly. + defp push_refspecs(repo, opts) do + case Keyword.fetch(opts, :refspecs) do + {:ok, refspecs} -> + {:ok, refspecs} + + :error -> + case RefStore.read(repo.ref_store, "HEAD") do + {:ok, {:symbolic, ref_name}} -> {:ok, [ref_name]} + _ -> {:error, :no_refspecs} + end end end @@ -365,7 +465,7 @@ defmodule Exgit do end defp plan_push_updates(refspecs, repo, remote_refs) do - Enum.reduce(refspecs, {[], []}, fn refspec, {upd, objs} -> + Enum.reduce_while(refspecs, {:ok, [], []}, fn refspec, {:ok, upd, objs} -> case plan_push(refspec, repo, remote_refs) do {:update, ref_name, old_sha, new_sha, sha_for_objects} -> new_objs = @@ -373,10 +473,13 @@ defmodule Exgit do do: collect_push_objects(repo.object_store, sha_for_objects, remote_refs), else: [] - {[{ref_name, old_sha, new_sha} | upd], objs ++ new_objs} + {:cont, {:ok, [{ref_name, old_sha, new_sha} | upd], objs ++ new_objs}} :skip -> - {upd, objs} + {:cont, {:ok, upd, objs}} + + {:error, _} = err -> + {:halt, err} end end) end @@ -420,7 +523,7 @@ defmodule Exgit do {:update, ref_name, old_sha, sha, sha} _ -> - :skip + {:error, {:local_ref_not_found, ref_name}} end end @@ -430,7 +533,7 @@ defmodule Exgit do # URL string that we can map to the built-in File/HTTP transports. defp to_transport(url, opts) when is_binary(url) do if String.starts_with?(url, "file://") do - path = String.trim_leading(url, "file://") + path = String.replace_prefix(url, "file://", "") Transport.File.new(path) else Transport.HTTP.new(url, opts) @@ -460,6 +563,8 @@ defmodule Exgit do config: config, path: path )} + else + {:error, reason} -> {:error, {:init_failed, reason}} end end @@ -518,7 +623,7 @@ defmodule Exgit do remote_ref = cond do String.starts_with?(ref, "refs/heads/") -> - branch = String.trim_leading(ref, "refs/heads/") + branch = String.replace_prefix(ref, "refs/heads/", "") "refs/remotes/#{remote_name}/#{branch}" String.starts_with?(ref, "refs/tags/") -> diff --git a/lib/exgit/error.ex b/lib/exgit/error.ex index 6f8eb4b..874b1ce 100644 --- a/lib/exgit/error.ex +++ b/lib/exgit/error.ex @@ -20,7 +20,8 @@ defmodule Exgit.Error do Well-known codes: * `:not_found` — object, ref, or path missing. - * `:invalid_ref_name` — ref name failed `Exgit.RefName.valid?/1`. + * `:invalid_ref_name` — ref name failed `Exgit.RefName.valid?/1` + (defensive validation of ref names, including wire input). * `:invalid_hex_header` — a commit/tag header expected 40-char hex and got something else. * `:malformed_tree_entry` — tree decode failed structurally. @@ -30,7 +31,6 @@ defmodule Exgit.Error do * `:zlib_error` — zlib decompression failed on untrusted input. * `:resolved_too_large`, `:object_too_large`, `:pack_too_large` — pack parser resource-limit trip. - * `:invalid_ref_name` — defensive ref name validation tripped. * `:http_error` — non-2xx response from a transport. * `:compare_and_swap_failed` — ref store CAS check failed. diff --git a/lib/exgit/fs.ex b/lib/exgit/fs.ex index a41be19..293eaab 100644 --- a/lib/exgit/fs.ex +++ b/lib/exgit/fs.ex @@ -32,7 +32,11 @@ defmodule Exgit.FS do @type ref :: String.t() | binary() @type path :: String.t() - @type stat :: %{type: :blob | :tree, mode: String.t(), size: non_neg_integer() | nil} + @type stat :: %{ + type: :blob | :tree | :submodule, + mode: String.t(), + size: non_neg_integer() | nil + } @doc """ Read the blob at `path`. Returns `{:ok, {mode, %Blob{}}, repo}` or @@ -41,23 +45,26 @@ defmodule Exgit.FS do ## Options - * `:resolve_lfs_pointers` (default `false`) — when `true`, blobs - detected as git-lfs pointer files are returned as + * `:detect_lfs_pointers` (default `false`) — when `true`, blobs + that parse as git-lfs pointer files are returned as `{:ok, {mode, {:lfs_pointer, info}}, repo}` instead of `{:ok, {mode, %Blob{}}, repo}`. `info` is a map with `:oid`, `:size`, and `:raw` (the original pointer bytes). - An agent reading blobs without this flag against an - LFS-using repo will silently receive ~130-byte pointer - text as if it were file content — a correctness cliff. - See `Exgit.LFS` for detection details. + Detection only — the actual LFS content is never fetched + (that requires a separate batch-API protocol against the LFS + server). Callers that need the real bytes can hand `info.raw` + to an LFS client. An agent reading blobs without this flag + against an LFS-using repo will silently receive ~130-byte + pointer text as if it were file content — a correctness + cliff. See `Exgit.LFS` for detection details. """ @spec read_path(Repository.t(), ref(), path(), keyword()) :: {:ok, {String.t(), Blob.t() | {:lfs_pointer, Exgit.LFS.pointer_info()}}, Repository.t()} - | {:error, :not_found | :not_a_blob | term()} + | {:error, :not_found | :not_a_blob | :submodule | term()} def read_path(%Repository{} = repo, reference, path, opts \\ []) do - resolve_lfs? = Keyword.get(opts, :resolve_lfs_pointers, false) + detect_lfs? = Keyword.get(opts, :detect_lfs_pointers, false) Exgit.Telemetry.span( [:exgit, :fs, :read_path], @@ -65,17 +72,26 @@ defmodule Exgit.FS do fn -> with {:ok, tree_sha, repo} <- resolve_tree(repo, reference), {:ok, {mode, sha}, repo} <- walk_path(repo, tree_sha, normalize_path(path)), + :ok <- reject_gitlink(mode), {:ok, obj, repo} <- fetch_object(repo, sha) do - wrap_blob(obj, mode, repo, resolve_lfs?) + wrap_blob(obj, mode, repo, detect_lfs?) end end ) end + # A gitlink's SHA names a commit in the submodule's own repository — + # it can never be fetched from this repo's store, so fail before + # `fetch_object` attempts a doomed lookup (or, on a lazy clone, a + # doomed network fetch). Mirrors `size/3`. + defp reject_gitlink(mode) do + if gitlink_mode?(mode), do: {:error, :submodule}, else: :ok + end + # Post-process a fetched object into the `read_path` return shape. # Split out so the main `with` chain stays flat; credo flags the # in-line nested `case + if + case` otherwise. - defp wrap_blob(%Blob{data: data} = b, mode, repo, true = _resolve_lfs?) do + defp wrap_blob(%Blob{data: data} = b, mode, repo, true = _detect_lfs?) do case Exgit.LFS.parse(data) do {:ok, info} -> {:ok, {mode, {:lfs_pointer, info}}, repo} {:error, _} -> {:ok, {mode, b}, repo} @@ -246,6 +262,10 @@ defmodule Exgit.FS do @doc """ Stat the path. Returns `{:ok, stat, repo}`. + + Gitlink (submodule) entries stat as `%{type: :submodule, size: nil}` + without fetching anything — the entry's SHA lives in the submodule's + own repository. """ @spec stat(Repository.t(), ref(), path()) :: {:ok, stat(), Repository.t()} | {:error, term()} def stat(%Repository{} = repo, reference, path) do @@ -257,13 +277,8 @@ defmodule Exgit.FS do {:ok, %{type: :tree, mode: "40000", size: nil}, repo} _ -> - with {:ok, {mode, sha}, repo} <- walk_path(repo, tree_sha, segments), - {:ok, obj, repo} <- fetch_object(repo, sha) do - case obj do - %Blob{data: d} -> {:ok, %{type: :blob, mode: mode, size: byte_size(d)}, repo} - %Tree{} -> {:ok, %{type: :tree, mode: mode, size: nil}, repo} - _ -> {:error, :unknown_type} - end + with {:ok, {mode, sha}, repo} <- walk_path(repo, tree_sha, segments) do + stat_entry(repo, mode, sha) end end end @@ -282,7 +297,10 @@ defmodule Exgit.FS do blob has not been materialized yet, returns `{:error, :not_local}` rather than triggering a possibly-multi-GB fetch — call `read_path/4` when you actually want the bytes. Directories return - `{:error, :not_a_blob}`. + `{:error, :not_a_blob}`. Gitlink (submodule) entries return + `{:error, :submodule}` — the entry's SHA names a commit in the + submodule's own repository, so it has no size here and no amount + of prefetching will make it local. {:ok, size, repo} = Exgit.FS.size(repo, "HEAD", "go.mod") @@ -292,12 +310,33 @@ defmodule Exgit.FS do def size(%Repository{} = repo, reference, path) do with {:ok, tree_sha, repo} <- resolve_tree(repo, reference), {:ok, {mode, sha}, repo} <- walk_path(repo, tree_sha, normalize_path(path)) do - if dir_mode?(mode) do - {:error, :not_a_blob} - else - case ObjectStore.object_size(repo.object_store, sha) do - {:ok, size} -> {:ok, size, repo} - {:error, _} = err -> err + cond do + dir_mode?(mode) -> + {:error, :not_a_blob} + + gitlink_mode?(mode) -> + {:error, :submodule} + + true -> + case ObjectStore.object_size(repo.object_store, sha) do + {:ok, size} -> {:ok, size, repo} + {:error, _} = err -> err + end + end + end + end + + # Stat one resolved tree entry. Gitlinks short-circuit before any + # object lookup — the commit they name is never in this repo's store. + defp stat_entry(repo, mode, sha) do + if gitlink_mode?(mode) do + {:ok, %{type: :submodule, mode: mode, size: nil}, repo} + else + with {:ok, obj, repo} <- fetch_object(repo, sha) do + case obj do + %Blob{data: d} -> {:ok, %{type: :blob, mode: mode, size: byte_size(d)}, repo} + %Tree{} -> {:ok, %{type: :tree, mode: mode, size: nil}, repo} + _ -> {:error, :unknown_type} end end end @@ -307,6 +346,9 @@ defmodule Exgit.FS do defp dir_mode?("040000"), do: true defp dir_mode?(_), do: false + defp gitlink_mode?("160000"), do: true + defp gitlink_mode?(_), do: false + @doc """ Return true if the path exists under the given reference. @@ -900,19 +942,22 @@ defmodule Exgit.FS do end @doc """ - Glob paths matching `pattern`. Streaming; does not grow the cache. + List the file paths matching `pattern`, like `Path.wildcard/1`. + + Walks the tree lazily (pure `ObjectStore.get/2`, does not grow the + cache) and returns the sorted list of matching paths. Unlike + `walk/2` and `grep/4` this is not a stream — sorting requires + collecting every match, so the whole tree is traversed before the + call returns. An unmatched pattern returns `[]`. """ - @spec glob(Repository.t(), ref(), String.t()) :: {:ok, [String.t()]} + @spec glob(Repository.t(), ref(), String.t()) :: [String.t()] def glob(%Repository{} = repo, reference, pattern) do regex = compile_glob(pattern) - paths = - walk(repo, reference) - |> Stream.map(&elem(&1, 0)) - |> Stream.filter(&Regex.match?(regex, &1)) - |> Enum.sort() - - {:ok, paths} + walk(repo, reference) + |> Stream.map(&elem(&1, 0)) + |> Stream.filter(&Regex.match?(regex, &1)) + |> Enum.sort() end @type grep_match :: %{ diff --git a/lib/exgit/lfs.ex b/lib/exgit/lfs.ex index fe69d65..625eb65 100644 --- a/lib/exgit/lfs.ex +++ b/lib/exgit/lfs.ex @@ -15,10 +15,12 @@ defmodule Exgit.LFS do text, not the file contents — a silent correctness cliff if the agent doesn't know to check. - `Exgit.FS.read_path/4` with `resolve_lfs_pointers: true` uses + `Exgit.FS.read_path/4` with `detect_lfs_pointers: true` uses `parse/1` here to surface detected pointers as a structured `{:lfs_pointer, info}` tuple instead of returning the pointer - text as if it were content. + text as if it were content. Detection is the whole story — + fetching the pointed-to content from the LFS server is left to + the caller. ## Strictness diff --git a/lib/exgit/repo_handle.ex b/lib/exgit/repo_handle.ex index 4d5cf10..e014538 100644 --- a/lib/exgit/repo_handle.ex +++ b/lib/exgit/repo_handle.ex @@ -40,7 +40,7 @@ defmodule Exgit.RepoHandle do reason — normal stop, crash, `Process.exit/2`, supervisor shutdown — the table is automatically destroyed by the BEAM. Callers holding a dead handle get `{:error, :dead_handle}` from - `get/1`. + `fetch/1`. Clients that want the handle to outlive a supervision tree are responsible for wiring it into the right supervisor themselves. @@ -53,17 +53,17 @@ defmodule Exgit.RepoHandle do {:ok, task} = Exgit.FS.prefetch_async(handle) # Meanwhile, foreground reads work against the current snapshot. - repo_snapshot = Exgit.RepoHandle.get(handle) + repo_snapshot = Exgit.RepoHandle.fetch!(handle) Exgit.FS.grep(repo_snapshot, "HEAD", "auth", max_count: 10) # Wait for prefetch to finish, then do a full-repo search. :ok = Exgit.FS.await_prefetch(task) - fresh_snapshot = Exgit.RepoHandle.get(handle) + fresh_snapshot = Exgit.RepoHandle.fetch!(handle) Exgit.FS.grep(fresh_snapshot, "HEAD", "auth") |> Enum.to_list() - ## Why ETS and not `Agent.get/1` + ## Why ETS and not `Agent.get/2` - `Agent.get/1` still sends a message to the agent process and + `Agent.get/2` still sends a message to the agent process and copies the state back. For a `Repository` with a large Promisor cache that copy would be 10-100 MB per read — unacceptable for a LiveView that reads on every keystroke. @@ -111,25 +111,8 @@ defmodule Exgit.RepoHandle do copy of the repo into this process's mailbox. Safe to call on every hot-loop iteration. - Raises `ArgumentError` if the handle is dead or if the table - doesn't exist. Callers that want to tolerate dead handles - should wrap in `try/rescue` or use `fetch/1`. - """ - @spec get(t()) :: Repository.t() - def get(handle) do - case fetch(handle) do - {:ok, repo} -> - repo - - {:error, reason} -> - raise ArgumentError, - "Exgit.RepoHandle.get/1: #{inspect(reason)} for handle #{inspect(handle)}" - end - end - - @doc """ - Non-raising variant of `get/1`. Returns `{:ok, repo}` on success - or `{:error, :dead_handle}` / `{:error, :no_table}`. + Returns `{:ok, repo}` on success or `{:error, :dead_handle}` / + `{:error, :no_table}`. See `fetch!/1` for a raising variant. """ @spec fetch(t()) :: {:ok, Repository.t()} | {:error, :dead_handle | :no_table} def fetch(handle) do @@ -147,6 +130,24 @@ defmodule Exgit.RepoHandle do ArgumentError -> {:error, :no_table} end + @doc """ + Same as `fetch/1`, but returns the repo directly and raises + `ArgumentError` if the handle is dead or if the table doesn't + exist. Callers that want to tolerate dead handles should use + `fetch/1`. + """ + @spec fetch!(t()) :: Repository.t() + def fetch!(handle) do + case fetch(handle) do + {:ok, repo} -> + repo + + {:error, reason} -> + raise ArgumentError, + "Exgit.RepoHandle.fetch!/1: #{inspect(reason)} for handle #{inspect(handle)}" + end + end + @doc """ Apply `fun` to the current repo value and store the result. @@ -192,7 +193,7 @@ defmodule Exgit.RepoHandle do Without dedup, each caller fires its own identical network call — wasteful. - With `fetch_once/3`: + With `fetch_once/4`: * First caller for `key` runs `fetch_fn(current_repo)` OUTSIDE the handle (in a linked Task) so the handle stays responsive @@ -218,9 +219,16 @@ defmodule Exgit.RepoHandle do ## Errors `{:error, :dead_handle}` if the handle isn't running. Propagates - `fetch_fn`'s errors verbatim. + `fetch_fn`'s errors verbatim. If `fetch_fn` raises, throws, or + exits — or the fetch task is killed — all waiters receive + `{:error, {:fetch_crashed, reason}}`. """ - @spec fetch_once(t(), term(), (Repository.t() -> {:ok, Repository.t()} | {:error, term()})) :: + @spec fetch_once( + t(), + term(), + (Repository.t() -> {:ok, Repository.t()} | {:error, term()}), + timeout() + ) :: {:ok, Repository.t()} | {:error, term()} def fetch_once(handle, key, fetch_fn, timeout \\ 300_000) when is_function(fetch_fn, 1) do GenServer.call(handle, {:fetch_once, key, fetch_fn}, timeout) @@ -261,10 +269,11 @@ defmodule Exgit.RepoHandle do true = :ets.insert(table, {:repo, repo}) - # `in_flight`: %{key => %{task_ref: ref, waiters: [from, ...]}} + # `in_flight`: %{key => %{monitor_ref: ref, waiters: [from, ...]}} # Entries are created when the first fetch_once for a key - # arrives; removed when the task completes and all waiters - # have been replied to. + # arrives; removed when the task completes (or dies — the + # monitor guarantees waiters are always replied to) and all + # waiters have been replied to. {:ok, %{table: table, in_flight: %{}}} end @@ -302,16 +311,27 @@ defmodule Exgit.RepoHandle do # updates. We'll reply to `from` (and any waiters that # register before the task finishes) when it completes. handle_pid = self() + table = state.table - _ = + start_result = Task.Supervisor.start_child(Exgit.TaskSupervisor, fn -> - [{:repo, repo}] = :ets.lookup(state.table, :repo) + [{:repo, repo}] = :ets.lookup(table, :repo) result = safe_fetch(fetch_fn, repo) send(handle_pid, {:fetch_once_done, key, result}) end) - entry = %{waiters: [from]} - {:noreply, %{state | in_flight: Map.put(state.in_flight, key, entry)}} + case start_result do + {:ok, pid} -> + # Monitor the task so waiters get a reply even if it + # dies without sending :fetch_once_done (e.g. a brutal + # kill that safe_fetch can't catch). + monitor_ref = Process.monitor(pid) + entry = %{monitor_ref: monitor_ref, waiters: [from]} + {:noreply, %{state | in_flight: Map.put(state.in_flight, key, entry)}} + + other -> + {:reply, {:error, {:fetch_task_start_failed, other}}, state} + end %{waiters: waiters} = entry -> # Already in-flight for this key. Add to waiters. @@ -328,7 +348,11 @@ defmodule Exgit.RepoHandle do # ourselves somehow. Log and continue. {:noreply, state} - {%{waiters: waiters}, remaining} -> + {%{monitor_ref: monitor_ref, waiters: waiters}, remaining} -> + # The task sent its result before exiting, so the pending + # :DOWN (if any) is stale — flush it. + Process.demonitor(monitor_ref, [:flush]) + # Commit the result to ETS if it succeeded, then reply to # all waiters with the same return value. reply_value = @@ -349,10 +373,29 @@ defmodule Exgit.RepoHandle do end end + def handle_info({:DOWN, ref, :process, _pid, reason}, state) do + # A fetch task died without sending :fetch_once_done (a brutal + # kill — anything catchable is turned into an error tuple by + # safe_fetch). Reply to all waiters so nobody hangs until the + # call timeout, and clear the entry so the next fetch_once for + # this key re-runs the fetch. + case Enum.find(state.in_flight, fn {_key, entry} -> entry.monitor_ref == ref end) do + nil -> + {:noreply, state} + + {key, %{waiters: waiters}} -> + for from <- Enum.reverse(waiters) do + GenServer.reply(from, {:error, {:fetch_crashed, reason}}) + end + + {:noreply, %{state | in_flight: Map.delete(state.in_flight, key)}} + end + end + defp safe_fetch(fun, repo) do fun.(repo) - rescue - e -> {:error, {:fetch_fn_raised, e, __STACKTRACE__}} + catch + kind, reason -> {:error, {:fetch_crashed, {kind, reason, __STACKTRACE__}}} end @impl true diff --git a/lib/exgit/workspace.ex b/lib/exgit/workspace.ex index 74bfb3c..f5ed179 100644 --- a/lib/exgit/workspace.ex +++ b/lib/exgit/workspace.ex @@ -144,7 +144,7 @@ defmodule Exgit.Workspace do end @doc """ - Stat the entry at `path`. Returns `%{type: :blob | :tree, mode:, size:}`. + Stat the entry at `path`. Returns `%{type: :blob | :tree | :submodule, mode:, size:}`. """ @spec stat(t(), String.t()) :: {:ok, FS.stat(), t()} | {:error, term()} def stat(%__MODULE__{} = ws, path) do diff --git a/lib/exgit/workspace/vfs.ex b/lib/exgit/workspace/vfs.ex index b32fab2..9986168 100644 --- a/lib/exgit/workspace/vfs.ex +++ b/lib/exgit/workspace/vfs.ex @@ -64,6 +64,11 @@ if Code.ensure_loaded?(VFS.Mountable) do {:ok, %{type: :tree}, ws} -> {:ok, %Stat{type: :directory, size: 0, mtime: @epoch}, ws} + # A submodule materializes as an (empty) directory, matching + # how `git checkout` presents gitlink entries. + {:ok, %{type: :submodule}, ws} -> + {:ok, %Stat{type: :directory, size: 0, mtime: @epoch}, ws} + {:error, reason} -> {:error, error_for(reason, path)} end diff --git a/test/exgit/clone_test.exs b/test/exgit/clone_test.exs index 9caa87b..a435786 100644 --- a/test/exgit/clone_test.exs +++ b/test/exgit/clone_test.exs @@ -112,5 +112,107 @@ defmodule Exgit.CloneTest do assert {:ok, ^commit_sha} = RefStore.Disk.read_ref(RefStore.Disk.new(origin), "refs/heads/main") end + + test "defaults to pushing HEAD's branch when :refspecs is omitted", + %{origin_path: origin, clone_path: clone} do + seed_origin(origin) + transport = Exgit.Transport.File.new(origin) + {:ok, repo} = Exgit.clone(transport, path: clone) + + store = repo.object_store + blob = Blob.new("default refspec\n") + {:ok, blob_sha} = ObjectStore.Disk.put_object(store, blob) + tree = Tree.new([{"100644", "default.txt", blob_sha}]) + {:ok, tree_sha} = ObjectStore.Disk.put_object(store, tree) + + {:ok, parent_sha} = RefStore.Disk.resolve_ref(repo.ref_store, "refs/heads/main") + + commit = + Commit.new( + tree: tree_sha, + parents: [parent_sha], + author: "Clone 2000000000 +0000", + committer: "Clone 2000000000 +0000", + message: "default refspec\n" + ) + + {:ok, commit_sha} = ObjectStore.Disk.put_object(store, commit) + :ok = RefStore.Disk.write_ref(repo.ref_store, "refs/heads/main", commit_sha) + + # HEAD is symbolic → refs/heads/main (set by clone), so a bare + # push must push that branch. + assert {:ok, %{ref_results: _}} = Exgit.push(repo, transport) + + assert {:ok, ^commit_sha} = + RefStore.Disk.read_ref(RefStore.Disk.new(origin), "refs/heads/main") + end + + test "returns :no_refspecs when :refspecs is omitted and HEAD is not a branch", + %{origin_path: origin} do + seed_origin(origin) + transport = Exgit.Transport.File.new(origin) + + # A fresh memory repo has no HEAD at all — nothing to push. + {:ok, repo} = Exgit.init() + assert {:error, :no_refspecs} = Exgit.push(repo, transport) + end + + test "fails the whole push when a named refspec has no local ref", + %{origin_path: origin, clone_path: clone} do + seed_origin(origin) + transport = Exgit.Transport.File.new(origin) + {:ok, repo} = Exgit.clone(transport, path: clone) + + assert {:error, {:local_ref_not_found, "refs/heads/missing"}} = + Exgit.push(repo, transport, refspecs: ["refs/heads/main", "refs/heads/missing"]) + end + + test "deleting a ref the remote doesn't have stays a silent no-op", + %{origin_path: origin, clone_path: clone} do + seed_origin(origin) + transport = Exgit.Transport.File.new(origin) + {:ok, repo} = Exgit.clone(transport, path: clone) + + assert {:ok, %{ref_results: []}} = + Exgit.push(repo, transport, refspecs: [{:delete, "refs/heads/gone"}]) + end + end + + describe "remote-tracking refs" do + # Regression: `String.trim_leading/2` strips *repeated* prefixes, so a + # branch literally named "refs/heads/foo" (full ref + # "refs/heads/refs/heads/foo") used to be recorded as + # "refs/remotes/origin/foo" instead of + # "refs/remotes/origin/refs/heads/foo". + test "branch named refs/heads/foo maps to the right tracking ref", + %{origin_path: origin} do + commit_sha = seed_origin(origin) + + :ok = + RefStore.Disk.write_ref( + RefStore.Disk.new(origin), + "refs/heads/refs/heads/foo", + commit_sha + ) + + transport = Exgit.Transport.File.new(origin) + assert {:ok, repo} = Exgit.clone(transport) + + assert {:ok, ^commit_sha} = + Exgit.RefStore.read(repo.ref_store, "refs/remotes/origin/refs/heads/foo") + + assert {:error, _} = Exgit.RefStore.read(repo.ref_store, "refs/remotes/origin/foo") + end + end + + describe "file:// URL strings" do + test "clones from a file:// URL", %{origin_path: origin} do + commit_sha = seed_origin(origin) + + assert {:ok, repo} = Exgit.clone("file://" <> origin) + + assert {:ok, %Commit{message: "initial\n"}} = + Exgit.ObjectStore.get(repo.object_store, commit_sha) + end end end diff --git a/test/exgit/fs_prefetch_async_test.exs b/test/exgit/fs_prefetch_async_test.exs index 7737e4a..8853fb3 100644 --- a/test/exgit/fs_prefetch_async_test.exs +++ b/test/exgit/fs_prefetch_async_test.exs @@ -61,10 +61,10 @@ defmodule Exgit.FSPrefetchAsyncTest do end test "handle is updated after task completes", %{handle: handle} do - before = RepoHandle.get(handle) + before = RepoHandle.fetch!(handle) {:ok, task} = FS.prefetch_async(handle) {:ok, :prefetched} = FS.await_prefetch(task) - after_ = RepoHandle.get(handle) + after_ = RepoHandle.fetch!(handle) # For Memory-backed store, the update is a no-op but the # call still runs (which is what we're testing — the task @@ -81,7 +81,7 @@ defmodule Exgit.FSPrefetchAsyncTest do # 100 reads while the task is (was) running. for _ <- 1..100 do - assert %Repository{} = RepoHandle.get(handle) + assert %Repository{} = RepoHandle.fetch!(handle) end {:ok, :prefetched} = FS.await_prefetch(task) diff --git a/test/exgit/fs_size_test.exs b/test/exgit/fs_size_test.exs index b23b13b..6658baa 100644 --- a/test/exgit/fs_size_test.exs +++ b/test/exgit/fs_size_test.exs @@ -93,7 +93,19 @@ defmodule Exgit.FsSizeTest do src_tree = Tree.new([{"40000", "deep", nested_tree_sha}]) {:ok, src_sha, store} = ObjectStore.put(store, src_tree) - root = Tree.new([{"100644", "README.md", readme_sha}, {"40000", "src", src_sha}]) + # Gitlink (submodule) entry: the SHA names a commit in the + # submodule's OWN repository, so it is never present in this + # object store — exactly the situation size/3 must not treat + # as a missing local object. + submodule_commit_sha = :crypto.strong_rand_bytes(20) + + root = + Tree.new([ + {"100644", "README.md", readme_sha}, + {"160000", "vendored", submodule_commit_sha}, + {"40000", "src", src_sha} + ]) + {:ok, root_sha, store} = ObjectStore.put(store, root) commit = @@ -137,6 +149,23 @@ defmodule Exgit.FsSizeTest do assert FS.size(repo, "HEAD", "src") == {:error, :not_a_blob} end + test "returns :submodule for a gitlink entry instead of a doomed lookup", %{repo: repo} do + # Before the short-circuit, this fell through to an + # object_size lookup of the submodule's commit SHA and + # surfaced a misleading :not_found / :not_local. + assert FS.size(repo, "HEAD", "vendored") == {:error, :submodule} + end + + test "read_path and stat agree on gitlink handling", %{repo: repo} do + # read_path must fail the same way as size/3 — before any + # object lookup of the submodule's commit SHA... + assert FS.read_path(repo, "HEAD", "vendored") == {:error, :submodule} + + # ...while stat reports the entry without fetching anything. + assert {:ok, %{type: :submodule, mode: "160000", size: nil}, %Repository{}} = + FS.stat(repo, "HEAD", "vendored") + end + test "returns :not_found for a missing path", %{repo: repo} do assert FS.size(repo, "HEAD", "nope.txt") == {:error, :not_found} end diff --git a/test/exgit/fs_test.exs b/test/exgit/fs_test.exs index 6063a4f..a0779ae 100644 --- a/test/exgit/fs_test.exs +++ b/test/exgit/fs_test.exs @@ -119,7 +119,7 @@ defmodule Exgit.FsTest do end end - describe "read_path/4 with resolve_lfs_pointers" do + describe "read_path/4 with detect_lfs_pointers" do # Build a repo containing: # /real.bin (200 bytes random — looks like a real binary) # /pointed.bin (a valid LFS pointer file, as if `git lfs` @@ -128,7 +128,7 @@ defmodule Exgit.FsTest do # # and verify: # 1. Default behavior (no flag) returns %Blob{} in all cases. - # 2. With resolve_lfs_pointers: true, pointed.bin surfaces as + # 2. With detect_lfs_pointers: true, pointed.bin surfaces as # {:lfs_pointer, info}; real.bin and README.md stay as blobs. setup do store = ObjectStore.Memory.new() @@ -202,7 +202,7 @@ defmodule Exgit.FsTest do test "with flag, pointer file surfaces as {:lfs_pointer, info}", %{repo: repo} do assert {:ok, {"100644", {:lfs_pointer, info}}, _} = - FS.read_path(repo, "HEAD", "pointed.bin", resolve_lfs_pointers: true) + FS.read_path(repo, "HEAD", "pointed.bin", detect_lfs_pointers: true) assert info.oid == "sha256:4d7a214614ab2935c943f9e0ff69d22eadbb8f32b1258daaa5e2ca24d17e2393" @@ -214,14 +214,14 @@ defmodule Exgit.FsTest do test "with flag, normal binary blob stays as %Blob{}", %{repo: repo} do # A 200-byte random blob must NOT be mistaken for a pointer. assert {:ok, {"100644", %Blob{} = blob}, _} = - FS.read_path(repo, "HEAD", "real.bin", resolve_lfs_pointers: true) + FS.read_path(repo, "HEAD", "real.bin", detect_lfs_pointers: true) assert byte_size(blob.data) == 200 end test "with flag, plain text blob stays as %Blob{}", %{repo: repo} do assert {:ok, {"100644", %Blob{data: "# Project\n"}}, _} = - FS.read_path(repo, "HEAD", "README.md", resolve_lfs_pointers: true) + FS.read_path(repo, "HEAD", "README.md", detect_lfs_pointers: true) end end @@ -293,43 +293,33 @@ defmodule Exgit.FsTest do describe "glob/3" do test "matches *.md in the root", %{repo: repo} do - assert {:ok, ["README.md"]} = FS.glob(repo, "HEAD", "*.md") + assert FS.glob(repo, "HEAD", "*.md") == ["README.md"] end - test "matches **/*.ex across subdirs", %{repo: repo} do - {:ok, paths} = FS.glob(repo, "HEAD", "**/*.ex") - - assert Enum.sort(paths) == ["src/a.ex", "src/b.ex", "src/nested/c.ex"] + test "matches **/*.ex across subdirs, sorted", %{repo: repo} do + assert FS.glob(repo, "HEAD", "**/*.ex") == ["src/a.ex", "src/b.ex", "src/nested/c.ex"] end test "matches src/*.ex (one level)", %{repo: repo} do - {:ok, paths} = FS.glob(repo, "HEAD", "src/*.ex") - assert Enum.sort(paths) == ["src/a.ex", "src/b.ex"] + assert FS.glob(repo, "HEAD", "src/*.ex") == ["src/a.ex", "src/b.ex"] end test "brace expansion: **/*.{md,ex} matches both extensions", %{repo: repo} do - {:ok, paths} = FS.glob(repo, "HEAD", "**/*.{md,ex}") - - assert Enum.sort(paths) == + assert FS.glob(repo, "HEAD", "**/*.{md,ex}") == ["README.md", "src/a.ex", "src/b.ex", "src/nested/c.ex"] end test "brace with a single option is equivalent to the literal", %{repo: repo} do - {:ok, a} = FS.glob(repo, "HEAD", "**/*.{ex}") - {:ok, b} = FS.glob(repo, "HEAD", "**/*.ex") - assert Enum.sort(a) == Enum.sort(b) + assert FS.glob(repo, "HEAD", "**/*.{ex}") == FS.glob(repo, "HEAD", "**/*.ex") end test "brace expansion at different positions in the pattern", %{repo: repo} do # {src,lib}/*.ex — no `lib/` exists, so only src hits match. - {:ok, paths} = FS.glob(repo, "HEAD", "{src,lib}/*.ex") - assert Enum.sort(paths) == ["src/a.ex", "src/b.ex"] + assert FS.glob(repo, "HEAD", "{src,lib}/*.ex") == ["src/a.ex", "src/b.ex"] end test "brace expansion with three options", %{repo: repo} do - {:ok, paths} = FS.glob(repo, "HEAD", "**/*.{md,ex,missing}") - - assert Enum.sort(paths) == + assert FS.glob(repo, "HEAD", "**/*.{md,ex,missing}") == ["README.md", "src/a.ex", "src/b.ex", "src/nested/c.ex"] end @@ -337,15 +327,13 @@ defmodule Exgit.FsTest do # `foo{,bar}` matches either `foo` or `foobar`. We exercise the # parse path; the fixture has no files matching either literal, # so we expect an empty result — but no crash. - {:ok, paths} = FS.glob(repo, "HEAD", "README{,.bak}.md") - assert paths == ["README.md"] + assert FS.glob(repo, "HEAD", "README{,.bak}.md") == ["README.md"] end test "unmatched opening brace is treated as a literal character", %{repo: repo} do # `{` without a closing `}` isn't a valid alternation — we treat # it as a literal (matches against a nonexistent file). - {:ok, paths} = FS.glob(repo, "HEAD", "README{.md") - assert paths == [] + assert FS.glob(repo, "HEAD", "README{.md") == [] end end diff --git a/test/exgit/repo_handle_test.exs b/test/exgit/repo_handle_test.exs index a3fcc3a..ee66661 100644 --- a/test/exgit/repo_handle_test.exs +++ b/test/exgit/repo_handle_test.exs @@ -38,9 +38,9 @@ defmodule Exgit.RepoHandleTest do end describe "lifecycle" do - test "start_link + get", %{repo: repo} do + test "start_link + fetch!", %{repo: repo} do {:ok, handle} = RepoHandle.start_link(repo) - assert %Repository{} = RepoHandle.get(handle) + assert %Repository{} = RepoHandle.fetch!(handle) RepoHandle.stop(handle) end @@ -50,7 +50,7 @@ defmodule Exgit.RepoHandleTest do RepoHandle.stop(handle) end - test "get/1 on dead handle raises" do + test "fetch!/1 on dead handle raises" do # Spawn a handle and kill it. {:ok, handle} = RepoHandle.start_link(%Repository{ @@ -63,7 +63,7 @@ defmodule Exgit.RepoHandleTest do RepoHandle.stop(handle) refute Process.alive?(handle) - assert_raise ArgumentError, fn -> RepoHandle.get(handle) end + assert_raise ArgumentError, fn -> RepoHandle.fetch!(handle) end end test "fetch/1 on dead handle returns {:error, _}" do @@ -83,8 +83,8 @@ defmodule Exgit.RepoHandleTest do name = :"test_named_handle_#{System.unique_integer([:positive])}" {:ok, pid} = RepoHandle.start_link(repo, name: name) - assert %Repository{} = RepoHandle.get(name) - assert RepoHandle.get(name) == RepoHandle.get(pid) + assert %Repository{} = RepoHandle.fetch!(name) + assert RepoHandle.fetch!(name) == RepoHandle.fetch!(pid) RepoHandle.stop(name) end @@ -117,7 +117,7 @@ defmodule Exgit.RepoHandleTest do %{r | object_store: store} end) - new_repo = RepoHandle.get(handle) + new_repo = RepoHandle.fetch!(handle) # Store has 5 objects now (blob + tree + commit + new blob = 4, plus... wait) # Originally 3 objects (blob, tree, commit). After update, 4. assert map_size(new_repo.object_store.objects) == 4 @@ -134,18 +134,18 @@ defmodule Exgit.RepoHandleTest do {:ok, %{r | object_store: store}} end) - assert map_size(RepoHandle.get(handle).object_store.objects) == 4 + assert map_size(RepoHandle.fetch!(handle).object_store.objects) == 4 RepoHandle.stop(handle) end test "fun returning {:error, _} leaves handle unchanged", %{repo: repo} do {:ok, handle} = RepoHandle.start_link(repo) - snapshot_before = RepoHandle.get(handle) + snapshot_before = RepoHandle.fetch!(handle) result = RepoHandle.update(handle, fn _r -> {:error, :test_error} end) assert result == {:error, :test_error} - assert RepoHandle.get(handle) == snapshot_before + assert RepoHandle.fetch!(handle) == snapshot_before RepoHandle.stop(handle) end @@ -170,7 +170,7 @@ defmodule Exgit.RepoHandleTest do Enum.each(tasks, &Task.await/1) # Initial 3 (commit+tree+blob) + n new blobs. - assert map_size(RepoHandle.get(handle).object_store.objects) == 3 + n_tasks + assert map_size(RepoHandle.fetch!(handle).object_store.objects) == 3 + n_tasks RepoHandle.stop(handle) end end @@ -185,13 +185,13 @@ defmodule Exgit.RepoHandleTest do :ok = RepoHandle.put(handle, new_repo) - stored = RepoHandle.get(handle) + stored = RepoHandle.fetch!(handle) assert map_size(stored.object_store.objects) == 4 RepoHandle.stop(handle) end end - describe "fetch_once/3 dedup" do + describe "fetch_once/4 dedup" do test "serializes concurrent fetches for the same key", %{repo: repo} do {:ok, handle} = RepoHandle.start_link(repo) @@ -267,6 +267,44 @@ defmodule Exgit.RepoHandleTest do RepoHandle.stop(handle) end + test "fetch fn that exits errors promptly instead of hanging", %{repo: repo} do + {:ok, handle} = RepoHandle.start_link(repo) + + # The 1s call timeout makes the test fail fast if the caller + # would otherwise hang until the default 300s timeout. + assert {:error, {:fetch_crashed, _}} = + RepoHandle.fetch_once(handle, :exit_key, fn _repo -> exit(:boom) end, 1_000) + + # The in_flight entry is cleared: a subsequent fetch_once for + # the SAME key re-runs the fetch and succeeds. + assert {:ok, %Repository{}} = + RepoHandle.fetch_once(handle, :exit_key, fn r -> {:ok, r} end, 1_000) + + RepoHandle.stop(handle) + end + + test "killed fetch task errors promptly instead of hanging", %{repo: repo} do + {:ok, handle} = RepoHandle.start_link(repo) + + # A brutal kill can't be caught inside the task, so this + # exercises the monitor path: the handle sees :DOWN and + # replies to waiters instead of leaking the in_flight entry. + kill_fn = fn _repo -> Process.exit(self(), :kill) end + + # Reason is :killed, or :noproc when the task dies before the + # handle's monitor attaches — both go through the :DOWN path. + assert {:error, {:fetch_crashed, reason}} = + RepoHandle.fetch_once(handle, :kill_key, kill_fn, 1_000) + + assert reason in [:killed, :noproc] + + # The entry is cleared: the same key fetches again and succeeds. + assert {:ok, %Repository{}} = + RepoHandle.fetch_once(handle, :kill_key, fn r -> {:ok, r} end, 1_000) + + RepoHandle.stop(handle) + end + test "handle stays responsive to reads during a slow fetch", %{repo: repo} do {:ok, handle} = RepoHandle.start_link(repo) @@ -284,7 +322,7 @@ defmodule Exgit.RepoHandleTest do start = System.monotonic_time() for _ <- 1..100 do - _ = RepoHandle.get(handle) + _ = RepoHandle.fetch!(handle) end us = @@ -299,15 +337,15 @@ defmodule Exgit.RepoHandleTest do end describe "read performance" do - test "get/1 does not send a message to the handle", %{repo: repo} do + test "fetch!/1 does not send a message to the handle", %{repo: repo} do {:ok, handle} = RepoHandle.start_link(repo) # Capture the process's message queue length before and after - # many get/1 calls. If get/1 used GenServer.call, the handle - # would receive N :sys messages — but it's not the test pid's - # queue that grows, it's the handle's. Instead we check that - # get/1 returns fast even while the handle is busy in a - # slow update. + # many fetch!/1 calls. If fetch!/1 used GenServer.call, the + # handle would receive N :sys messages — but it's not the test + # pid's queue that grows, it's the handle's. Instead we check + # that fetch!/1 returns fast even while the handle is busy in + # a slow update. slow_update_task = Task.async(fn -> RepoHandle.update( @@ -326,7 +364,7 @@ defmodule Exgit.RepoHandleTest do start = System.monotonic_time() for _ <- 1..100 do - _ = RepoHandle.get(handle) + _ = RepoHandle.fetch!(handle) end reads_us = @@ -336,7 +374,7 @@ defmodule Exgit.RepoHandleTest do # If reads went through the GenServer they'd block behind the # 200ms sleep and this would be ~200ms. assert reads_us < 10_000, - "100 get/1 calls took #{reads_us}µs — likely blocked on GenServer, indicating a regression" + "100 fetch!/1 calls took #{reads_us}µs — likely blocked on GenServer, indicating a regression" Task.await(slow_update_task) RepoHandle.stop(handle) diff --git a/test/exgit/repo_registry_test.exs b/test/exgit/repo_registry_test.exs index 469f0cf..acd1e2d 100644 --- a/test/exgit/repo_registry_test.exs +++ b/test/exgit/repo_registry_test.exs @@ -161,7 +161,7 @@ defmodule Exgit.RepoRegistryTest do assert {:ok, ^handle} = RepoRegistry.lookup(url) # Can fetch through the handle. - assert %Exgit.Repository{} = RepoHandle.get(handle) + assert %Exgit.Repository{} = RepoHandle.fetch!(handle) RepoHandle.stop(handle) end From 80df2e8247c8eba7a6d38abe3e565db9a4b6ff38 Mon Sep 17 00:00:00 2001 From: Ivar Vong Date: Wed, 1 Jul 2026 16:44:29 -0400 Subject: [PATCH 13/13] docs: consolidate CHANGELOG into 0.1.0, refresh README claims for release MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fold [Unreleased] into the [0.1.0] entry dated 2026-07-01 — nothing was ever published, so the first hex release carries it all. - README: real test counts (870 across default/slow/real_git), the actual three-tier CI matrix, and the consumer-install smoke job. - mix.exs: docs/PERFORMANCE.md added to ex_doc extras so the README link resolves on hexdocs.pm instead of 404ing; stale CI-matrix comment corrected. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01VEo6srWr3aFwhtwaXDTCZZ --- CHANGELOG.md | 34 ++++++++++++++++++++-------------- README.md | 13 ++++++++----- mix.exs | 8 ++++---- 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e444c7..6526173 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,14 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). -## [Unreleased] +## [0.1.0] — 2026-07-01 + +Initial release: pure-Elixir git client for clone, fetch, push over +smart HTTP v2, with lazy partial-clone support and a path-oriented FS +API for agents. + +See [README](./README.md) and [BENCHMARKS on the smoketest +repo](https://github.com/ivarvong/exgit_smoketest/blob/main/BENCHMARKS.md). ### Security — credential redaction + ref bounds @@ -18,10 +25,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 `:auth` field regardless — it was already redacted. - **`ls-refs` responses are capped at 1,000,000 refs.** A hostile or broken server can no longer stream unbounded refs into client - memory; the transport stream halts once the cap trips. Real repos - (linux, esp-idf) sit far below this. + memory; the transport stream halts once the cap trips and the + caller gets `{:error, {:too_many_refs, cap}}`. Tunable via the + `:max_refs` option on `Exgit.Transport.HTTP.new/2`. Real repos + (linux, esp-idf) sit far below the default. - **Dependencies bumped to clear HTTP-stack advisories.** `req` - `0.5.17 → 0.6.2` and `mint `1.7.1 → 1.9.0` resolve the decompression- + `0.5.17 → 0.6.2` and `mint` `1.7.1 → 1.9.0` resolve the decompression- bomb DoS (CVE-2026-49755), multipart header injection (CVE-2026-49756), and HTTP/2 CONTINUATION flood (CVE-2026-49754). The cross-origin credential-leak test suite was re-run against the @@ -47,7 +56,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 the header. Resolving the path may fetch trees (small) on a lazy clone, but never the blob — an un-fetched blob returns `{:error, :not_local}` instead of triggering a possibly-multi-GB - fetch. Directories return `{:error, :not_a_blob}`. + fetch. Directories return `{:error, :not_a_blob}`; gitlink + (submodule) entries return `{:error, :submodule}` — as do + `read_path/4` lookups on them, while `stat/3` reports + `%{type: :submodule}` without fetching. - **`Exgit.ObjectStore.object_size/2`** — new protocol callback backing the above. Memory keeps a parallel `sha => size` index (no extra decompression); `Promisor` answers from cache or @@ -64,7 +76,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **`Exgit.Repository.memory_report/1`** — structured memory report (object counts by type, cache_bytes, max_cache_bytes, mode, backend) with consistent shape across all object-store - backends. Suitable for emission into observability stacks. + backends; counts report `:unknown` for backends that can't be + introspected (Disk). Suitable for emission into observability + stacks. - **`bench/agent_workload.exs`** — realistic agent-session benchmark: clone + prefetch + ls + grep + reads, with `:cold` and `:hot` variants. Reports per-op breakdown + peak cache @@ -438,11 +452,3 @@ taking the opportunity to land the right shapes before v0.1. corrupt or tampered objects return tagged errors instead of crashing. -## [0.1.0] — 2026-04-17 - -Initial release: pure-Elixir git client for clone, fetch, push over -smart HTTP v2, with lazy partial-clone support and a path-oriented FS -API for agents. - -See [README](./README.md) and [BENCHMARKS on the smoketest -repo](https://github.com/ivarvong/exgit_smoketest/blob/main/BENCHMARKS.md). diff --git a/README.md b/README.md index 5cf766a..9d22121 100644 --- a/README.md +++ b/README.md @@ -338,11 +338,14 @@ What's in place: eviction for long-running agent loops. - **Protocol v2 `symrefs`** — `Exgit.clone/2` picks the server's actual HEAD target instead of guessing main/master. -- **399 tests, 29 properties, 0 failures** across default, slow, - real_git, and live-integration tiers. -- **CI gates**: Elixir 1.17 / OTP 27 on ubuntu-24.04 with - warnings-as-errors, Credo, Dialyzer, format check, partial-clone - roundtrip against GitHub. +- **870 tests (52 properties), 0 failures** across the default, + slow, and real_git tiers; live-integration and authenticated + tiers run in CI on top. +- **CI gates**: three-tier matrix on ubuntu-24.04 — Elixir 1.20 / + OTP 29 (primary: warnings-as-errors, Credo, Dialyzer, format + check, partial-clone roundtrip against GitHub), plus 1.19 / OTP 28 + and 1.17 / OTP 27 (minimum-supported) compile-and-test tiers, and + a consumer-install smoke test of the built hex package. See [CHANGELOG.md](./CHANGELOG.md) for details and [SECURITY.md](./SECURITY.md) for the threat model. diff --git a/mix.exs b/mix.exs index 3790b67..5a23d8c 100644 --- a/mix.exs +++ b/mix.exs @@ -9,10 +9,10 @@ defmodule Exgit.MixProject do app: :exgit, version: @version, elixir: "~> 1.17", - # The library doesn't use any 1.19-only features; pinning the + # The library doesn't use any post-1.17 features; pinning the # lower bound at 1.17 keeps it installable for older consumers. - # CI matrix tests both 1.17 (minimum-supported) and 1.19 - # (primary + stricter type checks). + # CI matrix tests 1.20/OTP 29 (primary, all quality gates), + # 1.19/OTP 28, and 1.17/OTP 27 (minimum-supported). start_permanent: Mix.env() == :prod, deps: deps(), elixirc_paths: elixirc_paths(Mix.env()), @@ -61,7 +61,7 @@ defmodule Exgit.MixProject do defp docs do [ main: "readme", - extras: ["README.md", "CHANGELOG.md", "SECURITY.md"], + extras: ["README.md", "CHANGELOG.md", "SECURITY.md", "docs/PERFORMANCE.md"], source_url: @source_url, source_ref: "v#{@version}" ]