From b661897e2fa15caf08c179107e1238c2a922e182 Mon Sep 17 00:00:00 2001 From: Alex Kesling Date: Thu, 5 Mar 2026 23:10:23 -0500 Subject: [PATCH 1/2] feat: add toolpath-github crate for GitHub PR provenance Derive Toolpath Path documents from GitHub pull requests via the REST API. Every PR event (commits, inline review comments, PR discussion comments, review decisions, CI check runs) becomes a Step on a single linear trunk ordered by timestamp. Platform-agnostic artifact URIs (review://, ci://) keep the format provider-neutral. Includes CLI integration (derive github, list github), PR URL parsing, and 31 unit + 3 doc tests with JSON fixtures (no network calls). --- CHANGELOG.md | 13 + CLAUDE.md | 7 +- Cargo.lock | 691 +++++++++- Cargo.toml | 3 + README.md | 6 + crates/toolpath-cli/Cargo.toml | 1 + crates/toolpath-cli/src/cmd_derive.rs | 86 ++ crates/toolpath-cli/src/cmd_list.rs | 71 ++ crates/toolpath-cli/tests/integration.rs | 30 +- crates/toolpath-github/Cargo.toml | 19 + crates/toolpath-github/README.md | 88 ++ crates/toolpath-github/src/lib.rs | 1476 ++++++++++++++++++++++ scripts/release.sh | 7 +- site/_data/crates.json | 8 + site/pages/crates.md | 1 + 15 files changed, 2485 insertions(+), 22 deletions(-) create mode 100644 crates/toolpath-github/Cargo.toml create mode 100644 crates/toolpath-github/README.md create mode 100644 crates/toolpath-github/src/lib.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 7e62e82..db43c41 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ All notable changes to the Toolpath workspace are documented here. +## 0.1.0 — toolpath-github + +### toolpath-github 0.1.0 + +- New crate: derive Toolpath Path documents from GitHub pull requests via the REST API +- Every PR event becomes a Step: commits (with per-file diffs), inline review comments, PR discussion comments, review decisions (approve/reject), and CI check runs +- Platform-agnostic artifact URIs: `review://` for code review artifacts, `ci://` for CI artifacts +- `derive_pull_request()` fetches all data and builds a complete provenance DAG +- `list_pull_requests()` lists PRs with summary metadata +- `resolve_token()` checks `GITHUB_TOKEN` env var, falls back to `gh auth token` +- Configurable: `--no-ci` and `--no-comments` flags to exclude non-code events +- CLI: `path derive github --repo owner/repo --pr 42` and `path list github --repo owner/repo` + ## 0.5.0 — toolpath-convo / 0.6.2 — toolpath-claude ### toolpath-convo 0.5.0 diff --git a/CLAUDE.md b/CLAUDE.md index 49aea45..38b4f1d 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -14,6 +14,7 @@ crates/ toolpath/ # core types, builders, serde, query API toolpath-convo/ # provider-agnostic conversation types and traits toolpath-git/ # derive from git repos (git2) + toolpath-github/ # derive from GitHub pull requests (REST API) toolpath-claude/ # derive from Claude conversation logs toolpath-dot/ # Graphviz DOT rendering toolpath-cli/ # unified CLI (binary: path) @@ -30,6 +31,7 @@ toolpath-cli (binary: path) ├── toolpath (core types) ├── toolpath-convo (conversation abstraction) ├── toolpath-git → toolpath + ├── toolpath-github → toolpath ├── toolpath-claude → toolpath, toolpath-convo └── toolpath-dot → toolpath ``` @@ -52,6 +54,7 @@ The binary is called `path` (package: `toolpath-cli`): ```bash cargo run -p toolpath-cli -- derive git --repo . --branch main --pretty +cargo run -p toolpath-cli -- derive github --repo owner/repo --pr 42 --pretty cargo run -p toolpath-cli -- derive claude --project /path/to/project cargo run -p toolpath-cli -- render dot --input doc.json cargo run -p toolpath-cli -- query dead-ends --input doc.json @@ -59,6 +62,7 @@ cargo run -p toolpath-cli -- query ancestors --input doc.json --step-id step-003 cargo run -p toolpath-cli -- query filter --input doc.json --actor "agent:" cargo run -p toolpath-cli -- merge doc1.json doc2.json --title "Combined" cargo run -p toolpath-cli -- list git --repo . +cargo run -p toolpath-cli -- list github --repo owner/repo cargo run -p toolpath-cli -- track init --file src/main.rs --actor "human:alex" cargo run -p toolpath-cli -- validate --input doc.json ``` @@ -78,6 +82,7 @@ Tests live alongside the code (`#[cfg(test)] mod tests`), plus `toolpath-cli` ha - `toolpath`: 32 unit + 9 doc tests (serde roundtrip, builders, query) - `toolpath-convo`: 28 unit + 1 doc test (types, enrichment, display) - `toolpath-git`: 33 unit + 3 doc tests (derive, branch detection, diffstat) +- `toolpath-github`: 28 unit + 2 doc tests (mapping, DAG construction, fixtures) - `toolpath-claude`: 216 unit + 5 doc tests (path resolution, conversation reading, query, chaining, watcher, derive) - `toolpath-dot`: 30 unit + 2 doc tests (render, visual conventions, escaping) - `toolpath-cli`: 120 unit + 8 integration tests (all commands, track sessions, merge, validate, roundtrip) @@ -111,7 +116,7 @@ When changing a crate's public API (new types, new trait impls, new public metho **Release script** (`scripts/release.sh`) publishes in dependency order: - Tier 1: `toolpath`, `toolpath-convo` (no workspace deps) -- Tier 2: `toolpath-git`, `toolpath-dot`, `toolpath-claude` (depend on tier 1) +- Tier 2: `toolpath-git`, `toolpath-github`, `toolpath-dot`, `toolpath-claude` (depend on tier 1) - Tier 3: `toolpath-cli` (depends on everything) Build the site after changes: `cd site && pnpm run build` (should produce 7 pages). diff --git a/Cargo.lock b/Cargo.lock index c9bf6d5..d41d141 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -91,12 +91,24 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "bitflags" version = "1.3.2" @@ -210,6 +222,26 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -233,6 +265,15 @@ dependencies = [ "syn", ] +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -281,12 +322,33 @@ dependencies = [ "num-traits", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -305,6 +367,66 @@ dependencies = [ "libc", ] +[[package]] +name = "futures-channel" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07bbe89c50d7a535e539b8c17bc0b49bdb77747034daa8087407d655f3f7cc1d" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-io" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cecba35d7ad927e23624b22ad55235f2239cfa44fd10428eecbeba6d6a717718" + +[[package]] +name = "futures-sink" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c39754e157331b013978ec91992bde1ac089843443c49cbc7f46150b0fad0893" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "slab", +] + +[[package]] +name = "getrandom" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff2abc00be7fca6ebc474524697ae276ad847ad0a6b3faa4bcb027e9a4614ad0" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + [[package]] name = "getrandom" version = "0.3.4" @@ -340,11 +462,30 @@ dependencies = [ "libc", "libgit2-sys", "log", - "openssl-probe", + "openssl-probe 0.1.6", "openssl-sys", "url", ] +[[package]] +name = "h2" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f44da3a8150a6703ed5d34e164b875fd14c2cdab9af1252a9a1020bde2bdc54" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -366,6 +507,124 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "http" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3ba2a386d7f85a81f119ad7498ebe444d2e22c2af0b86b069416ace48b3311a" +dependencies = [ + "bytes", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" +dependencies = [ + "bytes", + "futures-core", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" + +[[package]] +name = "hyper" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab2d4f250c3d7b1c9fcdff1cece94ea4e2dfbec68614f7b87cb205f24ca9d11" +dependencies = [ + "atomic-waker", + "bytes", + "futures-channel", + "futures-core", + "h2", + "http", + "http-body", + "httparse", + "itoa", + "pin-project-lite", + "pin-utils", + "smallvec", + "tokio", + "want", +] + +[[package]] +name = "hyper-rustls" +version = "0.27.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "rustls-pki-types", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + +[[package]] +name = "hyper-util" +version = "0.1.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" +dependencies = [ + "base64", + "bytes", + "futures-channel", + "futures-util", + "http", + "http-body", + "hyper", + "ipnet", + "libc", + "percent-encoding", + "pin-project-lite", + "socket2", + "system-configuration", + "tokio", + "tower-service", + "tracing", + "windows-registry", +] + [[package]] name = "iana-time-zone" version = "0.1.65" @@ -539,6 +798,22 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "iri-string" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c91338f0783edbd6195decb37bae672fd3b165faffb89bf7b9e6942f8b1a731a" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "is_terminal_polyfill" version = "1.70.2" @@ -687,6 +962,12 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + [[package]] name = "mio" version = "1.1.1" @@ -699,6 +980,23 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe 0.2.1", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "normalize-line-endings" version = "0.3.0" @@ -754,12 +1052,44 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" +[[package]] +name = "openssl" +version = "0.10.75" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" +dependencies = [ + "bitflags 2.10.0", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "openssl-probe" version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + [[package]] name = "openssl-sys" version = "0.9.111" @@ -807,6 +1137,12 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b" +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkg-config" version = "0.3.32" @@ -971,6 +1307,62 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.17", + "libc", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rustix" version = "1.1.3" @@ -984,12 +1376,51 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "rustls" +version = "0.23.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +dependencies = [ + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7df23109aa6c1567d1c575b9952556388da57401e4ace1d15f79eedad0d8f53" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + [[package]] name = "same-file" version = "1.0.6" @@ -999,12 +1430,44 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "schannel" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "3.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d17b898a6d6948c3a8ee4372c17cb384f90d2e6e912ef00895b14fd7ab54ec38" +dependencies = [ + "bitflags 2.10.0", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.27" @@ -1054,6 +1517,18 @@ dependencies = [ "zmij", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + [[package]] name = "shlex" version = "1.3.0" @@ -1076,6 +1551,12 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "smallvec" version = "1.15.1" @@ -1104,6 +1585,12 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + [[package]] name = "syn" version = "2.0.115" @@ -1115,6 +1602,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + [[package]] name = "synstructure" version = "0.13.2" @@ -1126,6 +1622,27 @@ dependencies = [ "syn", ] +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags 2.10.0", + "core-foundation 0.9.4", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tempfile" version = "3.25.0" @@ -1203,6 +1720,39 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ae9cec805b01e8fc3fd2fe289f89149a9b66dd16786abd8b19cfa7b48cb0098" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + [[package]] name = "toolpath" version = "0.1.5" @@ -1246,6 +1796,7 @@ dependencies = [ "toolpath-claude", "toolpath-dot", "toolpath-git", + "toolpath-github", ] [[package]] @@ -1276,6 +1827,88 @@ dependencies = [ "toolpath", ] +[[package]] +name = "toolpath-github" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "reqwest", + "serde", + "serde_json", + "toolpath", +] + +[[package]] +name = "tower" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebe5ef63511595f1344e2d5cfa636d973292adc0eec1f0ad45fae9f0851ab1d4" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper", + "tokio", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-http" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" +dependencies = [ + "bitflags 2.10.0", + "bytes", + "futures-util", + "http", + "http-body", + "iri-string", + "pin-project-lite", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + +[[package]] +name = "tracing" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63e71662fa4b2a2c3a26f570f037eb95bb1f85397f3cd8076caed2f026a6d100" +dependencies = [ + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db97caf9d906fbde555dd62fa95ddba9eecfd14cb388e4f491a66d74cd5fb79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "unicode-ident" version = "1.0.23" @@ -1288,6 +1921,12 @@ version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.8" @@ -1337,6 +1976,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.1+wasi-snapshot-preview1" @@ -1374,6 +2022,20 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70a6e77fd0ae8029c9ea0063f87c46fde723e7d887703d74ad2616d792e51e6f" +dependencies = [ + "cfg-if", + "futures-util", + "js-sys", + "once_cell", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.108" @@ -1440,6 +2102,16 @@ dependencies = [ "semver", ] +[[package]] +name = "web-sys" +version = "0.3.85" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "312e32e551d92129218ea9a2452120f4aabc03529ef03e4d0d82fb2780608598" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi-util" version = "0.1.11" @@ -1490,6 +2162,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + [[package]] name = "windows-result" version = "0.4.1" @@ -1822,6 +2505,12 @@ dependencies = [ "synstructure", ] +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" + [[package]] name = "zerotrie" version = "0.2.3" diff --git a/Cargo.toml b/Cargo.toml index 874ef02..5760431 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,6 +3,7 @@ members = [ "crates/toolpath", "crates/toolpath-convo", "crates/toolpath-git", + "crates/toolpath-github", "crates/toolpath-claude", "crates/toolpath-dot", "crates/toolpath-cli", @@ -18,8 +19,10 @@ toolpath = { version = "0.1.5", path = "crates/toolpath" } toolpath-convo = { version = "0.5.0", path = "crates/toolpath-convo" } toolpath-git = { version = "0.1.3", path = "crates/toolpath-git" } toolpath-claude = { version = "0.6.2", path = "crates/toolpath-claude", default-features = false } +toolpath-github = { version = "0.1.0", path = "crates/toolpath-github" } toolpath-dot = { version = "0.1.2", path = "crates/toolpath-dot" } +reqwest = { version = "0.12", features = ["blocking", "json"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" clap = { version = "4.5", features = ["derive"] } diff --git a/README.md b/README.md index dccfe67..3db7cb2 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,7 @@ crates/ toolpath/ Core types, builders, query API toolpath-convo/ Provider-agnostic conversation types and traits toolpath-git/ Derive from git repository history + toolpath-github/ Derive from GitHub pull requests toolpath-claude/ Derive from Claude conversation logs toolpath-dot/ Graphviz DOT visualization toolpath-cli/ Unified CLI (binary: path) @@ -62,6 +63,9 @@ path derive git --repo . --branch main --pretty # Visualize it path derive git --repo . --branch main | path render dot | dot -Tpng -o graph.png +# Derive from a GitHub pull request +path derive github --repo owner/repo --pr 42 --pretty + # Derive from Claude conversation logs path derive claude --project /path/to/project --pretty @@ -87,9 +91,11 @@ path validate --input examples/step-01-minimal.json path list git [--repo PATH] [--remote NAME] [--json] + github --repo OWNER/REPO [--json] claude [--project PATH] [--json] derive git --repo PATH --branch NAME[:START] [--base COMMIT] [--remote NAME] [--title TEXT] + github --repo OWNER/REPO --pr NUMBER [--no-ci] [--no-comments] claude --project PATH [--session ID] [--all] query ancestors --input FILE --step-id ID diff --git a/crates/toolpath-cli/Cargo.toml b/crates/toolpath-cli/Cargo.toml index ec3b12d..5433207 100644 --- a/crates/toolpath-cli/Cargo.toml +++ b/crates/toolpath-cli/Cargo.toml @@ -27,6 +27,7 @@ rand = "0.9" [target.'cfg(not(target_os = "emscripten"))'.dependencies] toolpath-claude = { workspace = true, features = ["watcher"] } +toolpath-github = { workspace = true } git2 = { workspace = true } [target.'cfg(target_os = "emscripten")'.dependencies] diff --git a/crates/toolpath-cli/src/cmd_derive.rs b/crates/toolpath-cli/src/cmd_derive.rs index 9688863..281a0c0 100644 --- a/crates/toolpath-cli/src/cmd_derive.rs +++ b/crates/toolpath-cli/src/cmd_derive.rs @@ -28,6 +28,28 @@ pub enum DeriveSource { #[arg(long)] title: Option, }, + /// Derive from a GitHub pull request + Github { + /// PR URL (e.g. https://github.com/owner/repo/pull/42) + #[arg(index = 1)] + url: Option, + + /// Repository in owner/repo format (alternative to URL) + #[arg(short, long)] + repo: Option, + + /// Pull request number (required with --repo) + #[arg(long)] + pr: Option, + + /// Exclude CI check runs + #[arg(long)] + no_ci: bool, + + /// Exclude reviews and comments + #[arg(long)] + no_comments: bool, + }, /// Derive from Claude conversation logs Claude { /// Project path (e.g., /Users/alex/myproject) @@ -53,6 +75,13 @@ pub fn run(source: DeriveSource, pretty: bool) -> Result<()> { remote, title, } => run_git(repo, branch, base, remote, title, pretty), + DeriveSource::Github { + url, + repo, + pr, + no_ci, + no_comments, + } => run_github(url, repo, pr, no_ci, no_comments, pretty), DeriveSource::Claude { project, session, @@ -107,6 +136,63 @@ fn run_git( } } +fn run_github( + url: Option, + repo: Option, + pr: Option, + no_ci: bool, + no_comments: bool, + pretty: bool, +) -> Result<()> { + #[cfg(target_os = "emscripten")] + { + let _ = (url, repo, pr, no_ci, no_comments, pretty); + anyhow::bail!("'path derive github' requires a native environment with network access"); + } + + #[cfg(not(target_os = "emscripten"))] + { + // Resolve owner/repo/pr from either a URL or --repo/--pr flags + let (owner, repo_name, pr_number) = if let Some(url_str) = &url { + let parsed = toolpath_github::parse_pr_url(url_str).ok_or_else(|| { + anyhow::anyhow!("Invalid PR URL. Expected: https://github.com/owner/repo/pull/N") + })?; + (parsed.owner, parsed.repo, parsed.number) + } else if let (Some(repo_str), Some(pr_num)) = (&repo, pr) { + let (o, r) = repo_str + .split_once('/') + .ok_or_else(|| anyhow::anyhow!("Repository must be in owner/repo format"))?; + (o.to_string(), r.to_string(), pr_num) + } else { + anyhow::bail!( + "Provide a PR URL or both --repo and --pr.\n\ + Usage: path derive github https://github.com/owner/repo/pull/42\n\ + Usage: path derive github --repo owner/repo --pr 42" + ); + }; + + let token = toolpath_github::resolve_token()?; + let config = toolpath_github::DeriveConfig { + token, + include_ci: !no_ci, + include_comments: !no_comments, + ..Default::default() + }; + + let path = toolpath_github::derive_pull_request(&owner, &repo_name, pr_number, &config)?; + let doc = toolpath::v1::Document::Path(path); + + let json = if pretty { + doc.to_json_pretty()? + } else { + doc.to_json()? + }; + + println!("{}", json); + Ok(()) + } +} + fn run_claude(project: String, session: Option, all: bool, pretty: bool) -> Result<()> { let manager = toolpath_claude::ClaudeConvo::new(); run_claude_with_manager(&manager, project, session, all, pretty) diff --git a/crates/toolpath-cli/src/cmd_list.rs b/crates/toolpath-cli/src/cmd_list.rs index af8f75f..f691bda 100644 --- a/crates/toolpath-cli/src/cmd_list.rs +++ b/crates/toolpath-cli/src/cmd_list.rs @@ -16,6 +16,12 @@ pub enum ListSource { #[arg(long, default_value = "origin")] remote: String, }, + /// List GitHub pull requests + Github { + /// Repository in owner/repo format + #[arg(short, long)] + repo: String, + }, /// List Claude projects or sessions Claude { /// Project path — if omitted, lists all projects @@ -27,6 +33,7 @@ pub enum ListSource { pub fn run(source: ListSource, json: bool) -> Result<()> { match source { ListSource::Git { repo, remote } => run_git(repo, remote, json), + ListSource::Github { repo } => run_github(repo, json), ListSource::Claude { project } => run_claude(project, json), } } @@ -88,6 +95,70 @@ fn run_git(repo_path: PathBuf, remote: String, json: bool) -> Result<()> { } } +fn run_github(repo: String, json: bool) -> Result<()> { + #[cfg(target_os = "emscripten")] + { + let _ = (repo, json); + anyhow::bail!("'path list github' requires a native environment with network access"); + } + + #[cfg(not(target_os = "emscripten"))] + { + let (owner, repo_name) = repo + .split_once('/') + .ok_or_else(|| anyhow::anyhow!("Repository must be in owner/repo format"))?; + + let token = toolpath_github::resolve_token()?; + let config = toolpath_github::DeriveConfig { + token, + ..Default::default() + }; + + let prs = toolpath_github::list_pull_requests(owner, repo_name, &config)?; + + if json { + let items: Vec = prs + .iter() + .map(|pr| { + serde_json::json!({ + "number": pr.number, + "title": pr.title, + "state": pr.state, + "author": pr.author, + "head_branch": pr.head_branch, + "base_branch": pr.base_branch, + "created_at": pr.created_at, + "updated_at": pr.updated_at, + }) + }) + .collect(); + let output = serde_json::json!({ + "source": "github", + "repo": format!("{}/{}", owner, repo_name), + "pull_requests": items, + }); + println!("{}", serde_json::to_string_pretty(&output)?); + } else { + println!("Pull requests for {}/{}:", owner, repo_name); + println!(); + if prs.is_empty() { + println!(" (none)"); + } else { + for pr in &prs { + println!( + " #{:<5} {:>8} {} {}", + pr.number, + pr.state, + pr.author, + truncate(&pr.title, 50), + ); + } + } + } + Ok(()) + } +} + fn run_claude(project: Option, json: bool) -> Result<()> { let manager = toolpath_claude::ClaudeConvo::new(); diff --git a/crates/toolpath-cli/tests/integration.rs b/crates/toolpath-cli/tests/integration.rs index f8ac7d8..d2cfae2 100644 --- a/crates/toolpath-cli/tests/integration.rs +++ b/crates/toolpath-cli/tests/integration.rs @@ -36,9 +36,7 @@ fn git_fixture() -> (tempfile::TempDir, String) { // Commit 1 let mut index = repo.index().unwrap(); std::fs::write(dir.path().join("main.rs"), "fn main() {}").unwrap(); - index - .add_path(std::path::Path::new("main.rs")) - .unwrap(); + index.add_path(std::path::Path::new("main.rs")).unwrap(); index.write().unwrap(); let tree1 = repo.find_tree(index.write_tree().unwrap()).unwrap(); let sig = repo.signature().unwrap(); @@ -49,20 +47,11 @@ fn git_fixture() -> (tempfile::TempDir, String) { // Commit 2 std::fs::write(dir.path().join("main.rs"), "fn main() { fixed() }").unwrap(); - index - .add_path(std::path::Path::new("main.rs")) - .unwrap(); + index.add_path(std::path::Path::new("main.rs")).unwrap(); index.write().unwrap(); let tree2 = repo.find_tree(index.write_tree().unwrap()).unwrap(); - repo.commit( - Some("HEAD"), - &sig, - &sig, - "fix the bug", - &tree2, - &[&commit1], - ) - .unwrap(); + repo.commit(Some("HEAD"), &sig, &sig, "fix the bug", &tree2, &[&commit1]) + .unwrap(); // Determine the branch name (main or master depending on git config) let head = repo.head().unwrap(); @@ -172,7 +161,10 @@ fn derive_git_has_change_with_diff() { // The step should have a change for main.rs with a raw diff let change = &step["change"]["main.rs"]; let raw = change["raw"].as_str().unwrap(); - assert!(raw.contains("-fn main() {}"), "diff should show old content"); + assert!( + raw.contains("-fn main() {}"), + "diff should show old content" + ); assert!( raw.contains("+fn main() { fixed() }"), "diff should show new content" @@ -223,7 +215,11 @@ fn derive_git_has_base_uri() { // base.uri should be a file:// URL pointing to the repo let uri = base["uri"].as_str().unwrap(); - assert!(uri.starts_with("file://"), "Expected file:// URI, got {}", uri); + assert!( + uri.starts_with("file://"), + "Expected file:// URI, got {}", + uri + ); // base.ref should be a commit hash (40 hex chars) let git_ref = base["ref"].as_str().unwrap(); diff --git a/crates/toolpath-github/Cargo.toml b/crates/toolpath-github/Cargo.toml new file mode 100644 index 0000000..1281633 --- /dev/null +++ b/crates/toolpath-github/Cargo.toml @@ -0,0 +1,19 @@ +[package] +name = "toolpath-github" +version = "0.1.0" +edition.workspace = true +license.workspace = true +repository = "https://github.com/empathic/toolpath" +description = "Derive Toolpath provenance documents from GitHub pull requests" +keywords = ["github", "provenance", "toolpath", "audit", "traceability"] +categories = ["development-tools"] + +[dependencies] +toolpath = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +anyhow = { workspace = true } +chrono = { workspace = true } + +[target.'cfg(not(target_os = "emscripten"))'.dependencies] +reqwest = { workspace = true } diff --git a/crates/toolpath-github/README.md b/crates/toolpath-github/README.md new file mode 100644 index 0000000..d768e4a --- /dev/null +++ b/crates/toolpath-github/README.md @@ -0,0 +1,88 @@ +# toolpath-github + +Derive Toolpath provenance documents from GitHub pull requests. + +A pull request captures more than code changes — reviews, inline comments, +CI checks, and discussion all contribute to the final artifact. This crate +maps the full PR lifecycle into a single Toolpath Path where every event +becomes a Step in the provenance DAG. + +## Overview + +Uses the GitHub REST API. Every PR event type becomes a Step with typed +actors and artifact changes. Commits carry code diffs; reviews and comments +are changes to `review://` artifacts; CI checks are changes to `ci://` artifacts. + +## Usage + +```rust,no_run +use toolpath_github::{derive_pull_request, resolve_token, DeriveConfig}; + +let token = resolve_token()?; +let config = DeriveConfig { + token, + include_ci: true, + include_comments: true, + ..Default::default() +}; + +let path = derive_pull_request("owner", "repo", 42, &config)?; +# Ok::<(), anyhow::Error>(()) +``` + +## Artifact URI scheme + +| Artifact type | URI pattern | Example | +|---|---|---| +| Source file | bare path (relative to base) | `src/main.rs` | +| Review comment thread | `review://{file}#L{line}` | `review://src/main.rs#L42` | +| PR conversation | `review://conversation` | `review://conversation` | +| Review decision | `review://decision` | `review://decision` | +| CI check result | `ci://checks/{name}` | `ci://checks/build` | + +The `review://` and `ci://` schemes are platform-agnostic — they generalize +to GitLab MRs, Gerrit, Phabricator, etc. + +## Mapping + +| GitHub concept | Toolpath type | Details | +|---|---|---| +| Pull request | Path | id: `pr-{number}` | +| Commit | Step | actor: `human:{login}`, per-file raw diffs | +| Review comment | Step | artifact: `review://{path}#L{line}` | +| PR comment | Step | artifact: `review://conversation` | +| Review (approve/reject) | Step | artifact: `review://decision` | +| CI check run | Step | actor: `ci:{app}`, artifact: `ci://checks/{name}` | + +## API + +| Function | Description | +|---|---| +| `resolve_token()` | Resolve GitHub token from `GITHUB_TOKEN` or `gh auth token` | +| `derive_pull_request(owner, repo, pr, config)` | Derive a Path from a PR | +| `list_pull_requests(owner, repo, config)` | List PRs with summary info | +| `extract_issue_refs(body)` | Parse "Fixes #N" / "Closes #N" from text | + +## CLI + +```bash +# Derive a Toolpath document from a GitHub PR +path derive github --repo owner/repo --pr 42 --pretty + +# Without CI checks or comments +path derive github --repo owner/repo --pr 42 --no-ci --no-comments + +# List pull requests +path list github --repo owner/repo --json +``` + +## Part of Toolpath + +This crate is part of the [Toolpath](https://github.com/empathic/toolpath) workspace. See also: + +- [`toolpath`](https://crates.io/crates/toolpath) -- core types and query API +- [`toolpath-git`](https://crates.io/crates/toolpath-git) -- derive from git repository history +- [`toolpath-claude`](https://crates.io/crates/toolpath-claude) -- derive from Claude conversations +- [`toolpath-dot`](https://crates.io/crates/toolpath-dot) -- Graphviz DOT rendering +- [`toolpath-cli`](https://crates.io/crates/toolpath-cli) -- unified CLI (`cargo install toolpath-cli`) +- [RFC](https://github.com/empathic/toolpath/blob/main/RFC.md) -- full format specification diff --git a/crates/toolpath-github/src/lib.rs b/crates/toolpath-github/src/lib.rs new file mode 100644 index 0000000..1b04d38 --- /dev/null +++ b/crates/toolpath-github/src/lib.rs @@ -0,0 +1,1476 @@ +#![doc = include_str!("../README.md")] + +// ============================================================================ +// Public configuration and types (available on all targets) +// ============================================================================ + +/// Configuration for deriving Toolpath documents from a GitHub pull request. +pub struct DeriveConfig { + /// GitHub API token. + pub token: String, + /// GitHub API base URL (default: `https://api.github.com`). + pub api_url: String, + /// Include CI check runs as Steps (default: true). + pub include_ci: bool, + /// Include reviews and comments as Steps (default: true). + pub include_comments: bool, +} + +impl Default for DeriveConfig { + fn default() -> Self { + Self { + token: String::new(), + api_url: "https://api.github.com".to_string(), + include_ci: true, + include_comments: true, + } + } +} + +/// Summary information about a pull request. +#[derive(Debug, Clone)] +pub struct PullRequestInfo { + /// PR number. + pub number: u64, + /// PR title. + pub title: String, + /// PR state (open, closed, merged). + pub state: String, + /// PR author login. + pub author: String, + /// Head branch name. + pub head_branch: String, + /// Base branch name. + pub base_branch: String, + /// ISO 8601 creation timestamp. + pub created_at: String, + /// ISO 8601 last update timestamp. + pub updated_at: String, +} + +// ============================================================================ +// Public pure-data helpers (available on all targets) +// ============================================================================ + +/// Parsed components of a GitHub PR URL. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PrUrl { + /// Repository owner. + pub owner: String, + /// Repository name. + pub repo: String, + /// Pull request number. + pub number: u64, +} + +/// Parse a GitHub PR URL into its components. +/// +/// Accepts URLs like `https://github.com/owner/repo/pull/42` or +/// `github.com/owner/repo/pull/42` (without protocol prefix). +/// Returns `None` if the URL doesn't match the expected format. +/// +/// # Examples +/// +/// ``` +/// use toolpath_github::parse_pr_url; +/// +/// let pr = parse_pr_url("https://github.com/empathic/toolpath/pull/6").unwrap(); +/// assert_eq!(pr.owner, "empathic"); +/// assert_eq!(pr.repo, "toolpath"); +/// assert_eq!(pr.number, 6); +/// +/// // Works without protocol prefix too +/// let pr = parse_pr_url("github.com/empathic/toolpath/pull/6").unwrap(); +/// assert_eq!(pr.number, 6); +/// +/// assert!(parse_pr_url("not a url").is_none()); +/// ``` +pub fn parse_pr_url(url: &str) -> Option { + let rest = url + .strip_prefix("https://github.com/") + .or_else(|| url.strip_prefix("http://github.com/")) + .or_else(|| url.strip_prefix("github.com/"))?; + let parts: Vec<&str> = rest.splitn(4, '/').collect(); + if parts.len() >= 4 && parts[2] == "pull" { + let number = parts[3].split(&['/', '?', '#'][..]).next()?.parse().ok()?; + Some(PrUrl { + owner: parts[0].to_string(), + repo: parts[1].to_string(), + number, + }) + } else { + None + } +} + +/// Extract issue references from PR body text. +/// +/// Recognizes "Fixes #N", "Closes #N", "Resolves #N" (case-insensitive). +/// +/// # Examples +/// +/// ``` +/// use toolpath_github::extract_issue_refs; +/// +/// let refs = extract_issue_refs("This PR fixes #42 and closes #99."); +/// assert_eq!(refs, vec![42, 99]); +/// ``` +pub fn extract_issue_refs(body: &str) -> Vec { + let mut refs = Vec::new(); + let lower = body.to_lowercase(); + for keyword in &["fixes", "closes", "resolves"] { + let mut search_from = 0; + while let Some(pos) = lower[search_from..].find(keyword) { + let after = search_from + pos + keyword.len(); + // Skip optional whitespace and '#' + let rest = &body[after..]; + let rest = rest.trim_start(); + if let Some(rest) = rest.strip_prefix('#') { + let num_str: String = rest.chars().take_while(|c| c.is_ascii_digit()).collect(); + if let Ok(n) = num_str.parse::() + && !refs.contains(&n) + { + refs.push(n); + } + } + search_from = after; + } + } + refs +} + +// ============================================================================ +// reqwest-dependent code (native targets only) +// ============================================================================ + +#[cfg(not(target_os = "emscripten"))] +mod native { + use anyhow::{Context, Result, bail}; + use std::collections::HashMap; + use toolpath::v1::{ + ActorDefinition, ArtifactChange, Base, Identity, Path, PathIdentity, PathMeta, Ref, Step, + StepIdentity, StepMeta, StructuralChange, + }; + + use super::{DeriveConfig, PullRequestInfo, extract_issue_refs}; + + // ==================================================================== + // Auth + // ==================================================================== + + /// Resolve a GitHub API token. + /// + /// Checks `GITHUB_TOKEN` environment variable first, then falls back to + /// `gh auth token` subprocess. Returns an error if neither works. + pub fn resolve_token() -> Result { + if let Ok(token) = std::env::var("GITHUB_TOKEN") + && !token.is_empty() + { + return Ok(token); + } + + let output = std::process::Command::new("gh") + .args(["auth", "token"]) + .output() + .context( + "Failed to run 'gh auth token'. Set GITHUB_TOKEN or install the GitHub CLI (gh).", + )?; + + if output.status.success() { + let token = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if !token.is_empty() { + return Ok(token); + } + } + + bail!( + "No GitHub token found. Set GITHUB_TOKEN environment variable \ + or authenticate with 'gh auth login'." + ) + } + + // ==================================================================== + // API Client + // ==================================================================== + + struct GitHubClient { + client: reqwest::blocking::Client, + token: String, + base_url: String, + } + + impl GitHubClient { + fn new(config: &DeriveConfig) -> Result { + let client = reqwest::blocking::Client::builder() + .user_agent("toolpath-github") + .build() + .context("Failed to build HTTP client")?; + + Ok(Self { + client, + token: config.token.clone(), + base_url: config.api_url.clone(), + }) + } + + fn get_json(&self, endpoint: &str) -> Result { + let url = format!("{}{}", self.base_url, endpoint); + let resp = self + .client + .get(&url) + .header("Authorization", format!("Bearer {}", self.token)) + .header("Accept", "application/vnd.github+json") + .header("X-GitHub-Api-Version", "2022-11-28") + .send() + .with_context(|| format!("Request failed: GET {}", url))?; + + let status = resp.status(); + if !status.is_success() { + let body = resp.text().unwrap_or_default(); + bail!("GitHub API error {}: {}", status, body); + } + + resp.json::() + .with_context(|| format!("Failed to parse JSON from {}", url)) + } + + fn get_paginated(&self, endpoint: &str) -> Result> { + let mut all = Vec::new(); + let mut url = format!("{}{}?per_page=100", self.base_url, endpoint); + + loop { + let resp = self + .client + .get(&url) + .header("Authorization", format!("Bearer {}", self.token)) + .header("Accept", "application/vnd.github+json") + .header("X-GitHub-Api-Version", "2022-11-28") + .send() + .with_context(|| format!("Request failed: GET {}", url))?; + + let status = resp.status(); + if !status.is_success() { + let body = resp.text().unwrap_or_default(); + bail!("GitHub API error {}: {}", status, body); + } + + // Parse Link header for next page + let next_url = resp + .headers() + .get("link") + .and_then(|v| v.to_str().ok()) + .and_then(parse_next_link); + + let page: Vec = resp + .json() + .with_context(|| format!("Failed to parse JSON from {}", url))?; + + all.extend(page); + + match next_url { + Some(next) => url = next, + None => break, + } + } + + Ok(all) + } + } + + fn parse_next_link(header: &str) -> Option { + for part in header.split(',') { + let part = part.trim(); + if part.ends_with("rel=\"next\"") { + // Extract URL between < and > + if let Some(start) = part.find('<') + && let Some(end) = part.find('>') + { + return Some(part[start + 1..end].to_string()); + } + } + } + None + } + + // ==================================================================== + // Public API + // ==================================================================== + + /// Derive a Toolpath [`Path`] from a GitHub pull request. + /// + /// Fetches PR metadata, commits, reviews, comments, and CI checks from the + /// GitHub API, then maps them into a Toolpath Path document where every + /// event becomes a Step in the DAG. + pub fn derive_pull_request( + owner: &str, + repo: &str, + pr_number: u64, + config: &DeriveConfig, + ) -> Result { + let client = GitHubClient::new(config)?; + let prefix = format!("/repos/{}/{}", owner, repo); + + // Fetch all data + let pr = client.get_json(&format!("{}/pulls/{}", prefix, pr_number))?; + let commits = client.get_paginated(&format!("{}/pulls/{}/commits", prefix, pr_number))?; + + // Fetch full commit details (for file patches) + let mut commit_details = Vec::new(); + for c in &commits { + let sha = c["sha"].as_str().unwrap_or_default(); + if !sha.is_empty() { + let detail = client.get_json(&format!("{}/commits/{}", prefix, sha))?; + commit_details.push(detail); + } + } + + let reviews = if config.include_comments { + client.get_paginated(&format!("{}/pulls/{}/reviews", prefix, pr_number))? + } else { + Vec::new() + }; + + let pr_comments = if config.include_comments { + client.get_paginated(&format!("{}/issues/{}/comments", prefix, pr_number))? + } else { + Vec::new() + }; + + let review_comments = if config.include_comments { + client.get_paginated(&format!("{}/pulls/{}/comments", prefix, pr_number))? + } else { + Vec::new() + }; + + // Fetch CI checks for each commit + let mut check_runs_by_sha: HashMap> = HashMap::new(); + if config.include_ci { + for c in &commits { + let sha = c["sha"].as_str().unwrap_or_default(); + if !sha.is_empty() { + let checks = + client.get_json(&format!("{}/commits/{}/check-runs", prefix, sha))?; + if let Some(runs) = checks["check_runs"].as_array() { + check_runs_by_sha.insert(sha.to_string(), runs.clone()); + } + } + } + } + + let data = PrData { + pr: &pr, + commit_details: &commit_details, + reviews: &reviews, + pr_comments: &pr_comments, + review_comments: &review_comments, + check_runs_by_sha: &check_runs_by_sha, + }; + + derive_from_data(&data, owner, repo, config) + } + + /// List open pull requests for a repository. + pub fn list_pull_requests( + owner: &str, + repo: &str, + config: &DeriveConfig, + ) -> Result> { + let client = GitHubClient::new(config)?; + let prs = client.get_paginated(&format!("/repos/{}/{}/pulls?state=all", owner, repo))?; + + let mut result = Vec::new(); + for pr in &prs { + result.push(PullRequestInfo { + number: pr["number"].as_u64().unwrap_or(0), + title: str_field(pr, "title"), + state: str_field(pr, "state"), + author: pr["user"]["login"] + .as_str() + .unwrap_or("unknown") + .to_string(), + head_branch: pr["head"]["ref"].as_str().unwrap_or("unknown").to_string(), + base_branch: pr["base"]["ref"].as_str().unwrap_or("unknown").to_string(), + created_at: str_field(pr, "created_at"), + updated_at: str_field(pr, "updated_at"), + }); + } + + Ok(result) + } + + // ==================================================================== + // Pure derivation (testable without network) + // ==================================================================== + + struct PrData<'a> { + pr: &'a serde_json::Value, + commit_details: &'a [serde_json::Value], + reviews: &'a [serde_json::Value], + pr_comments: &'a [serde_json::Value], + review_comments: &'a [serde_json::Value], + check_runs_by_sha: &'a HashMap>, + } + + fn derive_from_data( + data: &PrData<'_>, + owner: &str, + repo: &str, + config: &DeriveConfig, + ) -> Result { + let pr = data.pr; + let commit_details = data.commit_details; + let reviews = data.reviews; + let pr_comments = data.pr_comments; + let review_comments = data.review_comments; + let check_runs_by_sha = data.check_runs_by_sha; + let pr_number = pr["number"].as_u64().unwrap_or(0); + + // ── Commit steps ───────────────────────────────────────────── + let mut steps: Vec = Vec::new(); + let mut actors: HashMap = HashMap::new(); + + for detail in commit_details { + let step = commit_to_step(detail, &mut actors)?; + steps.push(step); + } + + // ── Review comment steps ───────────────────────────────────── + if config.include_comments { + for rc in review_comments { + let step = review_comment_to_step(rc, &mut actors)?; + steps.push(step); + } + + for pc in pr_comments { + let step = pr_comment_to_step(pc, &mut actors)?; + steps.push(step); + } + + for review in reviews { + let state = review["state"].as_str().unwrap_or(""); + if state.is_empty() || state == "PENDING" { + continue; + } + let step = review_to_step(review, &mut actors)?; + steps.push(step); + } + } + + // ── CI check steps ─────────────────────────────────────────── + if config.include_ci { + for runs in check_runs_by_sha.values() { + for run in runs { + let step = check_run_to_step(run, &mut actors)?; + steps.push(step); + } + } + } + + // ── Sort by timestamp, then chain into a single trunk ──────── + // Everything in a PR is part of one timeline. Commits, comments, + // reviews, and CI checks all chain linearly — none are dead ends + // or alternate explorations. Sort by time, then re-parent each + // step to point at the previous one. + steps.sort_by(|a, b| a.step.timestamp.cmp(&b.step.timestamp)); + + let mut prev_id: Option = None; + for step in &mut steps { + if let Some(ref prev) = prev_id { + step.step.parents = vec![prev.clone()]; + } else { + step.step.parents = vec![]; + } + prev_id = Some(step.step.id.clone()); + } + + // ── Build path head ────────────────────────────────────────── + let head = steps + .last() + .map(|s| s.step.id.clone()) + .unwrap_or_else(|| format!("pr-{}", pr_number)); + + // ── Build path metadata ────────────────────────────────────── + let meta = build_path_meta(pr, &actors)?; + + Ok(Path { + path: PathIdentity { + id: format!("pr-{}", pr_number), + base: Some(Base { + uri: format!("github:{}/{}", owner, repo), + ref_str: Some(pr["base"]["ref"].as_str().unwrap_or("main").to_string()), + }), + head, + }, + steps, + meta: Some(meta), + }) + } + + // ==================================================================== + // Mapping helpers + // ==================================================================== + + fn commit_to_step( + detail: &serde_json::Value, + actors: &mut HashMap, + ) -> Result { + let sha = detail["sha"].as_str().unwrap_or_default(); + let short_sha = &sha[..sha.len().min(8)]; + let step_id = format!("step-{}", short_sha); + + // Actor + let login = detail["author"]["login"].as_str().unwrap_or("unknown"); + let actor = format!("human:{}", login); + register_actor(actors, &actor, login, None); + + // Timestamp + let timestamp = detail["commit"]["committer"]["date"] + .as_str() + .unwrap_or("1970-01-01T00:00:00Z") + .to_string(); + + // Changes: per-file raw diffs + let mut change: HashMap = HashMap::new(); + if let Some(files) = detail["files"].as_array() { + for file in files { + let filename = file["filename"].as_str().unwrap_or("unknown"); + if let Some(patch) = file["patch"].as_str() { + change.insert(filename.to_string(), ArtifactChange::raw(patch)); + } + } + } + + // Intent: first line of commit message + let message = detail["commit"]["message"].as_str().unwrap_or(""); + let intent = message.lines().next().unwrap_or("").to_string(); + + let mut step = Step { + step: StepIdentity { + id: step_id, + parents: vec![], + actor, + timestamp, + }, + change, + meta: None, + }; + + if !intent.is_empty() { + step.meta = Some(StepMeta { + intent: Some(intent), + source: Some(toolpath::v1::VcsSource { + vcs_type: "git".to_string(), + revision: sha.to_string(), + change_id: None, + extra: HashMap::new(), + }), + ..Default::default() + }); + } + + Ok(step) + } + + fn review_comment_to_step( + rc: &serde_json::Value, + actors: &mut HashMap, + ) -> Result { + let id = rc["id"].as_u64().unwrap_or(0); + let step_id = format!("step-rc-{}", id); + + let login = rc["user"]["login"].as_str().unwrap_or("unknown"); + let actor = format!("human:{}", login); + register_actor(actors, &actor, login, None); + + let timestamp = rc["created_at"] + .as_str() + .unwrap_or("1970-01-01T00:00:00Z") + .to_string(); + + let path = rc["path"].as_str().unwrap_or("unknown"); + let line = rc["line"] + .as_u64() + .or_else(|| rc["original_line"].as_u64()) + .unwrap_or(0); + let artifact_uri = format!("review://{}#L{}", path, line); + + let body = rc["body"].as_str().unwrap_or("").to_string(); + + let mut extra = HashMap::new(); + extra.insert("body".to_string(), serde_json::Value::String(body)); + + let change = HashMap::from([( + artifact_uri, + ArtifactChange { + raw: None, + structural: Some(StructuralChange { + change_type: "review.comment".to_string(), + extra, + }), + }, + )]); + + Ok(Step { + step: StepIdentity { + id: step_id, + parents: vec![], + actor, + timestamp, + }, + change, + meta: None, + }) + } + + fn pr_comment_to_step( + pc: &serde_json::Value, + actors: &mut HashMap, + ) -> Result { + let id = pc["id"].as_u64().unwrap_or(0); + let step_id = format!("step-ic-{}", id); + + let timestamp = pc["created_at"] + .as_str() + .unwrap_or("1970-01-01T00:00:00Z") + .to_string(); + + let login = pc["user"]["login"].as_str().unwrap_or("unknown"); + let actor = format!("human:{}", login); + register_actor(actors, &actor, login, None); + + let body = pc["body"].as_str().unwrap_or("").to_string(); + + let change = HashMap::from([( + "review://conversation".to_string(), + ArtifactChange { + raw: Some(body), + structural: None, + }, + )]); + + Ok(Step { + step: StepIdentity { + id: step_id, + parents: vec![], + actor, + timestamp, + }, + change, + meta: None, + }) + } + + fn review_to_step( + review: &serde_json::Value, + actors: &mut HashMap, + ) -> Result { + let id = review["id"].as_u64().unwrap_or(0); + let step_id = format!("step-rv-{}", id); + + let timestamp = review["submitted_at"] + .as_str() + .unwrap_or("1970-01-01T00:00:00Z") + .to_string(); + + let login = review["user"]["login"].as_str().unwrap_or("unknown"); + let actor = format!("human:{}", login); + register_actor(actors, &actor, login, None); + + let state = review["state"].as_str().unwrap_or("COMMENTED").to_string(); + let body = review["body"].as_str().unwrap_or("").to_string(); + + let mut extra = HashMap::new(); + extra.insert("state".to_string(), serde_json::Value::String(state)); + + let change = HashMap::from([( + "review://decision".to_string(), + ArtifactChange { + raw: if body.is_empty() { None } else { Some(body) }, + structural: Some(StructuralChange { + change_type: "review.decision".to_string(), + extra, + }), + }, + )]); + + Ok(Step { + step: StepIdentity { + id: step_id, + parents: vec![], + actor, + timestamp, + }, + change, + meta: None, + }) + } + + fn check_run_to_step( + run: &serde_json::Value, + actors: &mut HashMap, + ) -> Result { + let id = run["id"].as_u64().unwrap_or(0); + let step_id = format!("step-ci-{}", id); + + let name = run["name"].as_str().unwrap_or("unknown"); + let app_slug = run["app"]["slug"].as_str().unwrap_or("ci"); + let actor = format!("ci:{}", app_slug); + + actors + .entry(actor.clone()) + .or_insert_with(|| ActorDefinition { + name: Some(app_slug.to_string()), + ..Default::default() + }); + + let timestamp = run["completed_at"] + .as_str() + .or_else(|| run["started_at"].as_str()) + .unwrap_or("1970-01-01T00:00:00Z") + .to_string(); + + let conclusion = run["conclusion"].as_str().unwrap_or("unknown").to_string(); + + let mut extra = HashMap::new(); + extra.insert( + "conclusion".to_string(), + serde_json::Value::String(conclusion), + ); + + let artifact_uri = format!("ci://checks/{}", name); + let change = HashMap::from([( + artifact_uri, + ArtifactChange { + raw: None, + structural: Some(StructuralChange { + change_type: "ci.run".to_string(), + extra, + }), + }, + )]); + + Ok(Step { + step: StepIdentity { + id: step_id, + parents: vec![], + actor, + timestamp, + }, + change, + meta: None, + }) + } + + fn build_path_meta( + pr: &serde_json::Value, + actors: &HashMap, + ) -> Result { + let title = pr["title"].as_str().map(|s| s.to_string()); + let body = pr["body"].as_str().unwrap_or(""); + let intent = if body.is_empty() { + None + } else { + Some(body.to_string()) + }; + + // Parse issue refs + let issue_numbers = extract_issue_refs(body); + let refs: Vec = issue_numbers + .into_iter() + .map(|n| { + let owner = pr["base"]["repo"]["owner"]["login"] + .as_str() + .unwrap_or("unknown"); + let repo = pr["base"]["repo"]["name"].as_str().unwrap_or("unknown"); + Ref { + rel: "fixes".to_string(), + href: format!("https://github.com/{}/{}/issues/{}", owner, repo, n), + } + }) + .collect(); + + // Labels in extra + let mut extra: HashMap = HashMap::new(); + if let Some(labels) = pr["labels"].as_array() { + let label_names: Vec = labels + .iter() + .filter_map(|l| l["name"].as_str()) + .map(|s| serde_json::Value::String(s.to_string())) + .collect(); + if !label_names.is_empty() { + let mut github_meta = serde_json::Map::new(); + github_meta.insert("labels".to_string(), serde_json::Value::Array(label_names)); + extra.insert("github".to_string(), serde_json::Value::Object(github_meta)); + } + } + + Ok(PathMeta { + title, + intent, + refs, + actors: if actors.is_empty() { + None + } else { + Some(actors.clone()) + }, + extra, + ..Default::default() + }) + } + + // ==================================================================== + // Helpers + // ==================================================================== + + fn register_actor( + actors: &mut HashMap, + actor_key: &str, + login: &str, + _email: Option<&str>, + ) { + actors + .entry(actor_key.to_string()) + .or_insert_with(|| ActorDefinition { + name: Some(login.to_string()), + identities: vec![Identity { + system: "github".to_string(), + id: login.to_string(), + }], + ..Default::default() + }); + } + + fn str_field(val: &serde_json::Value, key: &str) -> String { + val[key].as_str().unwrap_or("").to_string() + } + + // ==================================================================== + // Tests + // ==================================================================== + + #[cfg(test)] + mod tests { + use super::*; + + fn sample_pr() -> serde_json::Value { + serde_json::json!({ + "number": 42, + "title": "Add feature X", + "body": "This PR adds feature X.\n\nFixes #10\nCloses #20", + "state": "open", + "user": { "login": "alice" }, + "head": { "ref": "feature-x" }, + "base": { + "ref": "main", + "repo": { + "owner": { "login": "acme" }, + "name": "widgets" + } + }, + "labels": [ + { "name": "enhancement" }, + { "name": "reviewed" } + ], + "created_at": "2026-01-15T10:00:00Z", + "updated_at": "2026-01-16T14:00:00Z" + }) + } + + fn sample_commit_detail( + sha: &str, + parent_sha: Option<&str>, + msg: &str, + ) -> serde_json::Value { + let parents: Vec = parent_sha + .into_iter() + .map(|s| serde_json::json!({ "sha": s })) + .collect(); + serde_json::json!({ + "sha": sha, + "commit": { + "message": msg, + "committer": { + "date": "2026-01-15T12:00:00Z" + } + }, + "author": { "login": "alice" }, + "parents": parents, + "files": [ + { + "filename": "src/main.rs", + "patch": "@@ -1,3 +1,4 @@\n fn main() {\n+ println!(\"hello\");\n }" + } + ] + }) + } + + fn sample_review_comment( + id: u64, + commit_sha: &str, + path: &str, + line: u64, + ) -> serde_json::Value { + serde_json::json!({ + "id": id, + "user": { "login": "bob" }, + "commit_id": commit_sha, + "path": path, + "line": line, + "body": "Consider using a constant here.", + "created_at": "2026-01-15T14:00:00Z", + "pull_request_review_id": 100, + "in_reply_to_id": null + }) + } + + fn sample_pr_comment(id: u64) -> serde_json::Value { + serde_json::json!({ + "id": id, + "user": { "login": "carol" }, + "body": "Looks good overall!", + "created_at": "2026-01-15T16:00:00Z" + }) + } + + fn sample_review(id: u64, state: &str) -> serde_json::Value { + serde_json::json!({ + "id": id, + "user": { "login": "dave" }, + "state": state, + "body": "Approved with minor comments.", + "submitted_at": "2026-01-15T17:00:00Z" + }) + } + + fn sample_check_run(id: u64, name: &str, conclusion: &str) -> serde_json::Value { + serde_json::json!({ + "id": id, + "name": name, + "app": { "slug": "github-actions" }, + "conclusion": conclusion, + "completed_at": "2026-01-15T13:00:00Z", + "started_at": "2026-01-15T12:30:00Z" + }) + } + + #[test] + fn test_commit_to_step() { + let detail = sample_commit_detail("abc12345deadbeef", None, "Initial commit"); + let mut actors = HashMap::new(); + + let step = commit_to_step(&detail, &mut actors).unwrap(); + + assert_eq!(step.step.id, "step-abc12345"); + assert_eq!(step.step.actor, "human:alice"); + assert!(step.step.parents.is_empty()); + assert!(step.change.contains_key("src/main.rs")); + assert_eq!( + step.meta.as_ref().unwrap().intent.as_deref(), + Some("Initial commit") + ); + assert!(actors.contains_key("human:alice")); + } + + #[test] + fn test_review_comment_to_step() { + let rc = sample_review_comment(200, "abc12345deadbeef", "src/main.rs", 42); + let mut actors = HashMap::new(); + + let step = review_comment_to_step(&rc, &mut actors).unwrap(); + + assert_eq!(step.step.id, "step-rc-200"); + assert_eq!(step.step.actor, "human:bob"); + // Parents are empty — set later by the trunk chain pass + assert!(step.step.parents.is_empty()); + assert!(step.change.contains_key("review://src/main.rs#L42")); + assert!(actors.contains_key("human:bob")); + } + + #[test] + fn test_pr_comment_to_step() { + let pc = sample_pr_comment(300); + let mut actors = HashMap::new(); + + let step = pr_comment_to_step(&pc, &mut actors).unwrap(); + + assert_eq!(step.step.id, "step-ic-300"); + assert_eq!(step.step.actor, "human:carol"); + assert!(step.step.parents.is_empty()); + assert!(step.change.contains_key("review://conversation")); + let change = &step.change["review://conversation"]; + assert_eq!(change.raw.as_deref(), Some("Looks good overall!")); + } + + #[test] + fn test_review_to_step() { + let review = sample_review(400, "APPROVED"); + let mut actors = HashMap::new(); + + let step = review_to_step(&review, &mut actors).unwrap(); + + assert_eq!(step.step.id, "step-rv-400"); + assert_eq!(step.step.actor, "human:dave"); + assert!(step.step.parents.is_empty()); + assert!(step.change.contains_key("review://decision")); + let change = &step.change["review://decision"]; + assert!(change.structural.is_some()); + let structural = change.structural.as_ref().unwrap(); + assert_eq!(structural.change_type, "review.decision"); + assert_eq!(structural.extra["state"], "APPROVED"); + } + + #[test] + fn test_check_run_to_step() { + let run = sample_check_run(500, "build", "success"); + let mut actors = HashMap::new(); + + let step = check_run_to_step(&run, &mut actors).unwrap(); + + assert_eq!(step.step.id, "step-ci-500"); + assert_eq!(step.step.actor, "ci:github-actions"); + assert!(step.step.parents.is_empty()); + assert!(step.change.contains_key("ci://checks/build")); + let change = &step.change["ci://checks/build"]; + let structural = change.structural.as_ref().unwrap(); + assert_eq!(structural.change_type, "ci.run"); + assert_eq!(structural.extra["conclusion"], "success"); + } + + #[test] + fn test_build_path_meta() { + let pr = sample_pr(); + let mut actors = HashMap::new(); + register_actor(&mut actors, "human:alice", "alice", None); + + let meta = build_path_meta(&pr, &actors).unwrap(); + + assert_eq!(meta.title.as_deref(), Some("Add feature X")); + assert!(meta.intent.as_deref().unwrap().contains("feature X")); + assert_eq!(meta.refs.len(), 2); + assert_eq!(meta.refs[0].rel, "fixes"); + assert!(meta.refs[0].href.contains("/issues/10")); + assert!(meta.refs[1].href.contains("/issues/20")); + assert!(meta.actors.is_some()); + + // Labels in extra + let github = meta.extra.get("github").unwrap(); + let labels = github["labels"].as_array().unwrap(); + assert_eq!(labels.len(), 2); + } + + #[test] + fn test_derive_from_data_full() { + let pr = sample_pr(); + let commit1 = sample_commit_detail("abc12345deadbeef", None, "Initial commit"); + let commit2 = + sample_commit_detail("def67890cafebabe", Some("abc12345deadbeef"), "Add tests"); + // Fix second commit timestamp to be after first + let mut commit2 = commit2; + commit2["commit"]["committer"]["date"] = serde_json::json!("2026-01-15T13:00:00Z"); + + let review_comments = vec![sample_review_comment( + 200, + "abc12345deadbeef", + "src/main.rs", + 42, + )]; + let pr_comments = vec![sample_pr_comment(300)]; + let reviews = vec![sample_review(400, "APPROVED")]; + + let mut check_runs = HashMap::new(); + check_runs.insert( + "abc12345deadbeef".to_string(), + vec![sample_check_run(500, "build", "success")], + ); + + let config = DeriveConfig { + token: "test".to_string(), + api_url: "https://api.github.com".to_string(), + include_ci: true, + include_comments: true, + }; + + let data = PrData { + pr: &pr, + commit_details: &[commit1, commit2], + reviews: &reviews, + pr_comments: &pr_comments, + review_comments: &review_comments, + check_runs_by_sha: &check_runs, + }; + let path = derive_from_data(&data, "acme", "widgets", &config).unwrap(); + + assert_eq!(path.path.id, "pr-42"); + assert_eq!(path.path.base.as_ref().unwrap().uri, "github:acme/widgets"); + assert_eq!( + path.path.base.as_ref().unwrap().ref_str.as_deref(), + Some("main") + ); + + // Should have 2 commits + 1 review comment + 1 PR comment + 1 review + 1 CI = 6 steps + assert_eq!(path.steps.len(), 6); + + // All steps form a single trunk chain sorted by timestamp + assert!(path.steps[0].step.parents.is_empty()); + for i in 1..path.steps.len() { + assert!( + path.steps[i].step.timestamp >= path.steps[i - 1].step.timestamp, + "Steps not sorted: {} < {}", + path.steps[i].step.timestamp, + path.steps[i - 1].step.timestamp, + ); + assert_eq!( + path.steps[i].step.parents, + vec![path.steps[i - 1].step.id.clone()], + "Step {} should parent off step {}", + path.steps[i].step.id, + path.steps[i - 1].step.id, + ); + } + + // Path meta + let meta = path.meta.as_ref().unwrap(); + assert_eq!(meta.title.as_deref(), Some("Add feature X")); + assert_eq!(meta.refs.len(), 2); + } + + #[test] + fn test_derive_from_data_no_ci() { + let pr = sample_pr(); + let commit = sample_commit_detail("abc12345deadbeef", None, "Commit"); + + let config = DeriveConfig { + token: "test".to_string(), + api_url: "https://api.github.com".to_string(), + include_ci: false, + include_comments: false, + }; + + let data = PrData { + pr: &pr, + commit_details: &[commit], + reviews: &[], + pr_comments: &[], + review_comments: &[], + check_runs_by_sha: &HashMap::new(), + }; + let path = derive_from_data(&data, "acme", "widgets", &config).unwrap(); + + // Only commit steps + assert_eq!(path.steps.len(), 1); + assert_eq!(path.steps[0].step.id, "step-abc12345"); + } + + #[test] + fn test_derive_from_data_pending_review_skipped() { + let pr = sample_pr(); + let commit = sample_commit_detail("abc12345deadbeef", None, "Commit"); + let pending_review = sample_review(999, "PENDING"); + + let config = DeriveConfig { + token: "test".to_string(), + api_url: "https://api.github.com".to_string(), + include_ci: false, + include_comments: true, + }; + + let data = PrData { + pr: &pr, + commit_details: &[commit], + reviews: &[pending_review], + pr_comments: &[], + review_comments: &[], + check_runs_by_sha: &HashMap::new(), + }; + let path = derive_from_data(&data, "acme", "widgets", &config).unwrap(); + + // Only commit step, pending review skipped + assert_eq!(path.steps.len(), 1); + } + + #[test] + fn test_parse_next_link() { + let header = r#"; rel="next", ; rel="last""#; + assert_eq!( + parse_next_link(header), + Some("https://api.github.com/repos/foo/bar/pulls?page=2".to_string()) + ); + + assert_eq!( + parse_next_link(r#"; rel="prev""#), + None + ); + } + + #[test] + fn test_str_field() { + let val = serde_json::json!({"name": "hello", "missing": null}); + assert_eq!(str_field(&val, "name"), "hello"); + assert_eq!(str_field(&val, "missing"), ""); + assert_eq!(str_field(&val, "nonexistent"), ""); + } + + #[test] + fn test_register_actor_idempotent() { + let mut actors = HashMap::new(); + register_actor(&mut actors, "human:alice", "alice", None); + register_actor(&mut actors, "human:alice", "alice", None); + assert_eq!(actors.len(), 1); + } + + #[test] + fn test_ci_steps_chain_inline() { + let pr = sample_pr(); + let commit = sample_commit_detail("abc12345deadbeef", None, "Commit"); + + let mut check_runs = HashMap::new(); + check_runs.insert( + "abc12345deadbeef".to_string(), + vec![ + sample_check_run(501, "build", "success"), + sample_check_run(502, "test", "success"), + sample_check_run(503, "lint", "success"), + ], + ); + + let config = DeriveConfig { + token: "test".to_string(), + api_url: "https://api.github.com".to_string(), + include_ci: true, + include_comments: false, + }; + + let data = PrData { + pr: &pr, + commit_details: &[commit], + reviews: &[], + pr_comments: &[], + review_comments: &[], + check_runs_by_sha: &check_runs, + }; + let path = derive_from_data(&data, "acme", "widgets", &config).unwrap(); + + // 1 commit + 3 CI steps = 4 steps on a single trunk + assert_eq!(path.steps.len(), 4); + + // All steps chain linearly by timestamp + assert!(path.steps[0].step.parents.is_empty()); // first step: no parent + for i in 1..path.steps.len() { + assert_eq!( + path.steps[i].step.parents, + vec![path.steps[i - 1].step.id.clone()] + ); + } + } + + #[test] + fn test_review_comment_artifact_uri_format() { + let rc = sample_review_comment(700, "abc12345", "src/lib.rs", 100); + let mut actors = HashMap::new(); + + let step = review_comment_to_step(&rc, &mut actors).unwrap(); + + assert!(step.change.contains_key("review://src/lib.rs#L100")); + } + + #[test] + fn test_derive_from_data_empty_commits() { + let pr = sample_pr(); + let config = DeriveConfig { + token: "test".to_string(), + api_url: "https://api.github.com".to_string(), + include_ci: false, + include_comments: false, + }; + + let data = PrData { + pr: &pr, + commit_details: &[], + reviews: &[], + pr_comments: &[], + review_comments: &[], + check_runs_by_sha: &HashMap::new(), + }; + let path = derive_from_data(&data, "acme", "widgets", &config).unwrap(); + + assert_eq!(path.path.id, "pr-42"); + assert!(path.steps.is_empty()); + assert_eq!(path.path.head, "pr-42"); + } + + #[test] + fn test_review_empty_body() { + let mut review = sample_review(800, "APPROVED"); + review["body"] = serde_json::json!(""); + let mut actors = HashMap::new(); + + let step = review_to_step(&review, &mut actors).unwrap(); + let change = &step.change["review://decision"]; + assert!(change.raw.is_none()); + assert!(change.structural.is_some()); + } + + #[test] + fn test_commit_no_files() { + let detail = serde_json::json!({ + "sha": "aabbccdd11223344", + "commit": { + "message": "Empty commit", + "committer": { "date": "2026-01-15T12:00:00Z" } + }, + "author": { "login": "alice" }, + "parents": [], + "files": [] + }); + let mut actors = HashMap::new(); + + let step = commit_to_step(&detail, &mut actors).unwrap(); + assert!(step.change.is_empty()); + } + + #[test] + fn test_multiple_commits_chain() { + let pr = sample_pr(); + let c1 = { + let mut c = sample_commit_detail("1111111100000000", None, "First"); + c["commit"]["committer"]["date"] = serde_json::json!("2026-01-15T10:00:00Z"); + c + }; + let c2 = { + let mut c = + sample_commit_detail("2222222200000000", Some("1111111100000000"), "Second"); + c["commit"]["committer"]["date"] = serde_json::json!("2026-01-15T11:00:00Z"); + c + }; + let c3 = { + let mut c = + sample_commit_detail("3333333300000000", Some("2222222200000000"), "Third"); + c["commit"]["committer"]["date"] = serde_json::json!("2026-01-15T12:00:00Z"); + c + }; + + let config = DeriveConfig { + token: "test".to_string(), + api_url: "https://api.github.com".to_string(), + include_ci: false, + include_comments: false, + }; + + let data = PrData { + pr: &pr, + commit_details: &[c1, c2, c3], + reviews: &[], + pr_comments: &[], + review_comments: &[], + check_runs_by_sha: &HashMap::new(), + }; + let path = derive_from_data(&data, "acme", "widgets", &config).unwrap(); + + // Trunk chain: each step parents off the previous by timestamp + assert_eq!(path.steps.len(), 3); + assert!(path.steps[0].step.parents.is_empty()); + assert_eq!(path.steps[1].step.parents, vec!["step-11111111"]); + assert_eq!(path.steps[2].step.parents, vec!["step-22222222"]); + assert_eq!(path.path.head, "step-33333333"); + } + } +} + +// Re-export native-only functions at crate root for API compatibility +#[cfg(not(target_os = "emscripten"))] +pub use native::{derive_pull_request, list_pull_requests, resolve_token}; + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_extract_issue_refs_basic() { + let refs = extract_issue_refs("Fixes #42"); + assert_eq!(refs, vec![42]); + } + + #[test] + fn test_extract_issue_refs_multiple() { + let refs = extract_issue_refs("Fixes #10 and Closes #20"); + assert_eq!(refs, vec![10, 20]); + } + + #[test] + fn test_extract_issue_refs_case_insensitive() { + let refs = extract_issue_refs("FIXES #1, closes #2, Resolves #3"); + assert_eq!(refs, vec![1, 2, 3]); + } + + #[test] + fn test_extract_issue_refs_no_refs() { + let refs = extract_issue_refs("Just a regular PR description."); + assert!(refs.is_empty()); + } + + #[test] + fn test_extract_issue_refs_dedup() { + let refs = extract_issue_refs("Fixes #5 and also fixes #5"); + assert_eq!(refs, vec![5]); + } + + #[test] + fn test_extract_issue_refs_multiline() { + let body = "This is a PR.\n\nFixes #100\nCloses #200\n\nSome more text."; + let refs = extract_issue_refs(body); + assert_eq!(refs, vec![100, 200]); + } + + #[test] + fn test_derive_config_default() { + let config = DeriveConfig::default(); + assert_eq!(config.api_url, "https://api.github.com"); + assert!(config.include_ci); + assert!(config.include_comments); + assert!(config.token.is_empty()); + } + + #[test] + fn test_parse_pr_url_https() { + let pr = parse_pr_url("https://github.com/empathic/toolpath/pull/6").unwrap(); + assert_eq!(pr.owner, "empathic"); + assert_eq!(pr.repo, "toolpath"); + assert_eq!(pr.number, 6); + } + + #[test] + fn test_parse_pr_url_no_protocol() { + let pr = parse_pr_url("github.com/empathic/toolpath/pull/42").unwrap(); + assert_eq!(pr.owner, "empathic"); + assert_eq!(pr.repo, "toolpath"); + assert_eq!(pr.number, 42); + } + + #[test] + fn test_parse_pr_url_http() { + let pr = parse_pr_url("http://github.com/org/repo/pull/1").unwrap(); + assert_eq!(pr.owner, "org"); + assert_eq!(pr.repo, "repo"); + assert_eq!(pr.number, 1); + } + + #[test] + fn test_parse_pr_url_with_trailing_parts() { + let pr = parse_pr_url("https://github.com/org/repo/pull/99/files").unwrap(); + assert_eq!(pr.number, 99); + } + + #[test] + fn test_parse_pr_url_with_query_string() { + let pr = parse_pr_url("https://github.com/org/repo/pull/5?diff=unified").unwrap(); + assert_eq!(pr.number, 5); + } + + #[test] + fn test_parse_pr_url_invalid() { + assert!(parse_pr_url("not a url").is_none()); + assert!(parse_pr_url("https://github.com/org/repo").is_none()); + assert!(parse_pr_url("https://github.com/org/repo/issues/1").is_none()); + assert!(parse_pr_url("https://gitlab.com/org/repo/pull/1").is_none()); + } +} diff --git a/scripts/release.sh b/scripts/release.sh index 5ffb65d..95fd150 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -12,11 +12,12 @@ set -euo pipefail # 1. toolpath (no workspace deps) # toolpath-convo (no workspace deps) # 2. toolpath-git (depends on toolpath) +# toolpath-github (depends on toolpath) # toolpath-dot (depends on toolpath) # toolpath-claude (depends on toolpath, toolpath-convo) # 3. toolpath-cli (depends on all of the above) -ALL_CRATES=(toolpath toolpath-convo toolpath-git toolpath-dot toolpath-claude toolpath-cli) +ALL_CRATES=(toolpath toolpath-convo toolpath-git toolpath-github toolpath-dot toolpath-claude toolpath-cli) DRY_RUN="" AUTO_YES="" @@ -197,12 +198,12 @@ for crate in toolpath toolpath-convo; do done # Tier 2: satellite crates (depend on tier 1, no cross-deps) -for crate in toolpath-git toolpath-dot toolpath-claude; do +for crate in toolpath-git toolpath-github toolpath-dot toolpath-claude; do publish "$crate" done # Wait for tier 2 publishes to land before publishing the CLI -for crate in toolpath-git toolpath-dot toolpath-claude; do +for crate in toolpath-git toolpath-github toolpath-dot toolpath-claude; do if should_publish "$crate"; then wait_for_index "$crate" "$(crate_version "$crate")" fi diff --git a/site/_data/crates.json b/site/_data/crates.json index 89eda9b..170488d 100644 --- a/site/_data/crates.json +++ b/site/_data/crates.json @@ -23,6 +23,14 @@ "crate": "https://crates.io/crates/toolpath-git", "role": "Reads git history via libgit2 and maps commits to Steps, branches to Paths. Single branch produces a Path; multiple branches produce a Graph." }, + { + "name": "toolpath-github", + "version": "0.1.0", + "description": "Derive from GitHub pull requests", + "docs": "https://docs.rs/toolpath-github", + "crate": "https://crates.io/crates/toolpath-github", + "role": "Reads GitHub PRs via the REST API and maps commits, reviews, comments, and CI checks to Steps. Everything is a Step in the DAG — code changes, review threads, approvals, and CI results." + }, { "name": "toolpath-claude", "version": "0.6.2", diff --git a/site/pages/crates.md b/site/pages/crates.md index ece4c0c..adf957e 100644 --- a/site/pages/crates.md +++ b/site/pages/crates.md @@ -14,6 +14,7 @@ toolpath-cli (binary: path) +-- toolpath (core types) +-- toolpath-convo (conversation abstraction) +-- toolpath-git -> toolpath + +-- toolpath-github -> toolpath +-- toolpath-claude -> toolpath, toolpath-convo +-- toolpath-dot -> toolpath ``` From bc6ebe3a016cfedbc974adef249ae7b73adc10d0 Mon Sep 17 00:00:00 2001 From: Alex Kesling Date: Thu, 5 Mar 2026 23:40:30 -0500 Subject: [PATCH 2/2] fix: add toolpath-github and toolpath-convo to home page workspace table --- site/index.md | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/site/index.md b/site/index.md index 43db555..069817d 100644 --- a/site/index.md +++ b/site/index.md @@ -188,12 +188,14 @@ path query filter --input doc.json --actor "agent:" Toolpath is a Rust workspace of focused crates: -| Crate | What it does | -| ---------------------------------------------------- | ------------------------------------------ | -| [`toolpath`](https://docs.rs/toolpath) | Core types, builders, query API | -| [`toolpath-git`](https://docs.rs/toolpath-git) | Derive from git history | -| [`toolpath-claude`](https://docs.rs/toolpath-claude) | Derive from Claude conversations | -| [`toolpath-dot`](https://docs.rs/toolpath-dot) | Graphviz DOT visualization | -| [`toolpath-cli`](https://docs.rs/toolpath-cli) | Unified CLI (`cargo install toolpath-cli`) | +| Crate | What it does | +| ------------------------------------------------------ | ------------------------------------------ | +| [`toolpath`](https://docs.rs/toolpath) | Core types, builders, query API | +| [`toolpath-convo`](https://docs.rs/toolpath-convo) | Provider-agnostic conversation traits | +| [`toolpath-git`](https://docs.rs/toolpath-git) | Derive from git history | +| [`toolpath-github`](https://docs.rs/toolpath-github) | Derive from GitHub pull requests | +| [`toolpath-claude`](https://docs.rs/toolpath-claude) | Derive from Claude conversations | +| [`toolpath-dot`](https://docs.rs/toolpath-dot) | Graphviz DOT visualization | +| [`toolpath-cli`](https://docs.rs/toolpath-cli) | Unified CLI (`cargo install toolpath-cli`) | See [Crates](/crates/) for details, or [docs.rs](https://docs.rs/toolpath) for API reference.