From 26654f198198a52a09404df84017a03eaaa0e3df Mon Sep 17 00:00:00 2001 From: 0xProgress Date: Wed, 16 Sep 2026 22:52:49 +0100 Subject: [PATCH] phrase 1 kick off --- .github/ISSUE_TEMPLATE/bug_report.yml | 4 +- .github/dependabot.yml | 8 +- .github/pull_request_template.md | 4 +- .github/workflows/ci.yml | 12 +- .github/workflows/codeql.yml | 7 +- .github/workflows/release.yml | 16 +- .golangci.yml | 43 +++ .goreleaser.yaml | 50 +++ CHANGELOG.md | 111 ++++++ CONTRIBUTING.md | 60 +++- Dockerfile | 35 ++ Makefile | 38 +++ README.md | 41 ++- cmd/mock.go | 114 +++++++ cmd/root.go | 113 ++++++ config/config.go | 242 +++++++++++++ docs/architecture.md | 98 ++++-- docs/providers/TEMPLATE.md | 407 ++++++++++++++++++---- docs/providers/checklist.md | 131 +++++++ docs/webhookd-core.md | 261 +++++++++++--- go.mod | 10 + go.sum | 10 + main.go | 31 ++ output/event.go | 61 ++++ output/pretty.go | 82 +++++ output/writer.go | 54 +++ providers/mock/mock.go | 85 +++++ providers/mock/mock_test.go | 137 ++++++++ providers/provider.go | 71 ++++ providers/registry.go | 50 +++ providers/registry_test.go | 126 +++++++ server/handler.go | 199 +++++++++++ server/server.go | 198 +++++++++++ server/server_test.go | 474 ++++++++++++++++++++++++++ 34 files changed, 3193 insertions(+), 190 deletions(-) create mode 100644 .golangci.yml create mode 100644 .goreleaser.yaml create mode 100644 CHANGELOG.md create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 cmd/mock.go create mode 100644 cmd/root.go create mode 100644 config/config.go create mode 100644 docs/providers/checklist.md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 output/event.go create mode 100644 output/pretty.go create mode 100644 output/writer.go create mode 100644 providers/mock/mock.go create mode 100644 providers/mock/mock_test.go create mode 100644 providers/provider.go create mode 100644 providers/registry.go create mode 100644 providers/registry_test.go create mode 100644 server/handler.go create mode 100644 server/server.go create mode 100644 server/server_test.go diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 22ecd54..fd3b311 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -13,7 +13,7 @@ body: attributes: label: webhookd version description: Output of `webhookd --version` - placeholder: "v0.1.0" + placeholder: "0.1.0" validations: required: true @@ -76,4 +76,4 @@ body: description: Flags passed to webhookd. Redact secret values. placeholder: "webhookd github --port 8080 --path /hooks/github" validations: - required: false \ No newline at end of file + required: false diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a431fd6..163591e 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -8,8 +8,10 @@ updates: commit-message: prefix: "chore" ignore: - # webhookd uses standard library only in core. - # Any new dependency is a signal to review carefully. + # Major-version bumps are never automatic. The dependency surface + # is small — cobra in cmd/, plus its transitive deps pflag and + # mousetrap — and any new require entry is a decision that should + # be reviewed, not merged by a bot. - dependency-name: "*" update-types: ["version-update:semver-major"] @@ -25,4 +27,4 @@ updates: schedule: interval: monthly commit-message: - prefix: "chore" \ No newline at end of file + prefix: "chore" diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index bdc393a..b2752b4 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -37,7 +37,7 @@ Skip this section if this is not a provider PR. - [ ] Official webhook documentation link: - [ ] `docs/providers/.md` written from `docs/providers/TEMPLATE.md`, every section filled - [ ] All items on the self-review checklist in `CONTRIBUTING.md` are checked -- [ ] No new dependencies in `go.mod` +- [ ] Provider code imports only standard-library packages - [ ] Test vectors are real — taken from the provider's official docs, not fabricated - [ ] Both valid-signature and tampered-body test cases are present @@ -48,4 +48,4 @@ Skip this section if this is not a provider PR. - [ ] `make check` passes locally with no errors - [ ] PR title follows Conventional Commits (`feat:`, `fix:`, `docs:`, `test:`, `refactor:`, `chore:`, `ci:`) - [ ] Nothing is written to stdout except through the JSONL event writer -- [ ] No secrets, tokens, or real webhook payloads are committed \ No newline at end of file +- [ ] No secrets, tokens, or real webhook payloads are committed diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a0ae3d9..9c529b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,8 +23,10 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: '1.23' + go-version-file: "go.mod" cache: true + - name: Run go vet + run: go vet ./... - uses: golangci/golangci-lint-action@v6 with: version: latest @@ -37,7 +39,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: '1.23' + go-version-file: "go.mod" cache: true - name: Run tests with race detector run: go test -race -count=1 -coverprofile=coverage.out ./... @@ -63,13 +65,13 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-go@v5 with: - go-version: '1.23' + go-version-file: "go.mod" cache: true - name: Build env: GOOS: ${{ matrix.goos }} GOARCH: ${{ matrix.goarch }} - CGO_ENABLED: '0' + CGO_ENABLED: "0" run: go build -o /dev/null . commitlint: @@ -82,4 +84,4 @@ jobs: fetch-depth: 0 - uses: wagoid/commitlint-github-action@v6 with: - configFile: .commitlintrc.json \ No newline at end of file + configFile: .commitlintrc.json diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index c29e5e2..4e41c92 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -7,7 +7,7 @@ on: branches: [main] schedule: # Weekly on Monday at 06:00 UTC - - cron: '0 6 * * 1' + - cron: "0 6 * * 1" permissions: security-events: write @@ -24,7 +24,8 @@ jobs: - uses: actions/setup-go@v5 with: - go-version: '1.23' + # Single source of truth: go.mod's go directive. + go-version-file: "go.mod" cache: true - uses: github/codeql-action/init@v3 @@ -35,4 +36,4 @@ jobs: - uses: github/codeql-action/analyze@v3 with: - category: "/language:go" \ No newline at end of file + category: "/language:go" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6d2b00e..6154e19 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,7 +3,7 @@ name: release on: push: tags: - - 'v[0-9]+.[0-9]+.[0-9]+' + - "v[0-9]+.[0-9]+.[0-9]+" permissions: contents: write @@ -22,9 +22,19 @@ jobs: - uses: actions/setup-go@v5 with: - go-version: '1.23' + go-version-file: "go.mod" cache: true + # dockers_v2 builds multi-platform images with buildx. QEMU + # provides the arm64 emulation the amd64 runner lacks; buildx + # provides a builder capable of producing a multi-platform + # manifest in a single pass. + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - uses: docker/login-action@v3 with: registry: ghcr.io @@ -36,4 +46,4 @@ jobs: version: latest args: release --clean env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..6dbe7bd --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,43 @@ +# Configuration for golangci-lint. +# +# The default set of linters (errcheck, govet, ineffassign, staticcheck, +# unused) is enabled via `default: standard`. Four extras are added: +# +# bodyclose — catches HTTP response bodies that are not closed, which +# matters for the client-side code in server_test.go. +# gosec — security-focused checks. This tool verifies signatures; +# a dependency-free security linter is cheap insurance. +# misspell — catches typos in comments and string literals. +# gofmt / — formatters, not linters. Included so CI catches files +# goimports that were saved without running `make fmt` locally. +# +# The Makefile runs this via `golangci-lint run ./...`, as the `lint` +# target. `make check` chains fmt → vet → lint → test. + +version: "2" + +run: + # The default 1m timeout is tight for a cold CI run that includes + # race-instrumented tests. 5m gives headroom without being effectively + # unbounded. + timeout: 5m + tests: true + +linters: + default: standard + enable: + - bodyclose + - gosec + - misspell + +formatters: + enable: + - gofmt + - goimports + +issues: + # Do not cap the number of issues reported. The codebase is small; if + # a change produces fifty warnings, the reviewer wants to see all + # fifty, not a truncated list. + max-same-issues: 0 + max-issues-per-linter: 0 diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..736cbc8 --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,50 @@ +version: 2 + +builds: + - env: + - CGO_ENABLED=0 + goos: + - linux + - darwin + - windows + goarch: + - amd64 + - arm64 + ldflags: + - -s -w -X main.version={{.Version}} + +archives: + - formats: [tar.gz] + format_overrides: + - goos: windows + formats: [zip] + +# dockers_v2 replaced dockers + docker_manifests. It runs a single +# `docker buildx build` per image, producing a multi-arch manifest +# without intermediate per-arch tags. GoReleaser stages every artifact +# for the target platform under $TARGETPLATFORM/ in a temporary build +# context, which is what the Dockerfile copies from. +# +# Images are built and pushed in the publish phase, not the build +# phase. `goreleaser build` and `goreleaser release --skip=publish` +# do not produce images. +dockers_v2: + - images: + - ghcr.io/0xprogress/webhookd + tags: + - "{{.Version}}" + - latest + platforms: + - linux/amd64 + - linux/arm64 + +checksum: + name_template: checksums.txt + +changelog: + sort: asc + filters: + exclude: + - "^docs:" + - "^test:" + - "^chore:" \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..d58f78f --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,111 @@ +# Changelog + +All notable changes to webhookd are documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +--- + +## [Unreleased] + +### Planned + +- `github` provider — HMAC-SHA256 over the raw body, `X-Hub-Signature-256` +- `stripe` provider — HMAC-SHA256 over `{timestamp}.{body}`, 300s timestamp check +- `slack` provider — HMAC-SHA256 over `v0:{timestamp}:{body}`, 300s timestamp check +- `shopify` provider — base64-encoded HMAC-SHA256 over the raw body + +Nothing is committed to any of the above. Each provider is a separate PR +against a frozen core; see [CONTRIBUTING.md](CONTRIBUTING.md). + +--- + +## [0.1.0] + +The first release. This is the core: a working webhook receiver that verifies +signatures, streams verified events as JSONL on stdout, and does nothing else. + +### Added + +**Core pipeline** + +- HTTP listener with read and write timeouts of 10 seconds +- Raw request body captured once and never re-encoded — the same bytes are + passed to signature verification and emitted as `payload` +- Body size limit of 2 MB, enforced during the read via `http.MaxBytesReader` +- Requests rejected with the correct status for every failure mode: 401 for a + bad signature, 404 for an unknown provider, 405 for a non-POST method, 413 + for an oversized body, 415 for an unsupported content type, 500 for an + internal error +- Health endpoint at `GET /health` +- Default bind address of `127.0.0.1`; binding to `0.0.0.0` requires an + explicit `--host 0.0.0.0` + +**Provider interface and registry** + +- The `Provider` interface — five methods a provider implements to answer + five questions about a request +- Provider registry with self-registration via `init()` +- Registry panics on duplicate provider names, catching a programming error + at startup rather than at request time +- The interface is frozen at v0.1 + +**Output** + +- JSONL event stream on stdout, one object per line, identical shape for every + provider +- Output preserves key order, duplicate keys, and numeric precision from the + request body — the payload is emitted from the raw bytes, not re-encoded + from a decoded map +- Pretty mode (`--pretty`) for human-readable output during live demos +- All diagnostics — startup banner, errors, signature failures — on stderr; + nothing but data on stdout + +**CLI** + +- `webhookd ` to run a provider receiver +- `webhookd --list` to print registered providers +- `webhookd --version` to print the version +- Flags: `--secret-env`, `--port`, `--host`, `--path`, `--pretty`, + `--max-body`, `--timeout` +- Configuration resolves in the order flag → environment → default +- Secrets are passed by name (`--secret-env STRIPE_WEBHOOK_SECRET`) and read + from the environment; the value never appears on the command line + +**Mock provider** + +- A reference implementation of the `Provider` interface +- Accepts any request with the header `X-Mock-Signature: valid` +- Ships in every build; used by the server test suite +- Not intended for production + +**Documentation** + +- `README.md` with a five-minute demo that works end to end +- `CONTRIBUTING.md` covering the provider contribution process +- `docs/architecture.md` explaining the core/provider split +- `docs/webhookd-core.md` as the build specification +- `docs/providers/checklist.md` as a copy-pasteable self-review list +- `docs/providers/TEMPLATE.md` as the provider documentation template +- `SECURITY.md` with a vulnerability reporting process + +**Distribution** + +- Cross-compiled binaries for linux/amd64, linux/arm64, darwin/amd64, + darwin/arm64, windows/amd64, windows/arm64 +- Multi-architecture Docker image on `ghcr.io/0xprogress/webhookd` + +### Security + +- Signature comparison uses `hmac.Equal()` in every provider, avoiding timing + side channels +- Timestamps in providers that use them are rejected if older than 300 seconds, + protecting against replay +- Request bodies are capped at 2 MB, enforced before the body is fully read +- Secrets are read from environment variables, never from flags + +--- + +[Unreleased]: https://github.com/0xProgress/webhookd/compare/v0.1.0...HEAD +[0.1.0]: https://github.com/0xProgress/webhookd/releases/tag/v0.1.0 \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d86ba66..f051e82 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,7 +44,7 @@ For providers specifically: check that an issue exists and is open before starti ## Development Setup **Requirements:** -- Go 1.21 or later +- Go 1.27.0 or later - `golangci-lint` — [install instructions](https://golangci-lint.run/usage/install/) - `make` @@ -84,6 +84,8 @@ The mock provider accepts any request with the header `X-Mock-Signature: valid` This is the full process for adding a new webhook provider. Read it completely before starting. +Providers live in this repository under `providers//`. There is no external provider mechanism — a new provider is a PR against this repo, not a separate module that users import. + ### 1. Check the open issue Find the issue for your provider (e.g. `provider: shopify`). Comment that you're working on it so no one else starts the same work. @@ -102,7 +104,7 @@ type Provider interface { } ``` -Read `providers/provider.go` for the full documentation on each method. +Read `providers/provider.go` for the full documentation on each method. The full specification is in `docs/webhookd-core.md`. ### 3. Study the mock provider @@ -131,18 +133,23 @@ providers/ docs/providers/ └── .md + +cmd/ +└── .go ``` ### 6. Implement the provider ```go -package +package name import ( "crypto/hmac" "crypto/sha256" + "encoding/hex" "errors" "net/http" + "os" "github.com/0xProgress/webhookd/providers" ) @@ -154,20 +161,20 @@ func init() { type Provider struct{} func (p *Provider) Name() string { - return "" + return "name" } func (p *Provider) Verify(r *http.Request, rawBody []byte) error { // Read the secret from environment secret := []byte(os.Getenv("PROVIDER_WEBHOOK_SECRET")) if len(secret) == 0 { - return errors.New(": PROVIDER_WEBHOOK_SECRET is not set") + return errors.New("name: PROVIDER_WEBHOOK_SECRET is not set") } // Get signature from header sig := r.Header.Get("X-Provider-Signature") if sig == "" { - return errors.New(": missing signature header") + return errors.New("name: missing signature header") } // Compute expected signature @@ -177,7 +184,7 @@ func (p *Provider) Verify(r *http.Request, rawBody []byte) error { // MUST use hmac.Equal — never == or strings.Compare if !hmac.Equal([]byte(sig), []byte(expected)) { - return errors.New(": signature mismatch") + return errors.New("name: signature mismatch") } return nil @@ -197,12 +204,14 @@ func (p *Provider) EventID(r *http.Request, rawBody []byte) string { } ``` +Replace `name` with your provider's identifier (lowercase) and `PROVIDER` / `X-Provider-*` with the real names from the provider's docs. + **Rules that are not optional:** - Use `hmac.Equal()` for all MAC comparisons. Never `==`. Never `strings.Compare`. Never `bytes.Equal`. `hmac.Equal` is constant-time. The others are not. - If the provider sends a timestamp, reject requests older than 300 seconds. - Return error strings prefixed with the provider name: `"shopify: ..."`. -- No external dependencies. Standard library only. +- No external dependencies. A provider is standard-library-only. ### 7. Write tests @@ -221,6 +230,17 @@ Your test file must cover at minimum: Use real HMAC test vectors where the provider's docs include them. Do not invent example values. ```go +package name + +import ( + "bytes" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "net/http/httptest" + "testing" +) + func TestVerify_ValidSignature(t *testing.T) { // Use a known secret and body to compute a real signature, // then verify it. Do not hardcode a signature you made up. @@ -264,13 +284,21 @@ func TestVerify_TamperedBody(t *testing.T) { } ``` +Tests must use `testing` and `net/http/httptest` only. No assertion libraries. Assert on raw bytes where the output format is part of the contract, not on parsed structures. + ### 8. Write the documentation file Copy `docs/providers/TEMPLATE.md` and fill it in completely. Every section is required. The PR check will fail if sections are missing. ### 9. Add the subcommand -Add a file `cmd/.go` that wires the provider into the CLI. Follow the pattern of existing command files. +Add a file `cmd/.go` that wires the provider into the CLI: + +- Declares the provider subcommand +- Imports the provider package so its `init()` runs and it registers itself +- Sets any provider-specific defaults (for example, the conventional environment variable name for the signing secret, used when `--secret-env` is empty) + +Follow the pattern of existing command files. ### 10. Self-review with the checklist @@ -284,6 +312,7 @@ Interface [ ] Implements DeliveryID() [ ] Implements EventID() [ ] Calls providers.Register() in init() +[ ] Has a cmd/.go that imports the provider package Security [ ] hmac.Equal() used for all MAC comparisons @@ -312,8 +341,8 @@ Documentation Code quality [ ] make check passes clean -[ ] No new dependencies in go.mod -[ ] No external packages imported +[ ] No dependencies added to go.mod +[ ] No external packages imported by the provider ``` --- @@ -331,7 +360,7 @@ refactor: simplify registry lookup chore: update go version in workflows ``` -The PR title must follow this format. The CI check will fail if it doesn't. +The PR title must follow this format. The CI check will fail if it doesn't. The rules are enforced by `.commitlintrc.json` at the repository root — read it if you are unsure whether your commit will pass. **Types:** - `feat` — new provider or feature @@ -368,11 +397,11 @@ The PR title must follow this format. The CI check will fail if it doesn't. **Formatting:** `gofmt`. Run `make fmt` before committing. -**Imports:** Standard library only in provider implementations. No exceptions. +**Imports:** Standard library only in provider implementations. No exceptions. The core's only external dependency is `github.com/spf13/cobra`, used in `cmd/` for CLI wiring. -**Errors:** Return errors, don't panic. Prefix with the package or provider name. +**Errors:** Return errors, don't panic. Prefix with the package or provider name. The one exception is duplicate provider registration in `providers/registry.go`, which panics by design. -**Comments:** Public types and functions have doc comments. Private implementation details don't need them unless non-obvious. +**Comments:** Public types and functions have doc comments. Interface methods get doc comments that describe the contract, not the implementation. Comments explain *why*, not *what*. **Tests:** Use `testing` and `net/http/httptest`. No test framework dependencies. @@ -393,3 +422,4 @@ Include: - Your assessment of impact You'll receive a response within 48 hours. Security issues are treated as the highest priority. +\ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7349f5e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,35 @@ +# Dockerfile for webhookd. +# +# This file is consumed by GoReleaser's dockers_v2 pipeline, not by a +# plain `docker build` from a source checkout. GoReleaser stages the +# already-built binaries under $TARGETPLATFORM/ in a temporary build +# context, so this file copies them rather than compiling anything. +# +# A plain `docker build -t webhookd .` from a source checkout will fail +# with "COPY failed: no source files were specified" — the staged +# binary does not exist until GoReleaser has run. To produce the image +# locally, use: +# +# goreleaser release --snapshot --clean +# +# which runs the same pipeline without publishing. + +FROM scratch + +# TARGETPLATFORM is provided by buildx when the image is built with +# --platform. GoReleaser's dockers_v2 block passes linux/amd64 and +# linux/arm64, so this resolves to one of those two strings and matches +# the per-platform directory GoReleaser created in the build context. +ARG TARGETPLATFORM + +COPY $TARGETPLATFORM/webhookd /webhookd + +# Port 8080 is the default the binary binds. Note that the listener's +# default host is 127.0.0.1, so a container launched without --host +# 0.0.0.0 will not accept connections from outside the container. The +# README's docker run examples pass the subcommand but not --host; that +# is a known gap between the current README and the safe default, and +# it will be reconciled when the README is next revised. +EXPOSE 8080 + +ENTRYPOINT ["/webhookd"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..fad451d --- /dev/null +++ b/Makefile @@ -0,0 +1,38 @@ +.PHONY: build test lint fmt vet clean release + +# VERSION strips the leading "v" from a git tag so that a tag of +# "v0.1.0" produces the string "0.1.0". This matches what GoReleaser's +# {{.Version}} produces at release time, so `webhookd --version` reports +# the same string regardless of whether the binary was built locally +# with `make build` or downloaded from a release. When HEAD is not on a +# tag, `git describe --tags --always` returns a bare commit hash (no +# leading "v"), and sed is a no-op. +VERSION := $(shell git describe --tags --always | sed 's/^v//') + +build: + go build -ldflags="-s -w -X main.version=$(VERSION)" -o bin/webhookd . + +test: + go test -race -count=1 ./... + +test-cover: + go test -race -coverprofile=coverage.out ./... + go tool cover -html=coverage.out -o coverage.html + +lint: + golangci-lint run ./... + +fmt: + gofmt -w . + goimports -w . + +vet: + go vet ./... + +check: fmt vet lint test + +clean: + rm -rf bin/ coverage.out coverage.html + +release: + goreleaser release --clean diff --git a/README.md b/README.md index 7cdf4fc..2473aae 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ brew install 0xProgress/tap/webhookd go install github.com/0xProgress/webhookd@latest ``` -Requires Go 1.21 or later. +Requires Go 1.27.0 or later. ### Docker @@ -118,6 +118,10 @@ That's the whole tool. Everything below is providers and flags. ## Providers +All providers ship in this repository, under `providers//`. There is no +external provider mechanism — a new provider is a PR, not a separate module. +The set of providers a given binary supports is fixed at build time. + | Provider | Status | Signature | Timestamp check | Notes | |---|---|---|---|---| | `mock` | ✅ built-in | `X-Mock-Signature: valid` | No | Reference implementation. Not for production. | @@ -165,7 +169,8 @@ export STRIPE_WEBHOOK_SECRET=whsec_... webhookd stripe --secret-env STRIPE_WEBHOOK_SECRET ``` -For providers with a conventional variable name, `--secret-env` defaults to it: +For providers with a conventional variable name, the subcommand supplies the +default, so the flag can be omitted: ```bash export GITHUB_WEBHOOK_SECRET=... @@ -200,7 +205,7 @@ webhookd github --pretty ``` ``` -GitHub +github ────────────────────────────────────── ✓ Signature verified @@ -214,6 +219,9 @@ Received: 2026-09-15T19:44:03Z } ``` +The header line is the provider's lowercase name. The core does not know how to +title-case `github` into `GitHub`, and it does not pretend to. + **Custom path behind a reverse proxy:** ```bash @@ -247,7 +255,15 @@ The shape is identical across every provider: | `id` | string | Event ID if the provider supplies one, else `""` | | `delivery_id` | string | Delivery or request ID if supplied, else `""` | | `received_at` | string | ISO 8601 UTC timestamp of receipt | -| `payload` | object | Full parsed JSON body, unmodified | +| `payload` | object | Full JSON body, unmodified in structure and values | + +All seven fields are always present. Fields without a value are emitted as `""`, +never omitted. + +`payload` preserves key order, duplicate keys, and numeric precision exactly as +the provider sent them. The only change is whitespace normalisation: the value +is emitted compact on one line with insignificant whitespace removed and any +embedded newlines escaped, which is what makes the output line-oriented. ### stdout vs stderr @@ -280,7 +296,7 @@ webhookd: github: signature mismatch — 203.0.113.4 | Wrong method | `405 Method Not Allowed` | `{"error": "method not allowed"}` | | Body too large | `413 Payload Too Large` | `{"error": "request body too large"}` | | Bad content type | `415 Unsupported Media Type` | `{"error": "unsupported content type"}` | -| Internal error | `500 Internal Server Error` | `{"error": "internal error"}` | +| Malformed JSON body | `500 Internal Server Error` | `{"error": "internal error"}` | ### Health endpoint @@ -306,7 +322,8 @@ when exposed. - **Signature verification is constant-time.** Every provider uses `hmac.Equal()`. - **Timestamps are enforced.** Providers that send a signed timestamp reject requests older than 300 seconds. -- **Body size is capped.** Default 2MB, enforced before the body is read. +- **Body size is capped.** Default 2MB, enforced during the read via + `http.MaxBytesReader` — an oversized body is never fully buffered. - **Read and write timeouts are 10 seconds.** Connections cannot hang the process. - **Secrets never appear on the command line.** Only the *name* of the environment variable is passed via `--secret-env`. @@ -320,7 +337,7 @@ To report a vulnerability, see [SECURITY.md](SECURITY.md). Do not open a public ## Build from source -**Requirements:** Go 1.21+, `make`, optionally `golangci-lint`. +**Requirements:** Go 1.27.0+, `make`, optionally `golangci-lint`. ```bash git clone https://github.com/0xProgress/webhookd @@ -352,8 +369,8 @@ well-documented process: 1. Find or open the provider issue. 2. Read the [provider guide](docs/contributing/provider-guide.md). -3. Implement against the `Provider` interface. -4. Write tests, write the doc file, run `make check`. +3. Implement against the `Provider` interface under `providers//`. +4. Add `cmd/.go`, write tests, write the doc file, run `make check`. 5. Open a PR. Read [CONTRIBUTING.md](CONTRIBUTING.md) before you start. It covers the provider @@ -372,7 +389,9 @@ It captures the raw body, hands it to a registered provider, and writes the resu Three decisions shape everything else: 1. **Raw body first.** The bytes are captured before anything else touches the - request and are never re-encoded. Verification and parsing see the same bytes. + request and are never re-encoded. Verification and parsing see the same bytes, + and `payload` preserves them — key order, duplicate keys, and numeric precision + all survive to the consumer. 2. **stdout is data, stderr is everything else.** This is what makes `| jq` work without filters or `2>/dev/null`. 3. **Providers are leaf nodes.** A provider verifies a signature and extracts four @@ -384,4 +403,4 @@ Read [docs/architecture.md](docs/architecture.md) for the full picture. ## License -[MIT](LICENSE) © 0xProgress \ No newline at end of file +[MIT](LICENSE) © 0xProgress diff --git a/cmd/mock.go b/cmd/mock.go new file mode 100644 index 0000000..eec5e1a --- /dev/null +++ b/cmd/mock.go @@ -0,0 +1,114 @@ +package cmd + +import ( + "context" + "errors" + "fmt" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/spf13/cobra" + + "github.com/0xProgress/webhookd/config" + "github.com/0xProgress/webhookd/output" + "github.com/0xProgress/webhookd/providers" + _ "github.com/0xProgress/webhookd/providers/mock" + "github.com/0xProgress/webhookd/server" +) + +// shutdownGrace is the maximum time the server waits for in-flight +// requests to complete after a SIGINT or SIGTERM. Beyond it, Shutdown +// returns with an error and the process exits anyway; the HTTP +// listener is already closed by then, so nothing new arrives. +const shutdownGrace = 5 * time.Second + +var mockCmd = &cobra.Command{ + Use: "mock", + Short: "Run the mock provider (reference implementation, not for production)", + Long: `Run a webhook receiver that accepts any request carrying the header +X-Mock-Signature: valid and rejects everything else. + +The mock provider has no secret and does not compute a MAC. It exists so +contributors can see the full pipeline end to end and so the server test +suite has a provider to exercise.`, + Args: cobra.NoArgs, + RunE: runMock, +} + +func init() { + rootCmd.AddCommand(mockCmd) +} + +func runMock(cmd *cobra.Command, _ []string) error { + cfg, err := config.Load(cmd, "mock") + if err != nil { + return err + } + + // Fail fast if the blank import above did not actually register the + // provider. Without this check the binary would start, accept + // requests, and return 404 "unknown provider" for every one — a + // confusing failure mode for a mistake whose cause is a missing + // import in this file. + if _, ok := providers.Get(cfg.ProviderName); !ok { + return fmt.Errorf("%s: provider not registered", cfg.ProviderName) + } + + var out server.EventWriter + if cfg.Pretty { + out = output.NewPrettyWriter(cmd.OutOrStdout()) + } else { + out = output.NewWriter(cmd.OutOrStdout()) + } + + handler := server.NewHandler( + cfg.ProviderName, + out, + cmd.ErrOrStderr(), + cfg.MaxBody, + ) + + srv := server.New(server.Options{ + Host: cfg.Host, + Port: cfg.Port, + Path: cfg.Path, + Version: buildVersion, + Timeout: cfg.Timeout, + Handler: handler, + ErrOut: cmd.ErrOrStderr(), + }) + + // Catch SIGINT (Ctrl-C) and SIGTERM. On either signal the context + // is canceled and the select below takes the shutdown branch. + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + // ListenAndServe runs in a goroutine so the main flow can select on + // both the signal context and the serve error. The channel is + // buffered so the goroutine can always send, even if the main flow + // has already moved on through the shutdown branch. + errCh := make(chan error, 1) + go func() { + errCh <- srv.ListenAndServe() + }() + + select { + case err := <-errCh: + // ListenAndServe returned on its own. ErrServerClosed means + // Shutdown was called and is a clean exit, not an error. + if errors.Is(err, http.ErrServerClosed) { + return nil + } + return err + + case <-ctx.Done(): + // A signal arrived. Stop accepting, let in-flight requests + // finish, and return Shutdown's result. + shutdownCtx, cancel := context.WithTimeout(context.Background(), shutdownGrace) + defer cancel() + return srv.Shutdown(shutdownCtx) + } +} diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..c7d661b --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,113 @@ +// Package cmd implements the webhookd command-line interface. +// +// The root command carries the shared flags every provider subcommand +// inherits. Each provider contributes one file in this package +// (cmd/.go) that declares its subcommand and adds it to the root +// during init(). This package is the only place in the codebase that +// knows which providers exist; the core packages — server, output, +// providers — never do. +package cmd + +import ( + "fmt" + "os" + + "github.com/spf13/cobra" + + "github.com/0xProgress/webhookd/providers" +) + +// buildVersion is set by Execute from the value the linker injected +// into main.version. It defaults to "dev" so a plain `go build` still +// produces a runnable binary with a sensible version string. +var buildVersion = "dev" + +// rootCmd is the top-level command. It carries the shared persistent +// flags every provider subcommand inherits, and handles the two +// root-only operations: --list and --version. +var rootCmd = &cobra.Command{ + Use: "webhookd", + Short: "The Unix pipe for webhooks", + Long: `webhookd listens for webhook HTTP requests, verifies their signatures, +and writes one JSONL line per verified event to stdout. + +Pipe the output anywhere: webhookd github | jq, webhookd stripe | grep, +webhookd shopify | tee events.jsonl.`, + Args: cobra.NoArgs, + SilenceUsage: true, + SilenceErrors: true, + RunE: func(cmd *cobra.Command, args []string) error { + if ver, _ := cmd.Flags().GetBool("version"); ver { + return runVersion(cmd) + } + if list, _ := cmd.Flags().GetBool("list"); list { + return runList(cmd) + } + return cmd.Help() + }, +} + +func init() { + // Shared flags. Declared on PersistentFlags so every provider + // subcommand inherits them; a subcommand's RunE reads them via + // cmd.Flags() or cmd.InheritedFlags(). + pf := rootCmd.PersistentFlags() + pf.String("secret-env", "", "Name of the environment variable holding the signing secret") + pf.Int("port", 8080, "HTTP port") + pf.String("host", "127.0.0.1", "Bind address") + pf.String("path", "", "Endpoint path (default /)") + pf.Bool("pretty", false, "Human-readable output instead of JSONL") + pf.Int64("max-body", 2097152, "Max request body in bytes (2MB)") + pf.Int("timeout", 10, "Read and write timeout, seconds") + + // Root-only flags. Declared on Flags, not PersistentFlags, because + // --list and --version are meaningful only when no subcommand was + // given. webhookd mock --list is a user error, and cobra rejects + // it with "unknown flag" rather than silently ignoring it. + rootCmd.Flags().Bool("list", false, "List all registered providers and exit") + rootCmd.Flags().Bool("version", false, "Print version and exit") +} + +// Execute runs the root command. +// +// version is the build version, injected into main by the linker via +// -X main.version=... The spec's Makefile targets main.version, so +// main.go owns the variable and passes it here; Execute stashes it so +// subcommands can include it in the startup banner. +// +// The return value is the process exit code: 0 for success, 1 for any +// error. main.go passes it to os.Exit. +func Execute(version string) int { + buildVersion = version + if err := rootCmd.Execute(); err != nil { + fmt.Fprintf(os.Stderr, "webhookd: %v\n", err) + return 1 + } + return 0 +} + +// runList prints the name of every registered provider, one per line, +// to stdout. The list comes from providers.All(), which sees every +// provider whose cmd/.go is compiled into this binary. +// +// Output goes to stdout, not stderr: --list is a data query, matching +// the same stdout-is-data contract the JSONL stream follows. A caller +// can pipe it: `webhookd --list | xargs -n1 webhookd`. +func runList(cmd *cobra.Command) error { + out := cmd.OutOrStdout() + for _, name := range providers.All() { + if _, err := fmt.Fprintln(out, name); err != nil { + return err + } + } + return nil +} + +// runVersion prints the version string to stdout. +// +// The format is fixed by docs/webhookd-core.md §"--version": +// "webhookd ", one line, exit 0. +func runVersion(cmd *cobra.Command) error { + _, err := fmt.Fprintf(cmd.OutOrStdout(), "webhookd %s\n", buildVersion) + return err +} \ No newline at end of file diff --git a/config/config.go b/config/config.go new file mode 100644 index 0000000..b4273f5 --- /dev/null +++ b/config/config.go @@ -0,0 +1,242 @@ +// Package config resolves the webhookd runtime configuration from CLI +// flags, environment variables, and built-in defaults. +// +// Precedence, highest first, per docs/webhookd-core.md §"Config +// resolution order": +// +// 1. CLI flag, if the user changed it +// 2. Environment variable, if set +// 3. Built-in default +// +// "Changed" is the operative word for the first level: a flag whose +// value equals the default is indistinguishable from an unset flag +// unless the framework tells us, and cobra's Flags().Changed() does. +// That matters for --pretty, where --pretty=false is a real choice and +// must beat a WEBHOOKD_PRETTY=true environment variable. +// +// The secret value itself never enters a Config. Config stores the +// *name* of the environment variable that holds the secret. The +// provider reads the value at verification time, so the secret is not +// present in any process memory the core can leak through a diagnostic +// or a log line. +package config + +import ( + "errors" + "fmt" + "os" + "strconv" + "strings" + "time" + + "github.com/spf13/cobra" +) + +// Built-in defaults. Named rather than inlined so a reader can find +// the source of any default value by searching for the constant name. +const ( + DefaultPort = 8080 + DefaultHost = "127.0.0.1" + DefaultPretty = false + DefaultMaxBody = 2 * 1024 * 1024 // 2 MB, matching the spec's default + DefaultTimeout = 10 * time.Second + + // EnvPrefix namespaces every environment variable that overrides a + // flag. WEBHOOKD_PORT overrides --port, WEBHOOKD_HOST overrides + // --host, and so on. The one flag without an env-var override is + // --secret-env, which *names* another env var rather than taking a + // value from one. + EnvPrefix = "WEBHOOKD_" +) + +// maxPort is the largest valid TCP port number. +const maxPort = 65535 + +// Config is the fully-resolved runtime configuration. +// +// Every field is populated by Load. No caller re-derives defaults. +type Config struct { + // ProviderName is the subcommand the binary was invoked with, + // e.g. "mock" or "github". It is the registry key, the "provider" + // field in output, and the basis for the default Path. + ProviderName string + + // SecretEnv is the name of the environment variable that holds + // the signing secret. Empty means the provider must supply its own + // default (declared in its cmd/.go) or fail. + SecretEnv string + + // Port is the TCP port the server binds. 0 asks the operating + // system to choose. + Port int + + // Host is the bind address. + Host string + + // Path is the URL path the webhook handler is mounted at. Always + // begins with "/". + Path string + + // Pretty selects human-readable output over JSONL. + Pretty bool + + // MaxBody is the request body size cap in bytes. + MaxBody int64 + + // Timeout is applied as both ReadTimeout and WriteTimeout on the + // HTTP server. + Timeout time.Duration +} + +// Load resolves a Config for the named provider. +// +// cmd must already have the shared flags declared on it. In practice +// that is done by cmd/root.go before this function is called. cmd must +// not be nil. +// +// providerName must be non-empty. It is used to compute the default +// Path, so a missing name produces a wrong default, not just an ugly +// label. +func Load(cmd *cobra.Command, providerName string) (*Config, error) { + if cmd == nil { + return nil, errors.New("config: nil command") + } + if providerName == "" { + return nil, errors.New("config: empty provider name") + } + + cfg := &Config{ProviderName: providerName} + + // --secret-env: the name of an environment variable. The default + // is "", which means "no secret; the provider must know its own + // default." A provider's cmd/.go is the place to supply that + // default, not this package. + cfg.SecretEnv = stringValue(cmd, "secret-env", EnvPrefix+"SECRET_ENV", "") + + // --port + port, err := intValue(cmd, "port", EnvPrefix+"PORT", DefaultPort) + if err != nil { + return nil, err + } + if port < 0 || port > maxPort { + return nil, fmt.Errorf("config: port %d out of range [0, %d]", port, maxPort) + } + cfg.Port = port + + // --host + cfg.Host = stringValue(cmd, "host", EnvPrefix+"HOST", DefaultHost) + + // --path + defaultPath := "/" + providerName + cfg.Path = stringValue(cmd, "path", EnvPrefix+"PATH", defaultPath) + if !strings.HasPrefix(cfg.Path, "/") { + return nil, fmt.Errorf("config: path %q must begin with /", cfg.Path) + } + + // --pretty + pretty, err := boolValue(cmd, "pretty", EnvPrefix+"PRETTY", DefaultPretty) + if err != nil { + return nil, err + } + cfg.Pretty = pretty + + // --max-body + maxBody, err := int64Value(cmd, "max-body", EnvPrefix+"MAX_BODY", DefaultMaxBody) + if err != nil { + return nil, err + } + if maxBody <= 0 { + return nil, fmt.Errorf("config: max-body %d must be positive", maxBody) + } + cfg.MaxBody = maxBody + + // --timeout is expressed in seconds on the CLI but stored as a + // time.Duration, matching http.Server.ReadTimeout and WriteTimeout. + timeoutSecs, err := intValue(cmd, "timeout", EnvPrefix+"TIMEOUT", int(DefaultTimeout/time.Second)) + if err != nil { + return nil, err + } + if timeoutSecs <= 0 { + return nil, fmt.Errorf("config: timeout %d must be positive", timeoutSecs) + } + cfg.Timeout = time.Duration(timeoutSecs) * time.Second + + return cfg, nil +} + +// stringValue resolves a string flag. The precedence is flag (if +// changed) > environment variable (if present) > default. +func stringValue(cmd *cobra.Command, flag, envKey, def string) string { + if cmd.Flags().Changed(flag) { + v, _ := cmd.Flags().GetString(flag) + return v + } + if v, ok := os.LookupEnv(envKey); ok { + return v + } + return def +} + +// intValue resolves an int flag. The error from a malformed +// environment variable is wrapped with the variable name so the user +// knows which one to fix. +func intValue(cmd *cobra.Command, flag, envKey string, def int) (int, error) { + if cmd.Flags().Changed(flag) { + v, err := cmd.Flags().GetInt(flag) + if err != nil { + return 0, fmt.Errorf("config: flag --%s: %w", flag, err) + } + return v, nil + } + if raw, ok := os.LookupEnv(envKey); ok { + v, err := strconv.Atoi(raw) + if err != nil { + return 0, fmt.Errorf("config: env %s=%q is not an integer", envKey, raw) + } + return v, nil + } + return def, nil +} + +// int64Value is the same as intValue but for int64 flags. +func int64Value(cmd *cobra.Command, flag, envKey string, def int64) (int64, error) { + if cmd.Flags().Changed(flag) { + v, err := cmd.Flags().GetInt64(flag) + if err != nil { + return 0, fmt.Errorf("config: flag --%s: %w", flag, err) + } + return v, nil + } + if raw, ok := os.LookupEnv(envKey); ok { + v, err := strconv.ParseInt(raw, 10, 64) + if err != nil { + return 0, fmt.Errorf("config: env %s=%q is not an integer", envKey, raw) + } + return v, nil + } + return def, nil +} + +// boolValue resolves a bool flag. +// +// Environment values accept the same spellings as strconv.ParseBool: +// 1, t, T, TRUE, true, True, 0, f, F, FALSE, false, False. Anything +// else is an error, so a typo like WEBHOOKD_PRETTY=yes fails loudly +// rather than silently meaning false. +func boolValue(cmd *cobra.Command, flag, envKey string, def bool) (bool, error) { + if cmd.Flags().Changed(flag) { + v, err := cmd.Flags().GetBool(flag) + if err != nil { + return false, fmt.Errorf("config: flag --%s: %w", flag, err) + } + return v, nil + } + if raw, ok := os.LookupEnv(envKey); ok { + v, err := strconv.ParseBool(raw) + if err != nil { + return false, fmt.Errorf("config: env %s=%q is not a boolean", envKey, raw) + } + return v, nil + } + return def, nil +} diff --git a/docs/architecture.md b/docs/architecture.md index 54e7dcf..d687ed9 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -51,24 +51,27 @@ Every request follows this order. The order is mandatory — deviating from it breaks signature verification. ``` -1. Enforce body size limit (default 2MB) -2. Read entire raw body into []byte -3. Look up provider in registry by name -4. Call provider.Verify(request, rawBody) -5. If error: +1. Reject if method is not POST (405) +2. Reject if Content-Type is not application/json (415) +3. Wrap r.Body in http.MaxBytesReader(w, r.Body, maxBody) +4. Read entire raw body into []byte — a read error from step 3's wrapper + is reported as 413, any other read error as 500 +5. Look up provider in registry by name (404 if not found) +6. Call provider.Verify(request, rawBody) +7. If error: → write error to stderr → respond 401 → return — nothing goes to stdout -6. Call provider.EventType(request, rawBody) -7. Call provider.DeliveryID(request) -8. Call provider.EventID(request, rawBody) -9. JSON-decode payload from rawBody -10. Build normalized Event struct -11. Write one JSONL line to stdout -12. Respond 200 {"ok": true} +8. Call provider.EventType(request, rawBody) +9. Call provider.DeliveryID(request) +10. Call provider.EventID(request, rawBody) +11. Validate that rawBody is syntactically valid JSON (500 if not) +12. Build normalized Event struct +13. Write one JSONL line to stdout +14. Respond 200 {"ok": true} ``` -Steps 2 and 9 use the same bytes. The raw body is captured once and never +Steps 4 and 11 use the same bytes. The raw body is captured once and never re-encoded. If a provider hashes the body, it hashes the same bytes the caller will eventually see in `payload`. This is the single most important invariant in the codebase. @@ -145,17 +148,24 @@ Registering the same name twice means two packages think they own the same subcommand, and there is no correct way to resolve that at runtime. Registration happens through `init()` rather than an explicit registration -list in `main.go` so that adding a community provider is one blank import: +list in `main.go` so that the wiring for a provider is a single import in the +provider's own `cmd/.go` file. The core does not need to know the +provider exists until a request arrives for it. -```go -import ( - _ "github.com/0xProgress/webhookd/providers/shopify" -) +All providers live in this repository, under `providers//`. A provider +is added by a PR that touches three files: + +``` +providers//.go — the Provider implementation +providers//_test.go — the tests +cmd/.go — the subcommand, which imports the above ``` -The blank import runs the provider's `init()`, which calls `Register`. Nothing -else is required. The core does not need to know the provider exists until a -request arrives for it. +`cmd/.go` declares the subcommand, wires it into the root command, sets +any provider-specific defaults (for example, the conventional environment +variable name for the signing secret), and imports the provider package so +that its `init()` runs. That import is what registers the provider. Nothing +else in the codebase needs to change. `Get` and `All` are the read side. `Get` is called once per request, after the provider name is extracted from the URL path. `All` is called once at startup @@ -194,11 +204,20 @@ provider generated the event. Provider timestamps, when present, live inside `payload`. This distinction matters when the two are far apart, which is usually a sign of replay or clock skew. -`payload` is the full parsed JSON body, unmodified. webhookd does not strip, -reshape, or interpret it. If you need the raw bytes exactly as the provider -sent them, they are in `payload` as a decoded object — the original bytes are -not preserved on stdout because JSONL is line-oriented and raw bytes can -contain newlines. +`payload` is the full JSON body of the request. Key order, duplicate keys, and +numeric precision are preserved exactly as the provider sent them: the body is +passed through the core as raw bytes and stored in the `Event` as +`json.RawMessage`, not decoded into a Go map. Decoding into `map[string]any` +would re-sort keys, collapse duplicates, and convert every number to +`float64` — the float64 conversion silently loses precision on integers above +2^53, which includes real payment amounts. That is a reshape, and the output +contract forbids reshaping. + +The only transformation applied to `payload` is whitespace normalisation +performed by the JSONL encoder: the value is emitted compact on one line, with +insignificant whitespace removed and any embedded newlines escaped. This is +what makes the output line-oriented. The underlying structure and values are +unchanged. --- @@ -228,6 +247,10 @@ carries the human-readable form instead. This is a deliberate trade-off — pret mode exists for live demos and manual inspection, where piping is not the goal. The startup banner still goes to stderr in both modes. +The exact format of the startup banner and of failed-verification lines is +fixed by `docs/webhookd-core.md`. Everything else on stderr is free-form and +not a contract. + --- ## Security model @@ -239,12 +262,16 @@ a developer machine and explicit when exposed. | ------------------- | ------------------------------------------------------------------------ | | Signature forgery | Provider `Verify()` uses `hmac.Equal()` for constant-time comparison | | Replay | Providers that support timestamps reject requests older than 300 seconds | -| Body flooding | Body size limit enforced before read, default 2MB | +| Body flooding | Body size limit enforced during read, default 2MB | | Slowloris | Read and write timeouts of 10 seconds | | Accidental exposure | Default bind is `127.0.0.1`; `0.0.0.0` requires `--host` | | Secret leakage | Secrets are read from environment variables, never from flags | | Log injection | Payload content is never interpolated into log lines unsanitized | +The body size limit is enforced with `http.MaxBytesReader`, which rejects +during the read rather than after it. An oversized body is never fully +buffered. + The core cannot verify that a provider's `Verify()` is correct. It can only verify that the provider called `Register`, that it returned nil or an error, and that on error the request was rejected. Everything about the correctness of @@ -295,17 +322,18 @@ would make webhookd a different tool. There are exactly two ways to extend webhookd: -1. **Add a provider.** Implement the `Provider` interface, call `Register` in - `init()`, add a subcommand file in `cmd/`. See [CONTRIBUTING.md](../CONTRIBUTING.md). +1. **Add a provider.** Implement the `Provider` interface under + `providers//`, add a `cmd/.go` subcommand, and open a PR. See + [CONTRIBUTING.md](../CONTRIBUTING.md). 2. **Pipe the output somewhere.** The JSONL contract is stable. Anything that reads newline-delimited JSON can consume webhookd's output. There is no plugin system, no configuration file, no dynamic loading. A -provider is a Go package. Adding it means rebuilding the binary. This is -deliberate: a static binary with a fixed set of providers is easier to audit -than a dynamic loader, and the build is fast enough that this is not a real -constraint. +provider is a Go package in this repository. Adding it means rebuilding the +binary. This is deliberate: a static binary with a fixed set of providers is +easier to audit than a dynamic loader, and the build is fast enough that this +is not a real constraint. --- @@ -313,8 +341,8 @@ constraint. The core is complete and shippable before any real provider exists. The mock provider exercises the full pipeline and is the reference implementation for -contributors. Real providers — `github`, `stripe`, `slack` — are separate PRs -against a frozen core. +contributors. Real providers — `github`, `stripe`, `slack`, `shopify` — are +separate PRs against a frozen core. This ordering exists so that the core's interface can be reviewed, tested, and released without the pressure of a specific provider shaping it. If the first diff --git a/docs/providers/TEMPLATE.md b/docs/providers/TEMPLATE.md index 923d613..27ec35e 100644 --- a/docs/providers/TEMPLATE.md +++ b/docs/providers/TEMPLATE.md @@ -1,112 +1,383 @@ -# [Provider Name] — webhookd Provider +# Writing a Provider -> One sentence describing what [Provider Name] is and what events it sends. +> The long-form guide for adding a webhook provider to webhookd. -**Official docs:** [Link to provider's webhook documentation] +This is the deep version. The short version is in +[CONTRIBUTING.md §"Adding a Provider"](../../CONTRIBUTING.md#adding-a-provider). +Read that first if you have not. This document explains *why* the short +version says what it says. --- -## Signature Scheme +## What you are building -Describe exactly how this provider signs webhook requests. +A provider is a leaf node. It answers five questions about one HTTP request +and returns. It does not read the environment, does not talk to the network, +does not write to stdout or stderr, and does not know whether other providers +exist. The core is the pipeline; you are one stage of it. -| Detail | Value | -|--------|-------| -| Signature header | `X-Provider-Signature` | -| Signature format | `sha256=` or `v0=` etc. | -| Algorithm | HMAC-SHA256 | -| What is signed | Raw request body / `{timestamp}.{body}` / etc. | -| Timestamp header | `X-Provider-Timestamp` (or "None") | -| Max timestamp age | 300 seconds (or "Not applicable") | +That smallness is deliberate. Everything that runs before your code — the +HTTP listener, the body capture, the size limit, the routing — is written +once and reviewed once. Everything that runs after your code — the JSONL +encoding, the stream routing, the response — is written once and reviewed +once. The narrow part of the funnel is the part you own. --- -## Verification Steps +## The five methods -Numbered, exact steps matching the implementation in `.go`. +`providers/provider.go` defines the interface. Read the doc comments there; +they are the contract. What follows is the "why" behind each one. -1. Read the `X-Provider-Signature` header -2. Read the `X-Provider-Timestamp` header (if applicable) -3. Reject if timestamp is older than 300 seconds (if applicable) -4. Construct the signed string: `{timestamp}.{rawBody}` (or just `rawBody`) -5. Compute `HMAC-SHA256(secret, signedString)` -6. Compare result with the signature header value using constant-time comparison -7. Reject with 401 if comparison fails +### `Name() string` ---- +The lowercase identifier. It becomes the CLI subcommand, the `provider` +field in output, and the registry key. Two providers cannot share a name; +`Register` panics on a duplicate. -## Event Type +Do not title-case it. The core does not know how to render `github` as +`GitHub` and would rather show the lowercase form than maintain a +display-name table. Everything downstream, including pretty mode, prints the +name verbatim. -Where the event type is found and what format it takes. +### `Verify(r *http.Request, rawBody []byte) error` -| Field | Source | Example value | -|-------|--------|---------------| -| Event type | `X-Provider-Event` header or `type` field in body | `push`, `payment.succeeded` | +The security boundary. Nothing crosses it until it returns nil. ---- +`rawBody` is the exact bytes the client sent. If the provider signs the raw +body, hash `rawBody` directly. If it signs a composed string — +`{timestamp}.{rawBody}`, or `v0:{timestamp}:{rawBody}`, or anything else — +build that string and hash *that*. Do not re-read `r.Body`; it has already +been consumed and will return zero bytes. Do not call `json.Unmarshal` on +`rawBody` before hashing it; the JSON decoder does not preserve the original +byte sequence, and any hash computed after decoding will not match what the +provider sent. + +Return an error on any failure. The error string is written to stderr and +the client receives a 401. Nothing on the error path reaches stdout. + +The one hard rule: **use `hmac.Equal` for every MAC comparison.** See +[Why `hmac.Equal`](#why-hmacequal) below. + +### `EventType(r *http.Request, rawBody []byte) string` + +The event type. Some providers put it in a header (`X-GitHub-Event`), some in +the body (`"type": "payment_intent.succeeded"`), some in both. Read whichever +the provider's documentation specifies. + +Return `""` if the provider does not supply one. Do not invent a placeholder +and do not return an error — the method signature has no error return, and +the pipeline treats `""` as "no event type" rather than a failure. + +If the field lives in the body, decode only what you need. A tiny anonymous +struct with one field is enough: -## Environment Variables +```go +var envelope struct { + Type string `json:"type"` +} +if err := json.Unmarshal(rawBody, &envelope); err != nil { + return "" +} +return envelope.Type +``` + +Decoding the whole body into `map[string]any` is a mistake — it costs more +than it saves, and the fields you never read can mislead the next reader +into thinking they are part of the contract. + +### `DeliveryID(r *http.Request) string` -| Variable | Required | Description | -|----------|----------|-------------| -| `PROVIDER_WEBHOOK_SECRET` | Yes | The signing secret from your provider dashboard | +The delivery or request identifier, if the provider supplies one. This method +receives only the request, not the body, because delivery IDs are always in +headers for the providers that have them. + +Return `""` if absent. Return `""` even if the body contains something that +looks like an ID — that is what `EventID` is for. Do not pull a header that +is not documented as a delivery ID; the field is optional and an empty string +is honest, while a guess is not. + +### `EventID(r *http.Request, rawBody []byte) string` + +The event's own ID, if the provider supplies one. This method receives the +body because event IDs are often in the payload. Return `""` if absent. + +The distinction from `DeliveryID` is real and worth understanding. A delivery +ID identifies *this attempt to deliver an event* — a retry has a new delivery +ID but the same event ID. An event ID identifies *the event itself*. Stripe +has an event ID on the event object and no separate delivery ID. GitHub has +`X-GitHub-Delivery` and no event ID in the body. Both fields are optional. --- -## Usage +## Why `hmac.Equal` + +A MAC comparison with `==` leaks information through timing. Every byte that +matches lets the comparison run a few nanoseconds longer before it fails, and +an attacker who can measure response time can recover the correct signature +one byte at a time. This is a well-understood attack, and it works against +naive code. -```bash -export PROVIDER_WEBHOOK_SECRET=your_secret_here +`hmac.Equal` is constant-time: it takes the same amount of time whether the +two inputs differ in the first byte or the last. That is the only reason it +exists and the only reason the rule is absolute. -webhookd +Never use: + +```go +if sig == expected { ... } // leaks +if strings.Compare(sig, expected) == 0 { } // leaks +if bytes.Equal(sigBytes, expectedBytes) { } // leaks ``` -Default endpoint: `POST //` +Always use: + +```go +if hmac.Equal([]byte(sig), []byte(expected)) { ... } // constant-time +``` + +The comparison leaks only if the attacker can measure the timing of a request +they control. That is why the rule applies to MAC comparisons and not to, +say, comparing a public provider name against a registry key. + +--- + +## Timestamp validation + +Providers that include a timestamp in the signature — Stripe, Slack, and +others — are protecting against replay. Without a timestamp check, an +attacker who captures one valid request can send it again tomorrow, and the +signature will still verify because the body has not changed. + +The timestamp is part of the signed string, so an attacker cannot alter it +without invalidating the signature. The check is: parse the timestamp, +compare it against the current time, and reject if the difference exceeds +the tolerance. The spec fixes the tolerance at 300 seconds. + +**Order matters.** Validate the timestamp *after* computing and comparing the +MAC, not before. If you reject an old timestamp before checking the signature, +an attacker can distinguish "your timestamp is wrong" from "your signature is +wrong" by watching the error response — and can use that to enumerate valid +requests. Both checks should produce the same error string. -Custom path: +The correct order: -```bash -webhookd --path /webhooks/ +1. Read the signature header. +2. Read the timestamp header. +3. Parse the signature into `(scheme, hex_digest)`. Reject if malformed. +4. Compute `HMAC-SHA256(secret, signedString)` where `signedString` includes + the timestamp. +5. Compare with `hmac.Equal`. Reject if it fails. +6. Parse the timestamp. +7. Reject if the timestamp is more than 300 seconds old. + +Step 7 runs only if steps 1–6 succeed. Every rejection produces the same +error string on stderr and the same 401 to the client. + +A named constant for the tolerance keeps the 300 out of the middle of the +function: + +```go +const maxTimestampAge = 300 * time.Second ``` --- -## Example Output - -A complete example of the JSONL line produced for a real event from this provider. - -```json -{ - "provider": "", - "verified": true, - "event": "example.event", - "id": "evt_example123", - "delivery_id": "delivery_abc456", - "received_at": "2026-09-15T19:42:13Z", - "payload": { - "type": "example.event", - "data": { - "example": "value" +## A worked example: GitHub + +GitHub is the simplest real provider. Reading its implementation is a good +way to see the shape. The code below is illustrative; it is not in this +repository yet. + +GitHub's signature scheme, from +[Validating webhook deliveries](https://docs.github.com/en/webhooks/using-webhooks/validating-webhook-deliveries): + +| Detail | Value | +|---|---| +| Signature header | `X-Hub-Signature-256` | +| Signature format | `sha256=` | +| Algorithm | HMAC-SHA256 | +| What is signed | The raw request body | +| Timestamp header | None | +| Event type header | `X-GitHub-Event` | +| Delivery ID header | `X-GitHub-Delivery` | + +The implementation: + +```go +package github + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "errors" + "net/http" + "os" + "strings" + + "github.com/0xProgress/webhookd/providers" +) + +func init() { + providers.Register(&Provider{}) +} + +type Provider struct{} + +func (p *Provider) Name() string { return "github" } + +func (p *Provider) Verify(r *http.Request, rawBody []byte) error { + secret := []byte(os.Getenv("GITHUB_WEBHOOK_SECRET")) + if len(secret) == 0 { + return errors.New("github: GITHUB_WEBHOOK_SECRET is not set") + } + + sig := r.Header.Get("X-Hub-Signature-256") + if sig == "" { + return errors.New("github: missing X-Hub-Signature-256 header") + } + + // GitHub's format is "sha256=". Strip the prefix. + const prefix = "sha256=" + if !strings.HasPrefix(sig, prefix) { + return errors.New("github: signature missing sha256= prefix") + } + got := sig[len(prefix):] + + mac := hmac.New(sha256.New, secret) + mac.Write(rawBody) + want := hex.EncodeToString(mac.Sum(nil)) + + if !hmac.Equal([]byte(got), []byte(want)) { + return errors.New("github: signature mismatch") } - } + return nil +} + +func (p *Provider) EventType(r *http.Request, rawBody []byte) string { + return r.Header.Get("X-GitHub-Event") +} + +func (p *Provider) DeliveryID(r *http.Request) string { + return r.Header.Get("X-GitHub-Delivery") +} + +func (p *Provider) EventID(r *http.Request, rawBody []byte) string { + return "" } ``` +Several things to notice: + +- `Verify` reads the secret from the environment on every call. That is + cheap and avoids caching state across requests. +- The signature comparison is `hmac.Equal` on the hex-decoded halves, not + on the full `sha256=…` header. Stripping the prefix first is cleaner than + comparing the whole string, but both work as long as the comparison is + constant-time. +- `EventID` returns `""`. GitHub does not have a top-level event ID. Guessing + a value from `payload.pull_request.id` would be wrong — that is the pull + request's ID, not the event's. +- The error strings are all prefixed `github:`. That prefix is what appears + in the stderr diagnostic. + +Adding this provider to the repository would be three files: +`providers/github/github.go`, `providers/github/github_test.go`, and +`cmd/github.go`. + --- -## Notes +## The three files -Any provider-specific gotchas, edge cases, or things to be aware of. +A provider PR adds exactly three files. Nothing else. -For example: -- Some events from this provider do not include a delivery ID -- The timestamp is in Unix seconds, not milliseconds -- URL-verification challenges should be handled separately (Slack) +``` +providers//.go — the Provider implementation +providers//_test.go — the tests +cmd/.go — the subcommand, which imports the above +``` + +No `go.mod` change. No new dependency. No edit to any existing file. If your +provider seems to need a change to the core, that is a signal that either the +provider is wrong (it is trying to do something that belongs in the core) or +the interface is wrong (it needs something the interface does not provide, +which is a spec change and needs discussion first). + +--- + +## `cmd/.go` + +The subcommand file does three things: + +1. **Imports the provider package**, so its `init()` runs and it registers + itself. This is the only reason the provider appears at all — without + the import, the provider is compiled out and the binary does not know it + exists. +2. **Declares the subcommand** and adds it to the root command during + `init()`. +3. **Sets any provider-specific defaults**, most commonly the conventional + environment variable name for the signing secret. + +The pattern is the same for every provider. Look at `cmd/mock.go` for the +simplest working example. When your provider's secret has a well-known name — +`GITHUB_WEBHOOK_SECRET`, `STRIPE_WEBHOOK_SECRET`, `SLACK_SIGNING_SECRET` — set +it as the default in your subcommand's flag declaration. Users who have the +variable set can then omit `--secret-env` entirely. + +A provider whose secret name is not conventional should not set a default; +the user must pass `--secret-env` explicitly. --- -## References +## Common mistakes + +**Decoding the body before hashing it.** The most common error. Hash the +`rawBody` slice you received. If you find yourself reaching for `r.Body`, +you have already lost — it was consumed by the core before `Verify` was +called. + +**Using `==` for the signature comparison.** `hmac.Equal` is not a +stylistic preference. See [Why `hmac.Equal`](#why-hmacequal). + +**Rejecting old timestamps before checking the signature.** This leaks +information about which signatures are valid. See +[Timestamp validation](#timestamp-validation) for the correct order. + +**Returning a value from `EventID` that is not the event's ID.** A pull +request's ID, a message's ID, a delivery's ID — none of these are the event's +ID. If the provider does not supply one, return `""`. + +**Assuming `r.Body` is re-readable.** It is not. `http.Request.Body` is a +one-shot stream. The core reads it into `rawBody` before calling you. + +**Reading the environment in `init()` or in the `Provider` struct's +constructor.** Secrets can change between process start and a request, and +`Verify` is where you need the current value. Read it there. + +**Logging.** Do not log the secret, the signature, or the body. The core +does not, and a provider that does would be leaking material to stderr. + +**Adding a dependency.** Providers are standard-library only. If you find +yourself reaching for a third-party HTTP client or a JSON library, stop and +ask — the core exists to prevent this. + +--- -- [Provider Webhook Documentation](https://example.com/docs/webhooks) -- [Signature Verification Guide](https://example.com/docs/webhooks/signatures) -- [Event Types Reference](https://example.com/docs/webhooks/events) +## Where to look when stuck + +1. **`providers/provider.go`** — the interface's doc comments. The contract + you are implementing. +2. **`providers/mock/mock.go`** — a complete, minimal implementation. Your + provider is the same shape with a real signature scheme. +3. **`docs/webhookd-core.md`** — the build spec. Every rule the core enforces + about providers is stated there. +4. **`docs/architecture.md`** — the reasoning. Why the interface has five + methods, why `Verify` runs before decoding, why the output shape is fixed. +5. **`docs/providers/TEMPLATE.md`** — the documentation you will write. +6. **Your provider's own webhook documentation.** The only authoritative + source for the signature scheme, the header names, and the payload shape. + +If after all six the answer is still unclear, open an issue describing the +ambiguity. Do not invent behaviour. The interface is frozen at v0.1 and any +change needs discussion before code. \ No newline at end of file diff --git a/docs/providers/checklist.md b/docs/providers/checklist.md new file mode 100644 index 0000000..4d42fb5 --- /dev/null +++ b/docs/providers/checklist.md @@ -0,0 +1,131 @@ +# Provider self-review checklist + +> This is a companion to the canonical checklist in +> [CONTRIBUTING.md §"Adding a Provider", step 10](../../CONTRIBUTING.md#10-self-review-with-the-checklist). +> +> `CONTRIBUTING.md` is the source of truth. If this file and that one ever +> disagree, `CONTRIBUTING.md` wins. This file exists so you can open it in a +> second window, paste it into a PR description, or print it. + +Copy the checklist below into your PR description and tick each box. Every +box must be checked before the PR is ready for review. + +--- + +## Pre-work + +- [ ] An open issue exists for this provider (`provider: ` label) +- [ ] I have commented on the issue to claim it +- [ ] I have read the provider's official webhook documentation +- [ ] I have read [CONTRIBUTING.md §"Adding a Provider"](../../CONTRIBUTING.md#adding-a-provider) end to end +- [ ] I have read [docs/contributing/provider-guide.md](provider-guide.md) +- [ ] I have read [providers/provider.go](../../providers/provider.go) — the interface doc comments + +--- + +## Interface + +- [ ] `Name()` implemented — returns the lowercase identifier +- [ ] `Verify(r, rawBody)` implemented — uses `hmac.Equal()` +- [ ] `EventType(r, rawBody)` implemented +- [ ] `DeliveryID(r)` implemented +- [ ] `EventID(r, rawBody)` implemented +- [ ] `providers.Register()` called in `init()` +- [ ] `cmd/.go` added and imports the provider package + +--- + +## Security + +- [ ] `hmac.Equal()` used for all MAC comparisons +- [ ] No `==`, `strings.Compare`, or `bytes.Equal` on signature material +- [ ] Raw body is hashed directly — no `json.Unmarshal` before hashing +- [ ] Timestamp validated and rejected if older than 300 seconds (where applicable) +- [ ] Timestamp validated *after* the signature check, not before (where applicable) +- [ ] Empty or missing secret returns a clear error +- [ ] Missing signature header returns a clear error +- [ ] Malformed signature header returns a clear error +- [ ] Error messages are prefixed with the provider name (`": ..."`) +- [ ] Secret is read from the environment in `Verify()`, not cached in `init()` or on the struct +- [ ] Secret, signature, and body are never logged + +--- + +## Tests + +- [ ] Valid signature is accepted +- [ ] Tampered body is rejected +- [ ] Wrong secret is rejected +- [ ] Missing signature header is rejected +- [ ] Malformed signature header is rejected +- [ ] Timestamp too old is rejected (where applicable) +- [ ] `EventType` returns the correct value for the provider's events +- [ ] `DeliveryID` returns the correct value (where applicable) +- [ ] `EventID` returns the correct value (where applicable) +- [ ] Test vectors are real — taken from the provider's official docs, not fabricated +- [ ] Test computes the signature the same way the provider does, rather than hardcoding a value +- [ ] `testing` and `net/http/httptest` only — no assertion libraries +- [ ] All tests pass: `make test` + +--- + +## Documentation + +- [ ] `docs/providers/.md` exists +- [ ] Copied from [docs/providers/TEMPLATE.md](../providers/TEMPLATE.md) +- [ ] Every section filled in — no placeholders left +- [ ] Signature Scheme table complete +- [ ] Verification Steps numbered, matching the implementation +- [ ] Event Type section complete +- [ ] Environment Variables section complete +- [ ] Example Output section contains a real JSONL line for a real event +- [ ] Official docs URL linked in the header +- [ ] Notes section covers any provider-specific gotchas + +--- + +## Code quality + +- [ ] `make check` passes clean (fmt + vet + lint + test) +- [ ] Provider code imports only standard-library packages +- [ ] No new `require` entries in `go.mod` +- [ ] No panics in `Verify()` or any request-path method +- [ ] Comments explain *why*, not *what* +- [ ] Public types and methods have doc comments +- [ ] No `// TODO` or `// FIXME` in place of logic +- [ ] No dead code, no commented-out code + +--- + +## PR + +- [ ] PR title follows [Conventional Commits](https://www.conventionalcommits.org/) — `feat: add provider` +- [ ] PR description includes the issue reference (`Closes #123`) +- [ ] PR description includes a `curl` example used to test the provider +- [ ] PR description links to the provider's official webhook documentation +- [ ] No secrets, tokens, or real webhook payloads anywhere in the diff +- [ ] Three files added: `providers//.go`, `providers//_test.go`, `cmd/.go` +- [ ] Plus one doc file: `docs/providers/.md` + +--- + +## What to do if a box cannot be ticked + +Do not leave a box unchecked and open the PR anyway. Every box on this list +exists because a previous contribution got that thing wrong, and the review +process will catch it — asking you to fix it after the PR is open. + +If a box genuinely does not apply to your provider — for example, "timestamp +validated" for a provider that does not send timestamps — write **N/A** next +to it with a one-line reason. A reviewer who sees `N/A — provider sends no +timestamp header` does not need to open the code to know the box was +considered. + +If a box *should* apply but cannot be ticked, that is a signal that either: + +- the provider implementation is incomplete, or +- the box describes a rule the provider genuinely cannot satisfy, which is a + spec question worth raising as an issue before opening the PR. + +Do not open the PR "so the reviewer can tell me what's missing." That wastes +both of your time. The reviewer's job is reviewing, not debugging. \ No newline at end of file diff --git a/docs/webhookd-core.md b/docs/webhookd-core.md index 60d5060..0818aef 100644 --- a/docs/webhookd-core.md +++ b/docs/webhookd-core.md @@ -6,9 +6,15 @@ ## What This Document Covers -This is the build document for the **core** of webhookd. No providers are included in the core. The core is the complete, working foundation that providers are built on top of. +This is the build document for the **core** of webhookd. The core is the +complete, working foundation that providers are built on top of. It ships +with one provider — `mock` — which exists as a reference implementation and +as the test double for the server pipeline. No real provider (`github`, +`stripe`, `slack`, `shopify`) is part of the core. -When the core ships, it is fully functional — it just has no built-in providers yet. A developer can implement a provider against this core on day one. +When the core ships, it is fully functional — it accepts webhooks, verifies +them, and streams verified events to stdout. A developer can implement a +provider against this core on day one. --- @@ -59,15 +65,29 @@ Every provider, regardless of implementation, produces this exact structure on s | `id` | string | no | Event ID if the provider supplies one, else `""` | | `delivery_id` | string | no | Delivery/request ID if the provider supplies one, else `""` | | `received_at` | string | yes | ISO 8601 UTC timestamp of receipt | -| `payload` | object | yes | Full parsed JSON body | +| `payload` | object | yes | Full JSON body of the request | -`verified` is always `true` on stdout. A failed verification produces a 401 response and a stderr log line. Nothing goes to stdout. +`verified` is always `true` on stdout. A failed verification produces a 401 +response and a stderr log line. Nothing goes to stdout. + +**All seven fields are always present.** Fields without a value are emitted as +`""` (empty string), never omitted. The shape is identical for every event from +every provider — that is what makes `webhookd | jq` work without a +provider-specific filter. + +`payload` is emitted from the raw request bytes. Key order, duplicate keys, and +numeric precision are preserved exactly as the provider sent them. The only +transformation applied is whitespace normalisation performed by the JSONL +encoder: the value is emitted compact on one line, with insignificant +whitespace removed and embedded newlines escaped. Structure and numbers are +not touched. --- ## The Provider Interface -This is the single most important thing in the codebase. Every provider — built-in or community — implements this contract exactly. +This is the single most important thing in the codebase. Every provider +implements this contract exactly. ```go package providers @@ -144,7 +164,7 @@ func All() []string { } ``` -Built-in providers register in `init()`: +Every provider registers itself in `init()`: ```go func init() { @@ -152,13 +172,25 @@ func init() { } ``` -Community providers do the same. Adding a community provider to a build is one blank import: +**All providers live in this repository**, under `providers//`. There is +no external provider mechanism: a new provider is a PR against this repo, not +a separate module that users import. The set of providers a binary supports is +fixed at build time. -```go -import ( - _ "github.com/community/webhookd-shopify" - _ "github.com/community/webhookd-discord" -) +A provider package's `init()` runs because its `cmd/.go` file imports the +package. That command file is also where the provider's subcommand is wired +into the CLI and where any provider-specific defaults are declared — for +example, the conventional environment variable name for the signing secret. +Nothing else in the codebase needs to know the provider exists: the `init()` +call performs registration, and the registry is what the server consults at +request time. + +Adding a provider is therefore three files in one PR: + +``` +providers//.go — the Provider implementation +providers//_test.go — the tests +cmd/.go — the subcommand, which imports the above ``` --- @@ -168,10 +200,11 @@ import ( ``` webhookd/ │ -├── main.go # Entry point — wires cobra root +├── main.go # Entry point — wires the root command │ ├── cmd/ -│ └── root.go # Root cobra command, shared flags, provider dispatch +│ ├── root.go # Root command, shared flags, provider dispatch +│ └── .go # One file per provider subcommand │ ├── server/ │ ├── server.go # HTTP listener, routing, timeouts @@ -187,33 +220,54 @@ webhookd/ │ └── mock_test.go │ ├── output/ +│ ├── event.go # Normalized Event struct │ ├── writer.go # JSONL writer to stdout -│ ├── pretty.go # Human-readable formatter -│ └── event.go # Normalized Event struct +│ └── pretty.go # Human-readable formatter │ ├── config/ │ └── config.go # Flag + env resolution, Config struct │ ├── docs/ │ ├── architecture.md # How the core works +│ ├── webhookd-core.md # This document — the build spec │ ├── providers/ │ │ └── TEMPLATE.md # Provider documentation template │ └── contributing/ │ ├── provider-guide.md # Full guide for writing a provider │ └── checklist.md # Provider checklist │ +├── scripts/ +│ ├── AGENTS.md # Operating instructions for AI coding agents +│ └── setup-repo.sh +│ ├── .github/ -│ └── workflows/ # See repo-workflows document +│ ├── CODEOWNERS +│ ├── dependabot.yml +│ ├── ISSUE_TEMPLATE/ +│ ├── pull_request_template.md +│ ├── SECURITY.md +│ └── workflows/ +│ ├── ci.yml +│ ├── codeql.yml +│ └── release.yml │ -├── CONTRIBUTING.md +├── .commitlintrc.json +├── .gitignore +├── .goreleaser.yaml +├── .golangci.yml ├── CHANGELOG.md +├── CONTRIBUTING.md +├── Dockerfile ├── LICENSE ├── Makefile -├── Dockerfile -├── .goreleaser.yaml -└── README.md +├── README.md +├── SECURITY.md +└── go.mod ``` +`scripts/AGENTS.md` is intentionally not committed — `scripts/` is listed in +`.gitignore`. It is a local operating document, not a shipped artifact. + --- ## Request Handling Order @@ -221,24 +275,41 @@ webhookd/ This order is mandatory. Any deviation breaks signature verification. ``` -1. Enforce body size limit (default 2MB) -2. Read entire raw body into []byte — store it, close nothing -3. Look up provider in registry by name -4. Call provider.Verify(request, rawBody) -5. If error: +1. Reject if method is not POST (405) +2. Reject if Content-Type is not application/json (415) +3. Wrap r.Body in http.MaxBytesReader(w, r.Body, maxBody) +4. Read entire raw body into []byte — a read error from step 3's wrapper + is reported as 413, any other read error as 500 +5. Look up provider in registry by name (404 if not found) +6. Call provider.Verify(request, rawBody) +7. If error: → write error to stderr → respond 401 → return — nothing goes to stdout -6. Call provider.EventType(request, rawBody) -7. Call provider.DeliveryID(request) -8. Call provider.EventID(request, rawBody) -9. JSON-decode payload from rawBody -10. Build normalized Event struct -11. Write one JSONL line to stdout -12. Respond 200 {"ok": true} +8. Call provider.EventType(request, rawBody) +9. Call provider.DeliveryID(request) +10. Call provider.EventID(request, rawBody) +11. Validate that rawBody is syntactically valid JSON (500 if not) +12. Build normalized Event struct +13. Write one JSONL line to stdout +14. Respond 200 {"ok": true} ``` -Step 2 and step 9 use the **same bytes**. Never re-encode between them. +Steps 4 and 11 use the **same bytes**. Never re-encode between them. + +Step 3 uses `http.MaxBytesReader`, which returns a distinct error during the +read in step 4 when the limit is exceeded. That error is what drives the 413 +response. The limit is therefore enforced *during* the read, not after it — an +oversized body is never fully buffered. This is the mechanism the security +requirement "body size limit enforced before read" refers to. + +Verification (step 6) runs before JSON validity is checked (step 11) and before +`payload` is emitted. A request with a bad signature is rejected before any +parsing of its body, so a malformed or hostile body cannot reach the decoder. +Signature verification is a security boundary. + +The provider lookup in step 5 is by name; the name is extracted from the URL +path after routing. An unknown path yields 404 before any body is read. --- @@ -252,9 +323,21 @@ Step 2 and step 9 use the **same bytes**. Never re-encode between them. | Wrong method | `405 Method Not Allowed` | `{"error": "method not allowed"}` | | Body too large | `413 Payload Too Large` | `{"error": "request body too large"}` | | Bad content type | `415 Unsupported Media Type` | `{"error": "unsupported content type"}` | -| Internal error | `500 Internal Server Error` | `{"error": "internal error"}` | +| Malformed JSON body | `500 Internal Server Error` | `{"error": "internal error"}` | + +All error responses go to the HTTP client. Nothing goes to stdout. The error +description goes to stderr. -All error responses go to the HTTP client. Nothing goes to stdout. The error description goes to stderr. +### Content-Type policy + +The core accepts requests whose `Content-Type` header begins with +`application/json`. A missing or mismatched `Content-Type` yields 415 before +the body is read. Real providers may send a charset suffix +(`application/json; charset=utf-8`); the prefix match accommodates this. + +The policy is set by the core, not by the provider interface. A provider whose +upstream sends a different content type (form-encoded, for example) cannot be +supported without a change to this document. --- @@ -264,7 +347,7 @@ All error responses go to the HTTP client. Nothing goes to stdout. The error des |-------------|------| | MAC comparison | `hmac.Equal()` only. Never `==`. Never string comparison. | | Timestamp validation | Where provider supports it: reject if older than 300 seconds | -| Body size limit | Default 2MB. Configurable via `--max-body`. Enforced before read. | +| Body size limit | Default 2MB. Configurable via `--max-body`. Enforced during read via `http.MaxBytesReader`. | | Read timeout | 10 seconds default. Connections cannot hang. | | Write timeout | 10 seconds default. | | Bind address | Default `127.0.0.1`. Never `0.0.0.0` by default. | @@ -300,7 +383,14 @@ webhookd --version --secret-env STRIPE_WEBHOOK_SECRET ``` -This means: read the secret value from the environment variable named `STRIPE_WEBHOOK_SECRET`. The secret value itself never appears in a flag. +This means: read the secret value from the environment variable named +`STRIPE_WEBHOOK_SECRET`. The secret value itself never appears in a flag. + +When `--secret-env` is empty (`""`, the default), the subcommand for a +provider may supply a conventional default variable name — for example, the +`github` subcommand defaults to `GITHUB_WEBHOOK_SECRET`. The default is +declared in the provider's `cmd/.go`, not in the core. The core only +knows what it is told via `--secret-env`. ### Config resolution order @@ -310,6 +400,30 @@ This means: read the secret value from the environment variable named `STRIPE_WE 3. Built-in default ``` +### `--list` + +Prints the name of every registered provider, one per line, to stdout, in +the order returned by `providers.All()`. Exits 0. + +``` +github +mock +slack +stripe +``` + +### `--version` + +Prints a single line to stdout and exits 0. + +``` +webhookd v0.1.0 +``` + +The version string is injected at build time via +`-ldflags "-X main.version=..."`. When unset (a plain `go build`), the +value is `dev` and the line reads `webhookd dev`. + --- ## stdout vs stderr Rule @@ -327,6 +441,33 @@ webhookd github | jq '.payload.repository.full_name' Startup messages and errors on stderr never contaminate the data stream. +### Diagnostic formats + +Two diagnostic lines have fixed formats. Everything else on stderr is +free-form and not a contract. + +**Startup banner** — printed once at startup, to stderr: + +``` +webhookd v0.1.0 — listening on 127.0.0.1:8080, endpoint POST /mock +``` + +The version is the same string as `--version`. The endpoint is +`:` followed by `POST` and the resolved path. + +**Failed verification** — printed once per rejected request, to stderr: + +``` +webhookd: github: signature mismatch — 203.0.113.4 +``` + +The format is `webhookd: : — `. The remote IP +is taken from `r.RemoteAddr` with the port stripped. `X-Forwarded-For` is not +consulted — it is attacker-controlled unless webhookd is behind a trusted +proxy, and webhookd makes no assumption that it is. + +The `` is the provider's own error string, verbatim. + --- ## Output Modes @@ -350,7 +491,7 @@ webhookd github --pretty ``` ``` -GitHub +github ────────────────────────────────────── ✓ Signature verified @@ -363,7 +504,13 @@ Received: 2026-09-15T19:44:03Z } ``` -Pretty output goes to stdout so it can be redirected. Startup info still goes to stderr. +The provider line is the lowercase identifier returned by `Name()`. The core +does not know how to title-case it — `github` is not `GitHub` by any general +rule, and hard-coding a display name per provider would violate the +core/provider split. + +Pretty output goes to stdout so it can be redirected. Startup info still goes +to stderr. --- @@ -384,6 +531,10 @@ Response: Always returns 200. No dependency checks. No metrics. Nothing more. +The version value is the same string as `--version` prints (without the +`webhookd ` prefix and without the leading `v`). When unset at build time, +the value is `dev`. + --- ## The Mock Provider @@ -403,6 +554,8 @@ The mock provider ships with the core. It exists for two purposes: Every test that exercises the server pipeline uses the mock provider. +The mock provider has no secret. `--secret-env` is ignored for `mock`. + --- ## Makefile @@ -444,7 +597,7 @@ release: ## Dockerfile ```dockerfile -FROM golang:1.23-alpine AS builder +FROM golang:1.27-alpine AS builder WORKDIR /app COPY go.mod go.sum ./ RUN go mod download @@ -502,6 +655,23 @@ changelog: --- +## Dependencies + +The core prefers the standard library. The only external dependency in the +core is the CLI framework, `github.com/spf13/cobra`, used in `cmd/`. This is +a deliberate choice: the CLI surface (subcommands, shared flags, `--help`, +shell completion) is the one place where hand-rolled code would be more +error-prone and more verbose than a small, widely used dependency. + +Providers must not add dependencies. A provider is standard-library-only — +that is a rule, not a preference, because a provider runs on the request path +and every dependency in that path is a supply-chain risk. + +The review criterion is: does the dependency earn its place on the request +path or in the CLI? If not, it does not belong. + +--- + ## v0.1 Acceptance Criteria The core is shippable when: @@ -518,7 +688,9 @@ The core is shippable when: - [ ] Read and write timeouts are set - [ ] Default bind is `127.0.0.1` - [ ] `--list` prints registered providers +- [ ] `--version` prints the version - [ ] `--pretty` produces human-readable output +- [ ] Startup banner and failed-verification formats match this document - [ ] Health endpoint returns 200 - [ ] `make check` passes clean - [ ] Single binary builds for all five platforms @@ -530,10 +702,13 @@ The core is shippable when: ## What Ships After Core -Once the core is tagged and released, providers follow as separate PRs. Each provider is independent. The order is: +Once the core is tagged and released, providers follow as separate PRs. Each +provider is independent. The order is: 1. `providers/github` — simplest verification scheme, good first provider 2. `providers/stripe` — timestamp validation adds complexity 3. `providers/slack` — base string construction is the tricky part +4. `providers/shopify` — base64-encoded HMAC, no timestamp -Community can pick up any of these or add new ones. The core does not need to change for any of them. +Community can pick up any of these or add new ones. The core does not need to +change for any of them. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..56bed4a --- /dev/null +++ b/go.mod @@ -0,0 +1,10 @@ +module github.com/0xProgress/webhookd + +go 1.27.1 + +require github.com/spf13/cobra v1.10.2 + +require ( + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/spf13/pflag v1.0.9 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..a6ee3e0 --- /dev/null +++ b/go.sum @@ -0,0 +1,10 @@ +github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/spf13/cobra v1.10.2 h1:DMTTonx5m65Ic0GOoRY2c16WCbHxOOw6xxezuLaBpcU= +github.com/spf13/cobra v1.10.2/go.mod h1:7C1pvHqHw5A4vrJfjNwvOdzYu0Gml16OCs2GRiTUUS4= +github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= +github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/main.go b/main.go new file mode 100644 index 0000000..7b24a11 --- /dev/null +++ b/main.go @@ -0,0 +1,31 @@ +// Command webhookd is the entry point for the webhookd binary. +// +// webhookd listens for webhook HTTP requests, verifies their +// signatures, and writes one JSONL line per verified event to stdout. +// The CLI surface and provider dispatch live in the cmd package; this +// file exists only to inject the build version and pass the exit code +// from cobra to the operating system. +package main + +import ( + "os" + + "github.com/0xProgress/webhookd/cmd" +) + +// version is the build version. +// +// The linker injects it via -X main.version=... The Makefile's build +// target sets it to `git describe --tags --always`; GoReleaser sets it +// to the release tag. When neither is used — a plain `go build` with +// no ldflags — the value stays "dev", and both `webhookd --version` +// and the startup banner report "dev". +// +// This variable is the sole target of the -X main.version linker flag +// in the entire codebase. Do not rename it without updating the +// Makefile and .goreleaser.yaml in the same change. +var version = "dev" + +func main() { + os.Exit(cmd.Execute(version)) +} \ No newline at end of file diff --git a/output/event.go b/output/event.go new file mode 100644 index 0000000..82398ad --- /dev/null +++ b/output/event.go @@ -0,0 +1,61 @@ +// Package output defines the normalized Event struct that every +// provider's output is mapped to, and the writers that emit it. +package output + +import "encoding/json" + +// Event is the normalized form of a single verified webhook. +// +// Every provider produces this exact shape on stdout, one JSON object +// per line. A consumer that parses one webhookd event can parse all of +// them, regardless of which provider produced it. This is what makes +// `webhookd | jq` work without a provider-specific filter. +type Event struct { + // Provider is the lowercase provider name, e.g. "stripe". + Provider string `json:"provider"` + + // Verified is always true on stdout. Failed verification never + // reaches output; a 401 is returned to the sender and a line is + // written to stderr instead. The field exists so consumers can + // rely on a stable schema and so the semantic meaning of the line + // is self-describing: the presence of the line is the assertion + // that verification succeeded. + Verified bool `json:"verified"` + + // Event is the provider-specific event type string, or "" if the + // provider does not supply one. + Event string `json:"event"` + + // ID is the event's own ID, or "" if the provider does not supply + // one. + ID string `json:"id"` + + // DeliveryID is the delivery or request ID, or "" if the provider + // does not supply one. + DeliveryID string `json:"delivery_id"` + + // ReceivedAt is the time the core received the request, formatted + // as RFC 3339 UTC (e.g. "2026-09-15T19:42:13Z"). + // + // This is not the time the provider generated the event. Provider + // timestamps, when present, live inside Payload. The distinction + // matters when the two are far apart, which is usually a sign of + // replay or clock skew. + ReceivedAt string `json:"received_at"` + + // Payload is the JSON body of the request. + // + // Held as json.RawMessage so the bytes that Verify() saw are the + // bytes the consumer sees. Decoding into map[string]any would + // re-sort keys, collapse duplicate keys, and convert every number + // to float64 — all of which are reshape operations the output + // contract forbids, and the float64 conversion silently loses + // precision on integers above 2^53, which includes real payment + // amounts. + // + // encoding/json compacts a RawMessage when marshaling, so any + // whitespace in the original body is dropped and the value is + // emitted on one line. That is the only permitted change: the + // structure and the numbers survive untouched. + Payload json.RawMessage `json:"payload"` +} diff --git a/output/pretty.go b/output/pretty.go new file mode 100644 index 0000000..db87829 --- /dev/null +++ b/output/pretty.go @@ -0,0 +1,82 @@ +package output + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "strings" + "sync" +) + +const ( + // separatorWidth is the width of the horizontal rule drawn under + // the provider name. Matches the width shown in the pretty-mode + // examples in README.md and webhookd-core.md. + separatorWidth = 38 + + // fieldLabelWidth is the column width for the field labels in + // pretty mode. The widest label, "Delivery ID:", is 12 characters; + // 14 leaves two spaces of padding so the values line up. + fieldLabelWidth = 14 +) + +// PrettyWriter emits events in a human-readable form to a single +// io.Writer. +// +// This is the destination selected by --pretty. It is intended for +// live demos and manual inspection, where the JSONL contract is not the +// point. The startup banner still goes to stderr; only the event body +// is written here. +// +// Safe for concurrent use for the same reason as Writer: the HTTP +// server dispatches each request on its own goroutine. +type PrettyWriter struct { + mu sync.Mutex + w io.Writer +} + +// NewPrettyWriter returns a PrettyWriter that emits to w. +func NewPrettyWriter(w io.Writer) *PrettyWriter { + return &PrettyWriter{w: w} +} + +// Write emits e in the pretty format. +// +// The format is fixed by the examples in README.md and +// webhookd-core.md: provider name, a horizontal rule, a verification +// line, then Event / Delivery ID / Received as aligned label-value +// pairs, then the payload indented with two spaces. +// +// The payload is indented with json.Indent rather than re-marshaled, so +// the bytes that Verify() saw reach the terminal unchanged apart from +// the added indentation. This is the same fidelity guarantee that +// Writer provides in JSONL mode. +func (w *PrettyWriter) Write(e *Event) error { + var buf bytes.Buffer + + fmt.Fprintln(&buf, e.Provider) + fmt.Fprintln(&buf, strings.Repeat("─", separatorWidth)) + fmt.Fprintln(&buf, "✓ Signature verified") + fmt.Fprintln(&buf) + + fmt.Fprintf(&buf, "%-*s%s\n", fieldLabelWidth, "Event:", e.Event) + fmt.Fprintf(&buf, "%-*s%s\n", fieldLabelWidth, "Delivery ID:", e.DeliveryID) + fmt.Fprintf(&buf, "%-*s%s\n", fieldLabelWidth, "Received:", e.ReceivedAt) + + fmt.Fprintln(&buf) + + var indented bytes.Buffer + if err := json.Indent(&indented, e.Payload, "", " "); err != nil { + return fmt.Errorf("output: indent payload: %w", err) + } + buf.Write(indented.Bytes()) + fmt.Fprintln(&buf) + + w.mu.Lock() + defer w.mu.Unlock() + if _, err := w.w.Write(buf.Bytes()); err != nil { + return fmt.Errorf("output: write event: %w", err) + } + return nil +} diff --git a/output/writer.go b/output/writer.go new file mode 100644 index 0000000..47e5d8c --- /dev/null +++ b/output/writer.go @@ -0,0 +1,54 @@ +package output + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "sync" +) + +// Writer emits events as JSONL to a single io.Writer. +// +// The destination is provided by the caller rather than fixed to +// os.Stdout so that tests can capture the raw bytes and assert on +// them. Production code passes os.Stdout. +// +// Safe for concurrent use. The HTTP server handles requests in +// separate goroutines, so two handlers can reach Write at the same +// time; without the mutex their byte sequences could interleave on +// the underlying writer and produce invalid JSONL. +type Writer struct { + mu sync.Mutex + w io.Writer +} + +// NewWriter returns a Writer that emits to w. +func NewWriter(w io.Writer) *Writer { + return &Writer{w: w} +} + +// Write emits one JSONL line for e. +// +// The line is exactly one compact JSON object followed by '\n'. HTML +// escaping is disabled so that '<', '>', and '&' in the payload are +// emitted as-is rather than rewritten to \u003c, \u003e, and \u0026; +// the default encoding/json behaviour would silently alter bytes the +// output contract says are passed through unmodified. +func (w *Writer) Write(e *Event) error { + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + enc.SetEscapeHTML(false) + if err := enc.Encode(e); err != nil { + return fmt.Errorf("output: marshal event: %w", err) + } + // Encoder.Encode terminates the value with '\n'. Nothing else may + // be appended: the JSONL contract is one line per event. + + w.mu.Lock() + defer w.mu.Unlock() + if _, err := w.w.Write(buf.Bytes()); err != nil { + return fmt.Errorf("output: write event: %w", err) + } + return nil +} diff --git a/providers/mock/mock.go b/providers/mock/mock.go new file mode 100644 index 0000000..229e73f --- /dev/null +++ b/providers/mock/mock.go @@ -0,0 +1,85 @@ +// Package mock provides a reference implementation of the Provider +// interface. +// +// It exists for two purposes: as a complete, working example that +// contributors can read and copy, and as a test double for the server +// pipeline. It is not intended for production use — its signature check +// is a literal string comparison against a well-known sentinel value, +// not a cryptographic MAC. +package mock + +import ( + "encoding/json" + "errors" + "net/http" + + "github.com/0xProgress/webhookd/providers" +) + +func init() { + providers.Register(&Provider{}) +} + +// Provider is the mock webhook provider. +// +// It accepts any request whose X-Mock-Signature header is the literal +// string "valid" and rejects all others. Event type and event ID are +// read from the top-level "type" and "id" fields of the JSON body. +type Provider struct{} + +// Name returns the provider identifier, used as the CLI subcommand and +// the "provider" field in output. +func (p *Provider) Name() string { + return "mock" +} + +// Verify accepts the request if X-Mock-Signature is "valid" and rejects +// all others. +// +// This is not a MAC check and does not use hmac.Equal. There is no +// secret: the header is compared against a fixed sentinel value that is +// public knowledge. The rule requiring hmac.Equal applies to MAC +// comparisons, where the comparison outcome could leak information +// about a secret. Here there is no secret to leak, and a timing-safe +// comparison would protect nothing. +func (p *Provider) Verify(r *http.Request, rawBody []byte) error { + sig := r.Header.Get("X-Mock-Signature") + if sig == "" { + return errors.New("mock: missing X-Mock-Signature header") + } + if sig != "valid" { + return errors.New("mock: signature mismatch") + } + return nil +} + +// EventType returns the value of the top-level "type" field in the +// body, or "" if the body is not valid JSON or the field is absent or +// not a string. +func (p *Provider) EventType(r *http.Request, rawBody []byte) string { + var envelope struct { + Type string `json:"type"` + } + if err := json.Unmarshal(rawBody, &envelope); err != nil { + return "" + } + return envelope.Type +} + +// DeliveryID is not supplied by the mock provider. Always returns "". +func (p *Provider) DeliveryID(r *http.Request) string { + return "" +} + +// EventID returns the value of the top-level "id" field in the body, or +// "" if the body is not valid JSON or the field is absent or not a +// string. +func (p *Provider) EventID(r *http.Request, rawBody []byte) string { + var envelope struct { + ID string `json:"id"` + } + if err := json.Unmarshal(rawBody, &envelope); err != nil { + return "" + } + return envelope.ID +} diff --git a/providers/mock/mock_test.go b/providers/mock/mock_test.go new file mode 100644 index 0000000..e1f278a --- /dev/null +++ b/providers/mock/mock_test.go @@ -0,0 +1,137 @@ +package mock + +import ( + "bytes" + "net/http/httptest" + "testing" + + "github.com/0xProgress/webhookd/providers" +) + +func TestName(t *testing.T) { + p := &Provider{} + if got := p.Name(); got != "mock" { + t.Fatalf("Name() = %q, want %q", got, "mock") + } +} + +func TestRegister(t *testing.T) { + // The init() in mock.go should have registered the provider under + // the name returned by Name(). If this test fails, the blank import + // pattern is broken somewhere between mock.go and the caller. + got, ok := providers.Get("mock") + if !ok { + t.Fatal("providers.Get(\"mock\") reported not found; init() did not register") + } + if _, isMock := got.(*Provider); !isMock { + t.Fatalf("providers.Get(\"mock\") returned %T, want *mock.Provider", got) + } +} + +func TestVerify(t *testing.T) { + tests := []struct { + name string + sig string // "" means the header is omitted + wantErr bool + }{ + {name: "valid", sig: "valid", wantErr: false}, + {name: "wrong value", sig: "invalid", wantErr: true}, + {name: "empty value", sig: "", wantErr: true}, + {name: "case sensitive", sig: "Valid", wantErr: true}, + {name: "whitespace", sig: " valid ", wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + body := []byte(`{"type":"test.event","id":"evt_1"}`) + req := httptest.NewRequest("POST", "/mock", bytes.NewReader(body)) + if tt.sig != "" { + req.Header.Set("X-Mock-Signature", tt.sig) + } + + err := (&Provider{}).Verify(req, body) + if tt.wantErr && err == nil { + t.Fatal("Verify returned nil, want error") + } + if !tt.wantErr && err != nil { + t.Fatalf("Verify returned %v, want nil", err) + } + }) + } +} + +func TestVerifyDoesNotReadBody(t *testing.T) { + // The mock provider's signature is entirely in the header. Body + // contents must not affect the verification result. This mirrors + // the pipeline invariant from the other direction: Verify sees the + // same bytes the caller will see, but for the mock those bytes + // carry no signature information. + req := httptest.NewRequest("POST", "/mock", nil) + req.Header.Set("X-Mock-Signature", "valid") + + if err := (&Provider{}).Verify(req, []byte("not json at all")); err != nil { + t.Fatalf("Verify rejected a request with a valid header and garbage body: %v", err) + } +} + +func TestEventType(t *testing.T) { + tests := []struct { + name string + body string + want string + }{ + {name: "present", body: `{"type":"order.created"}`, want: "order.created"}, + {name: "absent", body: `{"id":"evt_1"}`, want: ""}, + {name: "not a string", body: `{"type":42}`, want: ""}, + {name: "empty body", body: ``, want: ""}, + {name: "invalid json", body: `{`, want: ""}, + {name: "top-level array", body: `["type"]`, want: ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest("POST", "/mock", nil) + got := (&Provider{}).EventType(req, []byte(tt.body)) + if got != tt.want { + t.Fatalf("EventType(%q) = %q, want %q", tt.body, got, tt.want) + } + }) + } +} + +func TestEventID(t *testing.T) { + tests := []struct { + name string + body string + want string + }{ + {name: "present", body: `{"id":"evt_abc123"}`, want: "evt_abc123"}, + {name: "absent", body: `{"type":"order.created"}`, want: ""}, + {name: "not a string", body: `{"id":123}`, want: ""}, + {name: "empty body", body: ``, want: ""}, + {name: "invalid json", body: `null`, want: ""}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + req := httptest.NewRequest("POST", "/mock", nil) + got := (&Provider{}).EventID(req, []byte(tt.body)) + if got != tt.want { + t.Fatalf("EventID(%q) = %q, want %q", tt.body, got, tt.want) + } + }) + } +} + +func TestDeliveryIDIsAlwaysEmpty(t *testing.T) { + // The mock provider does not supply a delivery ID. Even when the + // request carries a header that looks like one, the method returns + // the empty string — the contract is that absence is expressed as + // "", never inferred from unrelated headers. + req := httptest.NewRequest("POST", "/mock", nil) + req.Header.Set("X-Mock-Delivery", "should-be-ignored") + + if got := (&Provider{}).DeliveryID(req); got != "" { + t.Fatalf("DeliveryID() = %q, want empty string", got) + } +} diff --git a/providers/provider.go b/providers/provider.go new file mode 100644 index 0000000..af21a3c --- /dev/null +++ b/providers/provider.go @@ -0,0 +1,71 @@ +// Package providers defines the Provider interface that every webhook +// provider implements, and the registry that maps provider names to +// implementations. +// +// The core knows nothing about any specific provider. A provider is a +// leaf node that answers five questions about a request; the pipeline +// that calls it lives in the server package. +package providers + +import "net/http" + +// Provider is the interface every webhook provider must implement. +// +// A provider owns its signature verification scheme entirely. The core +// makes no assumptions about how any provider signs requests. +// Implementations must use constant-time comparison for all MAC +// operations. +// +// The interface is frozen at v0.1. If a provider needs something the +// interface does not provide, the provider is wrong, not the interface. +type Provider interface { + // Name returns the lowercase provider identifier. + // + // This becomes the CLI subcommand and the "provider" field in + // output. Examples: "stripe", "github", "slack", "shopify". + // + // Two providers cannot share a name; Register panics on a + // duplicate. + Name() string + + // Verify checks the request signature against rawBody. + // + // rawBody is the original request body bytes, unmodified. Returns + // nil on success, a descriptive error on failure. Errors are + // written to stderr; the request receives a 401 and nothing is + // written to stdout. + // + // Verify is the security boundary. It is responsible for reading + // the signature header, reading any timestamp header, rejecting + // old timestamps, computing the expected MAC, and comparing in + // constant time. + // + // Implementations MUST use hmac.Equal() for all MAC comparisons. + Verify(r *http.Request, rawBody []byte) error + + // EventType extracts the event type string from the request. + // + // This becomes the "event" field in output. It may read a header, + // parse the body, or both. Return an empty string if the provider + // does not supply an event type. + EventType(r *http.Request, rawBody []byte) string + + // DeliveryID returns a unique delivery or request ID. + // + // This becomes the "delivery_id" field in output. It receives + // only the request, not the body, because delivery IDs are always + // in headers for the providers that have them. Return an empty + // string if absent. + DeliveryID(r *http.Request) string + + // EventID returns the event's own ID. + // + // This becomes the "id" field in output. It receives the body + // because event IDs are often in the JSON payload. Return an + // empty string if absent. + // + // The distinction from DeliveryID is real: a delivery ID + // identifies this attempt to deliver an event, and an event ID + // identifies the event itself. Both are optional. + EventID(r *http.Request, rawBody []byte) string +} diff --git a/providers/registry.go b/providers/registry.go new file mode 100644 index 0000000..68fe59d --- /dev/null +++ b/providers/registry.go @@ -0,0 +1,50 @@ +package providers + +// registry holds every registered provider, keyed by name. +// +// Populated by Register from provider init() functions, read by Get +// and All. The map is written only during package initialisation, before +// any request is served, and is read-only thereafter, so no +// synchronization is required. +var registry = map[string]Provider{} + +// Register adds a provider to the global registry. +// +// Called from provider init() functions. Panics if a provider with the +// same name is already registered. +// +// The panic is intentional: a duplicate registration is a programming +// error that should be caught at startup, not a runtime condition to +// handle. Registering the same name twice means two packages think +// they own the same subcommand, and there is no correct way to resolve +// that at runtime. +func Register(p Provider) { + name := p.Name() + if _, exists := registry[name]; exists { + panic("webhookd: provider already registered: " + name) + } + registry[name] = p +} + +// Get retrieves a registered provider by name. +// +// The second return value is false if no provider with that name is +// registered. Get is called once per request, after the provider name +// is extracted from the URL path. +func Get(name string) (Provider, bool) { + p, ok := registry[name] + return p, ok +} + +// All returns the names of every registered provider. +// +// Called once at startup for --list. The order of the returned slice is +// unspecified; callers that need deterministic ordering must sort it +// themselves. +func All() []string { + names := make([]string, 0, len(registry)) + for name := range registry { + names = append(names, name) + } + return names +} diff --git a/providers/registry_test.go b/providers/registry_test.go new file mode 100644 index 0000000..5ab68a1 --- /dev/null +++ b/providers/registry_test.go @@ -0,0 +1,126 @@ +package providers + +import ( + "net/http" + "testing" +) + +// fakeProvider is a minimal Provider used to exercise the registry +// without importing providers/mock, which imports this package and +// would create an import cycle. +type fakeProvider struct{ name string } + +func (f *fakeProvider) Name() string { return f.name } +func (f *fakeProvider) Verify(*http.Request, []byte) error { return nil } +func (f *fakeProvider) EventType(*http.Request, []byte) string { return "" } +func (f *fakeProvider) DeliveryID(*http.Request) string { return "" } +func (f *fakeProvider) EventID(*http.Request, []byte) string { return "" } + +// withCleanRegistry replaces the package registry with an empty map for +// the duration of the test and restores the original on cleanup. This +// keeps tests from seeing each other's registrations and from seeing +// any provider registered by an init() in the same binary. +func withCleanRegistry(t *testing.T) { + t.Helper() + orig := registry + registry = map[string]Provider{} + t.Cleanup(func() { registry = orig }) +} + +func TestRegisterStoresProvider(t *testing.T) { + withCleanRegistry(t) + + p := &fakeProvider{name: "alpha"} + Register(p) + + got, ok := Get("alpha") + if !ok { + t.Fatal("Get(alpha) reported not found after Register") + } + if got != p { + t.Fatal("Get returned a different provider than was registered") + } +} + +func TestGetUnknownName(t *testing.T) { + withCleanRegistry(t) + + if _, ok := Get("missing"); ok { + t.Fatal("Get on an unregistered name reported found") + } +} + +func TestRegisterDuplicatePanics(t *testing.T) { + withCleanRegistry(t) + + Register(&fakeProvider{name: "dup"}) + + defer func() { + r := recover() + if r == nil { + t.Fatal("second Register with the same name did not panic") + } + msg, ok := r.(string) + if !ok { + t.Fatalf("panic value has type %T, want string", r) + } + const want = "webhookd: provider already registered: dup" + if msg != want { + t.Fatalf("panic message = %q, want %q", msg, want) + } + }() + + Register(&fakeProvider{name: "dup"}) +} + +func TestAllEmpty(t *testing.T) { + withCleanRegistry(t) + + got := All() + if got == nil { + t.Fatal("All() returned nil on an empty registry, want non-nil empty slice") + } + if len(got) != 0 { + t.Fatalf("All() = %v on an empty registry, want empty slice", got) + } +} + +func TestAllContents(t *testing.T) { + withCleanRegistry(t) + + Register(&fakeProvider{name: "alpha"}) + Register(&fakeProvider{name: "bravo"}) + Register(&fakeProvider{name: "charlie"}) + + got := All() + want := []string{"alpha", "bravo", "charlie"} + + // All() makes no ordering promise, so compare as a set. + if !sameSet(got, want) { + t.Fatalf("All() = %v, want the set %v", got, want) + } +} + +// sameSet reports whether two string slices contain the same elements, +// ignoring order. Used because All() does not promise an ordering. +func sameSet(a, b []string) bool { + if len(a) != len(b) { + return false + } + seen := make(map[string]int, len(a)) + for _, s := range a { + seen[s]++ + } + for _, s := range b { + seen[s]-- + if seen[s] < 0 { + return false + } + } + for _, n := range seen { + if n != 0 { + return false + } + } + return true +} diff --git a/server/handler.go b/server/handler.go new file mode 100644 index 0000000..778f127 --- /dev/null +++ b/server/handler.go @@ -0,0 +1,199 @@ +// Package server implements the HTTP listener and request handler for +// webhookd. +// +// The request handling order in Handler.ServeHTTP is mandatory and is +// specified in docs/webhookd-core.md §"Request Handling Order". Any +// deviation breaks signature verification. +package server + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "strings" + "time" + + "github.com/0xProgress/webhookd/output" + "github.com/0xProgress/webhookd/providers" +) + +// EventWriter is the interface a writer must satisfy to receive +// verified events. Both *output.Writer (JSONL) and *output.PrettyWriter +// (human-readable) satisfy it, so the caller chooses the output mode by +// choosing which writer to pass to NewHandler. +type EventWriter interface { + Write(*output.Event) error +} + +// Handler serves webhook requests for one provider. +// +// The provider name is fixed at construction — it is not extracted from +// the URL. The CLI resolves the name from the subcommand argument and +// passes it here; the mux in server.go mounts this handler at the +// configured --path. The registry lookup inside ServeHTTP is therefore +// a defensive check whose failure indicates a programming error, not a +// routing decision. +type Handler struct { + providerName string + out EventWriter + errOut io.Writer + maxBody int64 +} + +// NewHandler returns a Handler bound to the named provider. +// +// out receives verified events, one call per accepted request. errOut +// receives diagnostics — never event data. In production, out is +// os.Stdout wrapped by output.Writer or output.PrettyWriter, and errOut +// is os.Stderr. +// +// maxBody caps the request body in bytes. It is enforced during the +// read by http.MaxBytesReader, so an oversized body is never fully +// buffered. +func NewHandler(providerName string, out EventWriter, errOut io.Writer, maxBody int64) *Handler { + return &Handler{ + providerName: providerName, + out: out, + errOut: errOut, + maxBody: maxBody, + } +} + +// ServeHTTP implements the mandatory request handling order from +// docs/webhookd-core.md, steps 1 through 14. +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + // Step 1 — method check. + if r.Method != http.MethodPost { + writeError(w, http.StatusMethodNotAllowed, "method not allowed") + return + } + + // Step 2 — content-type check. A charset suffix is permitted, so + // the check is a prefix match rather than an equality check. + if !strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") { + writeError(w, http.StatusUnsupportedMediaType, "unsupported content type") + return + } + + // Steps 3 and 4 — wrap the body in a size-limited reader, then read + // it entirely. The wrapper returns *http.MaxBytesError during the + // read when the limit is exceeded; that error is what drives the + // 413 response. Any other read error is a 500. + r.Body = http.MaxBytesReader(w, r.Body, h.maxBody) + rawBody, err := io.ReadAll(r.Body) + if err != nil { + var maxErr *http.MaxBytesError + if errors.As(err, &maxErr) { + writeError(w, http.StatusRequestEntityTooLarge, "request body too large") + return + } + writeError(w, http.StatusInternalServerError, "internal error") + return + } + + // Step 5 — provider lookup. See the Handler doc comment: this is + // defensive, and failure means the CLI accepted a name it should + // not have. + provider, ok := providers.Get(h.providerName) + if !ok { + writeError(w, http.StatusNotFound, "unknown provider") + return + } + + // Steps 6 and 7 — verify. The error string from the provider is + // written verbatim to stderr in the format specified by + // docs/webhookd-core.md §"Diagnostic formats". Nothing is written + // to the event stream on a failed verification. + if err := provider.Verify(r, rawBody); err != nil { + fmt.Fprintf(h.errOut, "webhookd: %s — %s\n", err.Error(), remoteIP(r)) + writeError(w, http.StatusUnauthorized, "signature verification failed") + return + } + + // Steps 8, 9, and 10 — extract event metadata. These are called + // after verification by design: nothing a provider returns from + // these methods is trusted until the signature has been accepted. + eventType := provider.EventType(r, rawBody) + deliveryID := provider.DeliveryID(r) + eventID := provider.EventID(r, rawBody) + + // Step 11 — validate that the body is syntactically valid JSON. + // This runs after verification, so a hostile body is rejected at + // step 7 before ever reaching here. The check exists to catch the + // case of a valid signature over a body that is not JSON at all, + // which is a provider-integration error, not an attack. + if !json.Valid(rawBody) { + writeError(w, http.StatusInternalServerError, "internal error") + return + } + + // Step 12 — build the normalized Event. Payload holds the raw + // request bytes as json.RawMessage so that key order, duplicate + // keys, and numeric precision survive to the consumer. + event := &output.Event{ + Provider: h.providerName, + Verified: true, + Event: eventType, + ID: eventID, + DeliveryID: deliveryID, + ReceivedAt: time.Now().UTC().Format(time.RFC3339), + Payload: json.RawMessage(rawBody), + } + + // Step 13 — write one JSONL line to the event stream. A write + // failure is a diagnostic, not a data-line failure: nothing was + // written to the event stream, and the client learns the request + // was not recorded. + if err := h.out.Write(event); err != nil { + fmt.Fprintf(h.errOut, "webhookd: write event: %v\n", err) + writeError(w, http.StatusInternalServerError, "internal error") + return + } + + // Step 14 — respond. + writeOK(w) +} + +// remoteIP returns r.RemoteAddr with the port stripped. If the address +// is not in host:port form, the raw value is returned unchanged so the +// diagnostic line is never truncated. +// +// X-Forwarded-For is deliberately not consulted: it is attacker- +// controlled unless webhookd is behind a trusted proxy, and webhookd +// makes no assumption that it is. +func remoteIP(r *http.Request) string { + host, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + return r.RemoteAddr + } + return host +} + +// writeOK writes the 200 response body. +func writeOK(w http.ResponseWriter) { + writeJSON(w, http.StatusOK, map[string]bool{"ok": true}) +} + +// writeError writes a JSON error response. The message is a fixed +// string chosen by the caller; nothing from the request is interpolated +// into it. +func writeError(w http.ResponseWriter, status int, message string) { + writeJSON(w, status, map[string]string{"error": message}) +} + +// writeJSON writes a JSON body with the given status. The values passed +// by this file are always marshalable; a marshal failure is treated as +// a 500 with a plain-text body rather than a panic. +func writeJSON(w http.ResponseWriter, status int, v any) { + body, err := json.Marshal(v) + if err != nil { + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + w.Write(body) +} diff --git a/server/server.go b/server/server.go new file mode 100644 index 0000000..4902121 --- /dev/null +++ b/server/server.go @@ -0,0 +1,198 @@ +// Package server implements the HTTP listener for webhookd. +// +// The listener mounts a webhook handler at a configured path and +// exposes a health endpoint at /health. Request handling itself lives +// in handler.go; this file owns binding, routing, and shutdown. +package server + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "strconv" + "time" +) + +// Options configures a Server. +// +// Every field is set by the caller. Config resolution — flag, +// environment, default — belongs to the config package; server trusts +// what it is given and does not re-derive defaults. +type Options struct { + // Host is the bind address. The default of 127.0.0.1 and the rule + // that 0.0.0.0 must be explicit are enforced by the config layer, + // not here. An empty Host binds to all interfaces, which is a + // caller bug. + Host string + + // Port is the TCP port. Port 0 asks the operating system to choose + // a free port; tests use this to avoid collisions. + Port int + + // Path is the endpoint the webhook Handler is mounted at. It must + // begin with "/" and is matched exactly by the mux. Example: + // "/mock". + Path string + + // Version is the version string. It appears verbatim in the + // startup banner and, with any leading "v" stripped, in the + // /health response. + Version string + + // Timeout is applied as both ReadTimeout and WriteTimeout on the + // underlying http.Server. + Timeout time.Duration + + // Handler is the webhook request handler. In production this is a + // *Handler constructed by NewHandler. + Handler http.Handler + + // ErrOut receives the startup banner. It is never used for event + // data. Production passes os.Stderr. If nil, the banner is + // suppressed. + ErrOut io.Writer +} + +// Server is the HTTP listener. +type Server struct { + httpServer *http.Server + version string + path string + errOut io.Writer +} + +// New builds a Server from opts. +// +// New does not bind a port; call ListenAndServe, or Listen followed by +// Serve. Separating construction from binding lets a test read the +// port the operating system actually assigned before issuing +// requests. +func New(opts Options) *Server { + mux := http.NewServeMux() + mux.Handle(opts.Path, opts.Handler) + mux.Handle("/health", newHealthHandler(normalizeVersion(opts.Version))) + + return &Server{ + httpServer: &http.Server{ + Addr: net.JoinHostPort(opts.Host, strconv.Itoa(opts.Port)), + Handler: mux, + ReadTimeout: opts.Timeout, + WriteTimeout: opts.Timeout, + }, + version: opts.Version, + path: opts.Path, + errOut: opts.ErrOut, + } +} + +// Listen binds the configured address and returns the listener. The +// caller is responsible for calling Serve on the returned listener. +// +// The startup banner is printed to ErrOut once the listener is open, +// so the address it reports is the address actually bound. That may +// differ from the configured port when the caller asked for port 0. +func (s *Server) Listen() (net.Listener, error) { + ln, err := net.Listen("tcp", s.httpServer.Addr) + if err != nil { + return nil, err + } + s.printBanner(ln.Addr().String()) + return ln, nil +} + +// Serve serves requests on ln until the server is shut down. +// +// It always returns a non-nil error; http.ErrServerClosed indicates a +// clean shutdown. +func (s *Server) Serve(ln net.Listener) error { + return s.httpServer.Serve(ln) +} + +// ListenAndServe binds the configured address and serves until shut +// down. It is a convenience for the production path; tests use Listen +// and Serve separately so they can read the ephemeral port. +func (s *Server) ListenAndServe() error { + ln, err := s.Listen() + if err != nil { + return err + } + return s.Serve(ln) +} + +// Shutdown stops the server gracefully, allowing in-flight requests to +// complete or the context to expire. +func (s *Server) Shutdown(ctx context.Context) error { + return s.httpServer.Shutdown(ctx) +} + +// printBanner writes the startup banner to ErrOut. +// +// The format is fixed by docs/webhookd-core.md §"Diagnostic formats": +// +// webhookd — listening on :, endpoint POST +// +// A nil ErrOut suppresses the banner, which is what the test suite +// wants so its captured stderr contains only the diagnostics under +// test. +func (s *Server) printBanner(addr string) { + if s.errOut == nil { + return + } + fmt.Fprintf(s.errOut, "webhookd %s — listening on %s, endpoint POST %s\n", + s.version, addr, s.path) +} + +// healthResponse is the JSON shape returned by GET /health. Field +// order matches the example in docs/webhookd-core.md §"Health +// Endpoint"; encoding/json emits struct fields in declaration order. +type healthResponse struct { + Status string `json:"status"` + Version string `json:"version"` +} + +// newHealthHandler returns the /health handler. +// +// The response body is marshaled once at construction time. The +// handler writes it on every GET without touching any dependency, so +// the endpoint cannot fail — which is what "always returns 200" means +// in the spec. +// +// A non-GET method yields 405 with the standard error body. The spec +// states the endpoint as GET /health; other methods are not a health +// check and are rejected by the same rule that governs every other +// route. +func newHealthHandler(version string) http.Handler { + body, err := json.Marshal(healthResponse{Status: "ok", Version: version}) + if err != nil { + // Unreachable for this struct: its fields are strings and the + // tags are fixed. Falling back to a static body keeps the + // handler from panicking on a request path if this ever + // changes. + body = []byte(`{"status":"error"}`) + } + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + writeError(w, http.StatusMethodNotAllowed, "method not allowed") + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write(body) + }) +} + +// normalizeVersion strips a leading "v" so that the banner can report +// "v0.1.0" while /health reports "0.1.0". The rule is stated in +// docs/webhookd-core.md §"Health Endpoint": the health version is the +// --version string without the "webhookd " prefix and without the +// leading "v". +func normalizeVersion(v string) string { + if len(v) > 0 && v[0] == 'v' { + return v[1:] + } + return v +} diff --git a/server/server_test.go b/server/server_test.go new file mode 100644 index 0000000..71a10c1 --- /dev/null +++ b/server/server_test.go @@ -0,0 +1,474 @@ +// Package server_test exercises the HTTP pipeline end to end through a +// real listener. It asserts on raw bytes: HTTP status, response body, +// stdout, and stderr. It never parses stdout into a struct, because +// doing so would hide unexpected extra output. +package server_test + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "strings" + "sync" + "testing" + "time" + + "github.com/0xProgress/webhookd/output" + "github.com/0xProgress/webhookd/server" + + _ "github.com/0xProgress/webhookd/providers/mock" +) + +// syncBuffer is a bytes.Buffer guarded by a mutex. +// +// The server handler writes to stdout and stderr from the server's +// goroutine while the test reads the same buffers from its own +// goroutine. Without the mutex the race detector correctly reports a +// data race on the buffer. +type syncBuffer struct { + mu sync.Mutex + buf bytes.Buffer +} + +func (b *syncBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Write(p) +} + +func (b *syncBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.String() +} + +func (b *syncBuffer) Len() int { + b.mu.Lock() + defer b.mu.Unlock() + return b.buf.Len() +} + +// testServer is a running webhookd server plus the buffers its handler +// writes to. +type testServer struct { + URL string + Path string + Stdout *syncBuffer + Stderr *syncBuffer +} + +type testServerConfig struct { + providerName string + maxBody int64 + version string +} + +type testServerOption func(*testServerConfig) + +func withProviderName(name string) testServerOption { + return func(c *testServerConfig) { c.providerName = name } +} + +func withMaxBody(n int64) testServerOption { + return func(c *testServerConfig) { c.maxBody = n } +} + +// newTestServer starts a Server on an ephemeral port and returns it +// along with the buffers the handler writes to. The server is shut +// down when the test ends. +func newTestServer(t *testing.T, opts ...testServerOption) *testServer { + t.Helper() + + cfg := testServerConfig{ + providerName: "mock", + maxBody: 2 * 1024 * 1024, + version: "v0.1.0", + } + for _, o := range opts { + o(&cfg) + } + + stdout := &syncBuffer{} + stderr := &syncBuffer{} + + handler := server.NewHandler( + cfg.providerName, + output.NewWriter(stdout), + stderr, + cfg.maxBody, + ) + + path := "/" + cfg.providerName + + httpSrv := server.New(server.Options{ + Host: "127.0.0.1", + Port: 0, + Path: path, + Version: cfg.version, + Timeout: 10 * time.Second, + Handler: handler, + ErrOut: nil, // suppress the banner so stderr assertions are exact + }) + + ln, err := httpSrv.Listen() + if err != nil { + t.Fatalf("Listen: %v", err) + } + + go func() { + _ = httpSrv.Serve(ln) + }() + + t.Cleanup(func() { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + _ = httpSrv.Shutdown(ctx) + }) + + return &testServer{ + URL: "http://" + ln.Addr().String(), + Path: path, + Stdout: stdout, + Stderr: stderr, + } +} + +// postJSON sends a POST with Content-Type application/json and the +// given signature and body. An empty sig omits the X-Mock-Signature +// header entirely. +func postJSON(t *testing.T, ts *testServer, sig, body string) *http.Response { + t.Helper() + req, err := http.NewRequest(http.MethodPost, ts.URL+ts.Path, strings.NewReader(body)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Content-Type", "application/json") + if sig != "" { + req.Header.Set("X-Mock-Signature", sig) + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + return resp +} + +// drainBody reads and closes the response body. +func drainBody(t *testing.T, resp *http.Response) string { + t.Helper() + defer resp.Body.Close() + b, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + return string(b) +} + +// The em dash character used in the failed-verification diagnostic. +// It is the character the spec fixes, U+2014. +const emDash = "\u2014" + +func TestVerified_JSONLOutput(t *testing.T) { + ts := newTestServer(t) + const body = `{"type":"order.created","id":"evt_abc123","data":{"amount":4200}}` + + resp := postJSON(t, ts, "valid", body) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + if got := drainBody(t, resp); got != `{"ok":true}` { + t.Fatalf("response body = %q, want %q", got, `{"ok":true}`) + } + + if ts.Stderr.Len() != 0 { + t.Fatalf("stderr not empty: %q", ts.Stderr.String()) + } + + stdout := ts.Stdout.String() + if n := strings.Count(stdout, "\n"); n != 1 { + t.Fatalf("stdout has %d newlines, want exactly 1:\n%q", n, stdout) + } + + prefix := `{"provider":"mock","verified":true,"event":"order.created","id":"evt_abc123","delivery_id":"","received_at":"` + if !strings.HasPrefix(stdout, prefix) { + t.Fatalf("stdout prefix mismatch:\ngot: %q\nwant: %q...", stdout, prefix) + } + suffix := `","payload":` + body + "}\n" + if !strings.HasSuffix(stdout, suffix) { + t.Fatalf("stdout suffix mismatch:\ngot: %q\nwant: ...%q", stdout, suffix) + } + + // Extract the received_at value between the two known delimiters + // and check it is RFC3339 UTC. + rest := stdout[len(prefix):] + idx := strings.Index(rest, `","payload":`) + if idx < 0 { + t.Fatal("could not find payload delimiter in stdout") + } + received := rest[:idx] + if !strings.HasSuffix(received, "Z") { + t.Fatalf("received_at %q does not end in Z", received) + } + if _, err := time.Parse(time.RFC3339, received); err != nil { + t.Fatalf("received_at %q is not RFC3339: %v", received, err) + } +} + +func TestMethodNotAllowed(t *testing.T) { + ts := newTestServer(t) + resp, err := http.Get(ts.URL + ts.Path) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusMethodNotAllowed { + t.Fatalf("status = %d, want 405", resp.StatusCode) + } + if got := drainBody(t, resp); got != `{"error":"method not allowed"}` { + t.Fatalf("response body = %q", got) + } + if ts.Stdout.Len() != 0 { + t.Fatalf("stdout not empty: %q", ts.Stdout.String()) + } +} + +func TestUnsupportedContentType(t *testing.T) { + ts := newTestServer(t) + req, err := http.NewRequest(http.MethodPost, ts.URL+ts.Path, strings.NewReader(`{}`)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("X-Mock-Signature", "valid") + // Deliberately no Content-Type header. + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusUnsupportedMediaType { + t.Fatalf("status = %d, want 415", resp.StatusCode) + } + if got := drainBody(t, resp); got != `{"error":"unsupported content type"}` { + t.Fatalf("response body = %q", got) + } + if ts.Stdout.Len() != 0 { + t.Fatalf("stdout not empty: %q", ts.Stdout.String()) + } +} + +func TestContentTypeWithCharset(t *testing.T) { + ts := newTestServer(t) + req, err := http.NewRequest(http.MethodPost, ts.URL+ts.Path, strings.NewReader(`{"type":"t"}`)) + if err != nil { + t.Fatal(err) + } + req.Header.Set("Content-Type", "application/json; charset=utf-8") + req.Header.Set("X-Mock-Signature", "valid") + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + _ = drainBody(t, resp) +} + +func TestBodyTooLarge(t *testing.T) { + ts := newTestServer(t, withMaxBody(1024)) + // A 2KB body against a 1KB limit. + body := `{"pad":"` + strings.Repeat("a", 2048) + `"}` + resp := postJSON(t, ts, "valid", body) + if resp.StatusCode != http.StatusRequestEntityTooLarge { + t.Fatalf("status = %d, want 413", resp.StatusCode) + } + if got := drainBody(t, resp); got != `{"error":"request body too large"}` { + t.Fatalf("response body = %q", got) + } + if ts.Stdout.Len() != 0 { + t.Fatalf("stdout not empty: %q", ts.Stdout.String()) + } +} + +func TestMissingSignature(t *testing.T) { + ts := newTestServer(t) + resp := postJSON(t, ts, "", `{"type":"test"}`) + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401", resp.StatusCode) + } + if got := drainBody(t, resp); got != `{"error":"signature verification failed"}` { + t.Fatalf("response body = %q", got) + } + if ts.Stdout.Len() != 0 { + t.Fatalf("stdout not empty: %q", ts.Stdout.String()) + } + want := "webhookd: mock: missing X-Mock-Signature header " + emDash + " 127.0.0.1\n" + if got := ts.Stderr.String(); got != want { + t.Fatalf("stderr =\n %q\nwant:\n %q", got, want) + } +} + +func TestWrongSignature(t *testing.T) { + ts := newTestServer(t) + resp := postJSON(t, ts, "invalid", `{"type":"test"}`) + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401", resp.StatusCode) + } + if got := drainBody(t, resp); got != `{"error":"signature verification failed"}` { + t.Fatalf("response body = %q", got) + } + if ts.Stdout.Len() != 0 { + t.Fatalf("stdout not empty: %q", ts.Stdout.String()) + } + want := "webhookd: mock: signature mismatch " + emDash + " 127.0.0.1\n" + if got := ts.Stderr.String(); got != want { + t.Fatalf("stderr =\n %q\nwant:\n %q", got, want) + } +} + +func TestMalformedJSON(t *testing.T) { + ts := newTestServer(t) + resp := postJSON(t, ts, "valid", `{`) + if resp.StatusCode != http.StatusInternalServerError { + t.Fatalf("status = %d, want 500", resp.StatusCode) + } + if got := drainBody(t, resp); got != `{"error":"internal error"}` { + t.Fatalf("response body = %q", got) + } + if ts.Stdout.Len() != 0 { + t.Fatalf("stdout not empty: %q", ts.Stdout.String()) + } +} + +func TestUnknownProvider(t *testing.T) { + ts := newTestServer(t, withProviderName("nonexistent")) + resp := postJSON(t, ts, "valid", `{"type":"test"}`) + if resp.StatusCode != http.StatusNotFound { + t.Fatalf("status = %d, want 404", resp.StatusCode) + } + if got := drainBody(t, resp); got != `{"error":"unknown provider"}` { + t.Fatalf("response body = %q", got) + } + if ts.Stdout.Len() != 0 { + t.Fatalf("stdout not empty: %q", ts.Stdout.String()) + } +} + +func TestPayloadPreserved(t *testing.T) { + ts := newTestServer(t) + // Key order is not alphabetical; a number exceeds 2^53; a key is + // duplicated. All three survive only if the payload is emitted as + // the original request bytes rather than re-serialized from a map. + const body = `{"z":1,"a":2,"big":9007199254740993,"dup":1,"dup":2}` + resp := postJSON(t, ts, "valid", body) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + _ = drainBody(t, resp) + + stdout := ts.Stdout.String() + want := `"payload":` + body + "}" + if !strings.Contains(stdout, want) { + t.Fatalf("payload not preserved:\ngot: %s\nwant to contain: %s", stdout, want) + } +} + +func TestPayloadCompacted(t *testing.T) { + ts := newTestServer(t) + // Whitespace outside strings is dropped by the JSONL encoder; key + // order is preserved. The output payload must be the compact form + // of the request body. + const body = "{\n \"z\": 1,\n \"a\": 2\n}" + resp := postJSON(t, ts, "valid", body) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + _ = drainBody(t, resp) + + stdout := ts.Stdout.String() + want := `"payload":{"z":1,"a":2}}` + if !strings.Contains(stdout, want) { + t.Fatalf("payload not compacted:\ngot: %s\nwant to contain: %s", stdout, want) + } + if strings.Contains(stdout, "\n ") { + t.Fatalf("stdout contains raw indentation from the request body:\n%s", stdout) + } +} + +func TestPayloadHTMLNotEscaped(t *testing.T) { + ts := newTestServer(t) + // encoding/json escapes <, >, and & by default. output.Writer + // disables that so the payload survives with the exact bytes the + // provider sent. + const body = `{"msg":""}` + resp := postJSON(t, ts, "valid", body) + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + _ = drainBody(t, resp) + + stdout := ts.Stdout.String() + if !strings.Contains(stdout, `"payload":{"msg":""}}`) { + t.Fatalf("payload was HTML-escaped by the encoder:\n%s", stdout) + } +} + +func TestHealth(t *testing.T) { + ts := newTestServer(t) + resp, err := http.Get(ts.URL + "/health") + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want 200", resp.StatusCode) + } + want := `{"status":"ok","version":"0.1.0"}` + if got := drainBody(t, resp); got != want { + t.Fatalf("response body = %q, want %q", got, want) + } + if ts.Stdout.Len() != 0 { + t.Fatalf("stdout not empty: %q", ts.Stdout.String()) + } +} + +func TestHealthMethodNotAllowed(t *testing.T) { + ts := newTestServer(t) + resp, err := http.Post(ts.URL+"/health", "application/json", nil) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusMethodNotAllowed { + t.Fatalf("status = %d, want 405", resp.StatusCode) + } + if got := drainBody(t, resp); got != `{"error":"method not allowed"}` { + t.Fatalf("response body = %q", got) + } +} + +func TestStartupBanner(t *testing.T) { + stdout := &syncBuffer{} + stderr := &syncBuffer{} + + handler := server.NewHandler("mock", output.NewWriter(stdout), stderr, 2*1024*1024) + httpSrv := server.New(server.Options{ + Host: "127.0.0.1", + Port: 0, + Path: "/mock", + Version: "v0.1.0", + Timeout: 10 * time.Second, + Handler: handler, + ErrOut: stderr, + }) + + ln, err := httpSrv.Listen() + if err != nil { + t.Fatalf("Listen: %v", err) + } + defer ln.Close() + + want := fmt.Sprintf("webhookd v0.1.0 %s listening on %s, endpoint POST /mock\n", + emDash, ln.Addr().String()) + if got := stderr.String(); got != want { + t.Fatalf("banner =\n %q\nwant:\n %q", got, want) + } +}