diff --git a/.github/scripts/check_codecov_coverage.sh b/.github/scripts/check_codecov_coverage.sh new file mode 100755 index 0000000..c374027 --- /dev/null +++ b/.github/scripts/check_codecov_coverage.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Guards the artifact fan-in contract before the single Codecov upload. + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +ARTIFACTS_DIR="${1:-$ROOT_DIR/coverage-artifacts}" +ROOT_MODULE="$(awk '$1 == "module" { print $2; exit }' "$ROOT_DIR/go.mod")" + +fail() { + echo "coverage artifact guard: $*" >&2 + exit 1 +} + +integration_backends=(null sync workerpool redis mysql postgres sqlite nats sqs rabbitmq) +driver_modules=(mysqlqueue natsqueue postgresqueue rabbitmqqueue redisqueue sqlitequeue sqlqueuecore sqsqueue) + +expected_profiles=("$ARTIFACTS_DIR/coverage-unit/coverage-unit.out") +for backend in "${integration_backends[@]}"; do + expected_profiles+=("$ARTIFACTS_DIR/coverage-integration-$backend/coverage-integration-$backend.out") +done + +[[ -d "$ARTIFACTS_DIR" ]] || fail "artifact directory does not exist: $ARTIFACTS_DIR" +for profile in "${expected_profiles[@]}"; do + [[ -s "$profile" ]] || fail "expected profile is missing or empty: $profile" +done + +discovered_profile_count="$(find "$ARTIFACTS_DIR" -type f -name '*.out' -print | awk 'END { print NR }')" +[[ "$discovered_profile_count" -eq "${#expected_profiles[@]}" ]] || \ + fail "found $discovered_profile_count profiles, expected ${#expected_profiles[@]}" + +validate_profile() { + local profile="$1" + [[ "$(head -n 1 "$profile")" == "mode: atomic" ]] || fail "profile is not atomic: $profile" + [[ "$(grep -c '^mode:' "$profile")" -eq 1 ]] || fail "profile has multiple mode headers: $profile" + + awk -v prefix="$ROOT_MODULE/" ' + NR == 1 { next } + NF != 3 || $2 !~ /^[0-9]+$/ || $3 !~ /^[0-9]+$/ { + print "coverage artifact guard: malformed coverage record in " FILENAME ": " $0 > "/dev/stderr" + exit 1 + } + index($1, prefix) != 1 { + print "coverage artifact guard: non-repository profile path in " FILENAME ": " $1 > "/dev/stderr" + exit 1 + } + seen[$1]++ { + print "coverage artifact guard: duplicate source range in " FILENAME ": " $1 > "/dev/stderr" + exit 1 + } + ' "$profile" +} + +for profile in "${expected_profiles[@]}"; do + validate_profile "$profile" +done + +unit_profile="$ARTIFACTS_DIR/coverage-unit/coverage-unit.out" +manifest="$ARTIFACTS_DIR/coverage-unit/coverage-unit-modules.tsv" +[[ -s "$manifest" ]] || fail "unit module manifest is missing or empty: $manifest" + +expected_manifest="$({ + printf '.\t%s\n' "$ROOT_MODULE" + printf 'docs\t%s/docs\n' "$ROOT_MODULE" + for driver in "${driver_modules[@]}"; do + printf 'driver/%s\t%s/driver/%s\n' "$driver" "$ROOT_MODULE" "$driver" + done + printf 'examples\t%s/examples\n' "$ROOT_MODULE" + printf 'integration\t%s/integration\n' "$ROOT_MODULE" +} | LC_ALL=C sort)" +actual_manifest="$(awk -F '\t' '!/^#/ && NF { print $1 "\t" $2 }' "$manifest" | LC_ALL=C sort)" +if [[ "$actual_manifest" != "$expected_manifest" ]]; then + diff -u <(printf '%s\n' "$expected_manifest") <(printf '%s\n' "$actual_manifest") >&2 || true + fail "unit module manifest does not match the buildable module inventory" +fi + +require_path() { + local profile="$1" + local pattern="$2" + local label="$3" + grep -Eq "$pattern" "$profile" || fail "$label is absent from $profile" +} + +require_covered_path() { + local pattern="$1" + local label="$2" + shift 2 + awk -v pattern="$pattern" ' + NR > 1 && $1 ~ pattern && ($3 + 0) > 0 { found = 1; exit } + END { exit !found } + ' "$@" || fail "$label has no covered source range" +} + +require_covered_function() { + local profile="$1" + local relative_file="$2" + local function_name="$3" + local label="$4" + local function_profile + function_profile="$ARTIFACTS_DIR/functions-$(basename "$profile").txt" + + if ! GOWORK="$ROOT_DIR/go.work" go tool cover -func="$profile" >"$function_profile"; then + fail "could not summarize functions in $profile" + fi + awk -v path="$ROOT_MODULE/$relative_file:" -v function_name="$function_name" ' + index($1, path) == 1 && $2 == function_name { + percent = $3 + sub(/%$/, "", percent) + if ((percent + 0) > 0) { + found = 1 + } + } + END { exit !found } + ' "$function_profile" || fail "$label did not execute" + rm -f "$function_profile" +} + +require_path "$unit_profile" "^${ROOT_MODULE//./[.]}/[^/]+[.]go:" "root-module source" +for driver in "${driver_modules[@]}"; do + require_path "$unit_profile" "^${ROOT_MODULE//./[.]}/driver/$driver/.*[.]go:" "driver/$driver source" +done +require_path "$unit_profile" "^${ROOT_MODULE//./[.]}/integration/.*[.]go:" "integration-module source" +require_path "$unit_profile" "^${ROOT_MODULE//./[.]}/docs/readme/testcounts/.*[.]go:" "tagged documentation tooling source" + +require_covered_path "^${ROOT_MODULE//./[.]}/queue[.]go:" "representative root source" "${expected_profiles[@]}" +require_covered_path "^${ROOT_MODULE//./[.]}/bus/testhooks_integration[.]go:" "root integration-tagged bus fixture" "$unit_profile" +require_covered_function "$unit_profile" "docs/readme/testcounts/main.go" "loadIntegrationCountManifest" "generated test-count evidence validation" +for driver in "${driver_modules[@]}"; do + require_covered_path "^${ROOT_MODULE//./[.]}/driver/$driver/.*[.]go:" "driver/$driver source" "${expected_profiles[@]}" +done + +for backend in "${integration_backends[@]}"; do + profile="$ARTIFACTS_DIR/coverage-integration-$backend/coverage-integration-$backend.out" + dialect_evidence_file="" + dialect_evidence_function="" + case "$backend" in + null) + evidence_file="queue_null.go" + evidence_function="Dispatch" + ;; + sync) + evidence_file="queue_local.go" + evidence_function="enqueueNow" + ;; + workerpool) + evidence_file="queue_local.go" + evidence_function="worker" + ;; + redis) + evidence_file="driver/redisqueue/worker_redis_impl.go" + evidence_function="StartWorkers" + ;; + mysql) + evidence_file="driver/sqlqueuecore/queue_database_impl.go" + evidence_function="workerLoop" + dialect_evidence_file="driver/mysqlqueue/mysqlqueue.go" + dialect_evidence_function="NewWithConfig" + ;; + postgres) + evidence_file="driver/sqlqueuecore/queue_database_impl.go" + evidence_function="workerLoop" + dialect_evidence_file="driver/postgresqueue/postgresqueue.go" + dialect_evidence_function="NewWithConfig" + ;; + sqlite) + evidence_file="driver/sqlqueuecore/queue_database_impl.go" + evidence_function="workerLoop" + dialect_evidence_file="driver/sqlitequeue/sqlitequeue.go" + dialect_evidence_function="New" + ;; + nats) + evidence_file="driver/natsqueue/worker_nats_impl.go" + evidence_function="processMessage" + ;; + sqs) + evidence_file="driver/sqsqueue/worker_sqs_impl.go" + evidence_function="process" + ;; + rabbitmq) + evidence_file="driver/rabbitmqqueue/worker_rabbitmq_impl.go" + evidence_function="processDelivery" + ;; + *) fail "coverage evidence is not defined for backend $backend" ;; + esac + require_covered_function "$profile" "$evidence_file" "$evidence_function" "$backend integration backend" + if [[ -n "$dialect_evidence_file" ]]; then + require_covered_function "$profile" "$dialect_evidence_file" "$dialect_evidence_function" "$backend SQL dialect" + fi +done + +echo "coverage artifact guard: 1 multi-module unit profile and ${#integration_backends[@]} backend profiles verified" diff --git a/.github/scripts/test_wait_codecov_report.sh b/.github/scripts/test_wait_codecov_report.sh new file mode 100755 index 0000000..2f49587 --- /dev/null +++ b/.github/scripts/test_wait_codecov_report.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash +set -euo pipefail + +head_sha="2222222222222222222222222222222222222222" +base_sha="1111111111111111111111111111111111111111" + +# fake_curl keeps the guard tests deterministic while preserving the same +# process boundary and PATH lookup used by the real curl executable. +fake_curl() { + local url="${!#}" + + if [[ "${CODECOV_TEST_SCENARIO:-}" == "malformed" ]]; then + echo "temporarily not JSON" + return + fi + + case "$url" in + */uploads/*) + fake_uploads "$url" + ;; + */compare/*) + fake_comparison + ;; + */commits/*) + fake_commit + ;; + *) + return 22 + ;; + esac +} + +# fake_uploads emits single-run, rerun, and paginated upload inventories. +fake_uploads() { + local url="$1" + + case "$CODECOV_TEST_SCENARIO" in + aggregate|stale) + echo '{"count":2,"results":[{"name":"prior-run","state_name":"MERGED","totals":{"files":10,"lines":100,"hits":80,"misses":10,"partials":10,"coverage":80}},{"name":"current-run","state_name":"MERGED","totals":{"files":10,"lines":100,"hits":85,"misses":10,"partials":5,"coverage":85}}]}' + ;; + pagination) + if [[ "$url" == *"page=1&"* ]]; then + jq -cn '{count:151,results:[range(0;150) | {name:("old-" + tostring),state_name:"MERGED",totals:{files:1,lines:1,hits:1,misses:0,partials:0,coverage:100}}]}' + else + echo '{"count":151,"results":[{"name":"page-two-run","state_name":"MERGED","totals":{"files":1,"lines":1,"hits":1,"misses":0,"partials":0,"coverage":100}}]}' + fi + ;; + *) + echo '{"count":1,"results":[{"name":"single-run","state_name":"MERGED","totals":{"files":10,"lines":100,"hits":80,"misses":10,"partials":10,"coverage":80}}]}' + ;; + esac +} + +# fake_commit distinguishes a current aggregate from its individual uploads. +fake_commit() { + case "$CODECOV_TEST_SCENARIO" in + aggregate|stale) + echo "{\"commitid\":\"$head_sha\",\"state\":\"complete\",\"totals\":{\"files\":10,\"lines\":100,\"hits\":90,\"misses\":5,\"partials\":5,\"coverage\":90,\"sessions\":2}}" + ;; + pagination) + echo "{\"commitid\":\"$head_sha\",\"state\":\"complete\",\"totals\":{\"files\":1,\"lines\":1,\"hits\":1,\"misses\":0,\"partials\":0,\"coverage\":100,\"sessions\":151}}" + ;; + *) + echo "{\"commitid\":\"$head_sha\",\"state\":\"complete\",\"totals\":{\"files\":10,\"lines\":100,\"hits\":80,\"misses\":10,\"partials\":10,\"coverage\":80,\"sessions\":1}}" + ;; + esac +} + +# fake_comparison exposes both complete and intentionally stale comparison data. +fake_comparison() { + case "$CODECOV_TEST_SCENARIO" in + aggregate) + echo "{\"base_commit\":\"$base_sha\",\"head_commit\":\"$head_sha\",\"totals\":{\"head\":{\"files\":10,\"lines\":100,\"hits\":90,\"misses\":5,\"partials\":5,\"coverage\":90,\"sessions\":2},\"patch\":{\"files\":2,\"lines\":5,\"hits\":5,\"misses\":0,\"partials\":0,\"coverage\":100}}}" + ;; + stale) + echo "{\"base_commit\":\"$base_sha\",\"head_commit\":\"$head_sha\",\"totals\":{\"head\":{\"files\":10,\"lines\":100,\"hits\":80,\"misses\":10,\"partials\":10,\"coverage\":80,\"sessions\":1},\"patch\":{\"files\":2,\"lines\":5,\"hits\":4,\"misses\":1,\"partials\":0,\"coverage\":80}}}" + ;; + null-patch) + echo "{\"base_commit\":\"$base_sha\",\"head_commit\":\"$head_sha\",\"totals\":{\"head\":{\"files\":10,\"lines\":100,\"hits\":80,\"misses\":10,\"partials\":10,\"coverage\":80,\"sessions\":1},\"patch\":null}}" + ;; + pagination) + echo "{\"base_commit\":\"$base_sha\",\"head_commit\":\"$head_sha\",\"totals\":{\"head\":{\"files\":1,\"lines\":1,\"hits\":1,\"misses\":0,\"partials\":0,\"coverage\":100,\"sessions\":151},\"patch\":{\"files\":1,\"lines\":1,\"hits\":1,\"misses\":0,\"partials\":0,\"coverage\":100}}}" + ;; + *) + echo "{\"base_commit\":\"$base_sha\",\"head_commit\":\"$head_sha\",\"totals\":{\"head\":{\"files\":10,\"lines\":100,\"hits\":80,\"misses\":10,\"partials\":10,\"coverage\":80,\"sessions\":1},\"patch\":{\"files\":2,\"lines\":5,\"hits\":4,\"misses\":1,\"partials\":0,\"coverage\":80}}}" + ;; + esac +} + +if [[ "${0##*/}" == "curl" ]]; then + fake_curl "$@" + exit +fi + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +guard="$script_dir/wait_codecov_report.sh" +fake_bin="$(mktemp -d "${TMPDIR:-/tmp}/queue-codecov-guard.XXXXXX")" +trap 'rm -rf -- "$fake_bin"' EXIT +ln -s "$script_dir/test_wait_codecov_report.sh" "$fake_bin/curl" + +# run_guard isolates the API scenario while keeping guard timeouts short. +run_guard() { + local scenario="$1" + local upload_name="$2" + local requested_base="${3-$base_sha}" + + PATH="$fake_bin:$PATH" \ + CODECOV_API_BASE="https://mock.invalid" \ + CODECOV_BASE_SHA="$requested_base" \ + CODECOV_SHA="$head_sha" \ + CODECOV_TEST_SCENARIO="$scenario" \ + CODECOV_UPLOAD_NAME="$upload_name" \ + CODECOV_WAIT_POLL_SECONDS=1 \ + CODECOV_WAIT_TIMEOUT_SECONDS=1 \ + "$guard" +} + +# require_success fails with the captured guard diagnostics when acceptance regresses. +require_success() { + local scenario="$1" + local upload_name="$2" + local requested_base="${3-$base_sha}" + local expected="${4:-exact commit report complete}" + local output + + if ! output="$(run_guard "$scenario" "$upload_name" "$requested_base" 2>&1)"; then + echo "scenario $scenario unexpectedly failed:" >&2 + echo "$output" >&2 + exit 1 + fi + if [[ "$output" != *"$expected"* ]]; then + echo "scenario $scenario omitted expected output: $expected" >&2 + echo "$output" >&2 + exit 1 + fi +} + +# require_failure protects against accepting stale or structurally incomplete reports. +require_failure() { + local scenario="$1" + local upload_name="$2" + local expected="$3" + local output + + if output="$(run_guard "$scenario" "$upload_name" 2>&1)"; then + echo "scenario $scenario unexpectedly passed:" >&2 + echo "$output" >&2 + exit 1 + fi + if [[ "$output" != *"timed out after 1s"* || "$output" != *"$expected"* ]]; then + echo "scenario $scenario failed for the wrong reason:" >&2 + echo "$output" >&2 + exit 1 + fi +} + +require_success single single-run "$base_sha" "project 80%, patch 80%" +require_success aggregate current-run "$base_sha" "project 90%, patch 100%" +require_success pagination page-two-run "$base_sha" "project 100%, patch 100%" +require_success single single-run "" "project 80%, patch n/a%" +require_failure stale current-run "comparison: ready" +require_failure null-patch single-run "comparison: totals incomplete" +require_failure malformed single-run "upload: API unavailable or invalid" + +echo "Codecov report guard tests passed" diff --git a/.github/scripts/wait_codecov_report.sh b/.github/scripts/wait_codecov_report.sh new file mode 100755 index 0000000..0b693cb --- /dev/null +++ b/.github/scripts/wait_codecov_report.sh @@ -0,0 +1,240 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Codecov accepts uploads asynchronously, so a successful uploader process does +# not prove that the report for this workflow run is ready. The mutable pull +# request comment remains an eventually consistent presentation of this data. + +fail() { + echo "Codecov report guard: $*" >&2 + exit 1 +} + +require_sha() { + local label="$1" + local value="$2" + [[ "$value" =~ ^[0-9a-f]{40}$ ]] || fail "$label must be a full lowercase commit SHA" +} + +command -v curl >/dev/null 2>&1 || fail "curl is required" +command -v jq >/dev/null 2>&1 || fail "jq is required" + +sha="${CODECOV_SHA:-}" +base_sha="${CODECOV_BASE_SHA:-}" +upload_name="${CODECOV_UPLOAD_NAME:-}" +repository="${GITHUB_REPOSITORY:-goforj/queue}" +timeout_seconds="${CODECOV_WAIT_TIMEOUT_SECONDS:-180}" +poll_seconds="${CODECOV_WAIT_POLL_SECONDS:-5}" + +require_sha "CODECOV_SHA" "$sha" +[[ -n "$upload_name" ]] || fail "CODECOV_UPLOAD_NAME is required" +[[ "$repository" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || \ + fail "GITHUB_REPOSITORY must have the form owner/repository" +[[ "$timeout_seconds" =~ ^[1-9][0-9]*$ ]] || \ + fail "CODECOV_WAIT_TIMEOUT_SECONDS must be a positive integer" +[[ "$poll_seconds" =~ ^[1-9][0-9]*$ ]] || \ + fail "CODECOV_WAIT_POLL_SECONDS must be a positive integer" + +if [[ -n "$base_sha" ]]; then + require_sha "CODECOV_BASE_SHA" "$base_sha" +fi + +owner="${repository%%/*}" +repo="${repository#*/}" +api_base="${CODECOV_API_BASE:-https://api.codecov.io/api/v2/github/$owner/repos/$repo}" +api_base="${api_base%/}" +deadline=$((SECONDS + timeout_seconds)) + +upload_ready=false +commit_ready=false +comparison_ready=false +[[ -z "$base_sha" ]] && comparison_ready=true + +upload_state="not found" +commit_state="not found" +comparison_state="not requested" +merged_sessions=0 +commit_json="" +comparison_json="" + +fetch_json() { + local url="$1" + curl --fail --silent --show-error \ + --connect-timeout 10 \ + --max-time 20 \ + "$url" 2>/dev/null | jq -ce '.' 2>/dev/null +} + +fetch_uploads() { + local page_size=150 + local results='[]' + local page_json + local page_results + local expected_count=-1 + local result_count + local page=1 + local page_count + + while :; do + page_json="$(fetch_json \ + "$api_base/commits/$sha/uploads/?page=$page&page_size=$page_size" + )" || return 1 + jq -e ' + (.results | type) == "array" and + (.count | type) == "number" and + (.count >= 0) and + ((.count % 1) == 0) + ' >/dev/null <<<"$page_json" || return 1 + + page_count="$(jq '.count' <<<"$page_json")" || return 1 + if (( expected_count < 0 )); then + expected_count="$page_count" + elif (( page_count != expected_count )); then + return 1 + fi + + page_results="$(jq -c '.results' <<<"$page_json")" || return 1 + results="$(jq -cn \ + --argjson previous "$results" \ + --argjson current "$page_results" \ + '$previous + $current' + )" || return 1 + + result_count="$(jq 'length' <<<"$results")" || return 1 + (( result_count >= expected_count )) && break + (( page < 20 )) || return 1 + page=$((page + 1)) + done + + result_count="$(jq 'length' <<<"$results")" || return 1 + (( result_count == expected_count )) || return 1 + jq -cn --argjson results "$results" '{results: $results}' +} + +echo "Codecov report guard: waiting for upload $upload_name at $sha" + +while (( SECONDS < deadline )); do + previous_upload_ready="$upload_ready" + upload_ready=false + if uploads_json="$(fetch_uploads)"; then + if jq -e '(.results | type) == "array"' >/dev/null <<<"$uploads_json"; then + upload_state="$(jq -r --arg name "$upload_name" ' + [.results[]? | select(.name == $name) | (.state_name // .state // "unknown")] | + last // "not found" + ' <<<"$uploads_json")" + if jq -e --arg name "$upload_name" ' + any(.results[]?; + .name == $name and + ((.state_name // "") == "MERGED" or (.state // "") == "merged") and + ((.totals | type) == "object") + ) + ' >/dev/null <<<"$uploads_json"; then + merged_sessions="$(jq ' + [.results[]? | + select( + ((.state_name // "") == "MERGED" or (.state // "") == "merged") and + ((.totals | type) == "object") + ) + ] | length + ' <<<"$uploads_json")" + upload_ready=true + if [[ "$previous_upload_ready" != true ]]; then + echo "Codecov report guard: upload merged" + fi + fi + fi + else + upload_state="API unavailable or invalid" + fi + + commit_ready=false + if [[ "$upload_ready" == true ]]; then + if commit_json="$(fetch_json "$api_base/commits/$sha/")" && \ + jq -e 'type == "object" and (.totals | type) == "object"' \ + >/dev/null <<<"$commit_json"; then + commit_state="$(jq -r --argjson expected "$merged_sessions" ' + "\(.state // "unknown"), \(.totals.sessions // 0)/\($expected) sessions" + ' <<<"$commit_json")" + if jq -e --arg sha "$sha" --argjson expected "$merged_sessions" ' + def complete_coverage: + [.files, .lines, .hits, .misses, .partials, .coverage] | + all(.[]; type == "number"); + .commitid == $sha and + .state == "complete" and + (.totals | complete_coverage) and + ((.totals.sessions // 0) >= $expected) and + ($expected >= 1) + ' >/dev/null <<<"$commit_json"; then + commit_ready=true + fi + else + commit_state="API unavailable or invalid" + fi + fi + + comparison_ready=false + [[ -z "$base_sha" ]] && comparison_ready=true + if [[ "$commit_ready" == true && -n "$base_sha" ]]; then + if comparison_json="$(fetch_json "$api_base/compare/?base=$base_sha&head=$sha")" && \ + jq -e 'type == "object" and (.totals | type) == "object"' \ + >/dev/null <<<"$comparison_json"; then + comparison_state="$(jq -r ' + if (.totals.patch | type) == "object" + then "ready" + else "totals incomplete" + end + ' <<<"$comparison_json")" + if jq -e \ + --arg base "$base_sha" \ + --arg head "$sha" \ + --argjson expected "$merged_sessions" \ + --argjson commit "$commit_json" ' + def coverage_totals: + [.files, .lines, .hits, .misses, .partials, .coverage]; + def complete_patch: + [.files, .lines, .hits, .misses, .partials, .coverage] | + all(.[]; type == "number"); + .base_commit == $base and + .head_commit == $head and + (.totals | type) == "object" and + ((.totals.patch | type) == "object") and + (.totals.patch | complete_patch) and + ((.totals.head.sessions // 0) >= $expected) and + ((.totals.head.sessions // 0) == ($commit.totals.sessions // -1)) and + (.totals.head | coverage_totals) == ($commit.totals | coverage_totals) + ' >/dev/null <<<"$comparison_json"; then + comparison_ready=true + fi + else + comparison_state="API unavailable or invalid" + fi + fi + + if [[ "$upload_ready" == true && "$commit_ready" == true && "$comparison_ready" == true ]]; then + echo "Codecov report guard: exact commit report complete" + if [[ -n "$base_sha" ]]; then + echo "Codecov report guard: exact pull request comparison ready" + fi + project_coverage="$(jq -r '.totals.coverage // "n/a"' <<<"$commit_json")" + patch_coverage="n/a" + if [[ -n "$base_sha" ]]; then + patch_coverage="$(jq -r '.totals.patch.coverage // "n/a"' <<<"$comparison_json")" + fi + + echo "Codecov report guard: project ${project_coverage}%, patch ${patch_coverage}%" + if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then + { + echo "### Codecov report" + echo + echo "| Commit | Project | Patch |" + echo "| --- | ---: | ---: |" + echo "| \`$sha\` | ${project_coverage}% | ${patch_coverage}% |" + } >>"$GITHUB_STEP_SUMMARY" + fi + exit 0 + fi + + sleep "$poll_seconds" +done + +fail "timed out after ${timeout_seconds}s (upload: $upload_state; commit: $commit_state; comparison: $comparison_state)" diff --git a/.github/workflows/soak.yml b/.github/workflows/soak.yml index 6510bec..f620802 100644 --- a/.github/workflows/soak.yml +++ b/.github/workflows/soak.yml @@ -249,7 +249,6 @@ jobs: if-no-files-found: warn integration-chaos: - if: github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest timeout-minutes: 60 strategy: @@ -271,11 +270,12 @@ jobs: - name: Run integration chaos scenario subset env: INTEGRATION_BACKEND: ${{ matrix.backend }} + RUN_CHAOS: "1" run: | set -euo pipefail mkdir -p .artifacts go test -tags=integration ./integration/... \ - -run "TestIntegrationScenarios_AllBackends/.*/scenario_(dispatch_during_broker_fault|consume_after_broker_recovery|worker_restart_recovery|worker_restart_delay_recovery|shutdown_during_delay_retry|multi_worker_contention)$" \ + -run "(TestIntegrationChaos_RedisBrokerDisconnectRedelivery|TestIntegrationScenarios_AllBackends)/.*/scenario_(dispatch_during_broker_fault|consume_after_broker_recovery|worker_restart_recovery|worker_restart_delay_recovery|shutdown_during_delay_retry|multi_worker_contention)$" \ -count=1 -v \ | tee ".artifacts/integration-chaos-${{ matrix.backend }}.log" @@ -300,7 +300,6 @@ jobs: if-no-files-found: warn integration-flake-repeat: - if: github.event_name == 'workflow_dispatch' runs-on: ubuntu-latest timeout-minutes: 120 strategy: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5178d18..2aa76b9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,20 +27,34 @@ jobs: set -euo pipefail matrix_line="$(awk '/backend: \[/{print; exit}' .github/workflows/test.yml)" test -n "$matrix_line" - for backend in null sync workerpool redis mysql postgres sqlite nats sqs rabbitmq; do - if ! echo "$matrix_line" | grep -Eq "(^|[^a-z])${backend}([^a-z]|$)"; then - echo "missing required integration backend in matrix: ${backend}" + matrix_values="${matrix_line#*[}" + matrix_values="${matrix_values%%]*}" + IFS=',' read -r -a configured_backends <<< "$matrix_values" + required_backends=(null sync workerpool redis mysql postgres sqlite nats sqs rabbitmq) + if [[ "${#configured_backends[@]}" -ne "${#required_backends[@]}" ]]; then + echo "integration matrix has ${#configured_backends[@]} backends, want ${#required_backends[@]}" + exit 1 + fi + for backend in "${required_backends[@]}"; do + if ! grep -Fq "\"${backend}\"" <<< "$matrix_line"; then + echo "missing explicitly quoted integration backend in matrix: ${backend}" exit 1 fi done unit: runs-on: ubuntu-latest + env: + GOCACHE: /tmp/gocache + GOMODCACHE: /tmp/gomodcache steps: - name: Checkout uses: actions/checkout@v4 + - name: Verify Codecov report freshness guard + run: .github/scripts/test_wait_codecov_report.sh + - name: Set up Go uses: actions/setup-go@v5 with: @@ -49,6 +63,9 @@ jobs: - name: Install dependencies run: go mod download + - name: Verify generated docs are current and idempotent + run: ./scripts/check-generated-docs.sh + - name: Verify curated README manual snippets compile run: ./scripts/check-readme-snippets.sh @@ -56,19 +73,93 @@ jobs: working-directory: examples run: go test ./... -run TestExamplesBuild -count=1 - - name: Run unit tests - run: go test ./... -v + - name: Run all module unit tests and vet + run: FULL=1 VET=1 ./scripts/test-all-modules.sh - name: Run unit tests with coverage - run: ./scripts/coverage-codecov.sh + run: ./scripts/coverage-codecov.sh unit - - name: Upload results to Codecov - uses: codecov/codecov-action@v5 + - name: Preserve multi-module unit coverage + uses: actions/upload-artifact@v4 + with: + name: coverage-unit + path: | + coverage/coverage-unit.out + coverage/coverage-unit-modules.tsv + if-no-files-found: error + overwrite: true + retention-days: 1 + + minimum-go: + name: Minimum Go (${{ matrix.minimum_go_version }}) + runs-on: ubuntu-latest + env: + GOCACHE: /tmp/gocache + GOMODCACHE: /tmp/gomodcache + GOTOOLCHAIN: local + GOWORK: "off" + strategy: + fail-fast: false + matrix: + minimum_go_version: ["1.24.4", "1.25.0"] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.minimum_go_version }} + cache-dependency-path: | + go.sum + docs/go.sum + driver/*/go.sum + examples/go.sum + integration/go.sum + + - name: Compile modules at their declared minimum env: - CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + MODULE_GO_VERSION: ${{ matrix.minimum_go_version }} + run: | + set -euo pipefail + matched=0 + while read -r module go_version extra; do + [[ "$module" != \#* && -n "$module" ]] || continue + [[ -z "$extra" ]] || { echo "invalid module Go policy row for $module"; exit 1; } + [[ "$go_version" == "$MODULE_GO_VERSION" ]] || continue + matched=$((matched + 1)) + case "$module" in + docs) + (cd "$module" && go mod verify) + (cd "$module" && go test ./readme/main.go) + (cd "$module" && go test ./examplegen/main.go) + (cd "$module" && go test -tags=testcounts ./readme/testcounts -run '^$' -count=1) + (cd "$module" && go test -tags=benchrender ./bench -run '^$' -count=1) + ;; + examples) + (cd "$module" && go test ./... -run '^TestExamplesBuild$' -count=1) + ;; + integration) + (cd "$module" && INTEGRATION_BACKEND=sync go test -tags=integration ./... -run '^$' -count=1) + ;; + *) + (cd "$module" && go test ./... -run '^$' -count=1) + ;; + esac + done < scripts/module-go-versions.tsv + [[ "$matched" -gt 0 ]] || { echo "no modules declare Go $MODULE_GO_VERSION"; exit 1; } - race: + race_modules: + name: Race (${{ matrix.race_module }}) runs-on: ubuntu-latest + env: + GOCACHE: /tmp/gocache + GOMODCACHE: /tmp/gomodcache + GOWORK: "off" + strategy: + fail-fast: false + matrix: + race_module: [".", "driver/mysqlqueue", "driver/natsqueue", "driver/postgresqueue", "driver/rabbitmqqueue", "driver/redisqueue", "driver/sqlitequeue", "driver/sqlqueuecore", "driver/sqsqueue"] steps: - name: Checkout uses: actions/checkout@v4 @@ -77,20 +168,39 @@ jobs: uses: actions/setup-go@v5 with: go-version: "stable" + cache-dependency-path: | + go.sum + driver/*/go.sum - - name: Install dependencies + - name: Install module dependencies + working-directory: ${{ matrix.race_module }} run: go mod download - - name: Run race tests (root module scope) - run: go test -race ./... + - name: Run race tests + working-directory: ${{ matrix.race_module }} + run: go test -race ./... -count=1 + + race: + name: race + needs: race_modules + if: ${{ always() }} + runs-on: ubuntu-latest + steps: + - name: Require every race module + env: + RACE_RESULT: ${{ needs.race_modules.result }} + run: test "$RACE_RESULT" = "success" integration-all: needs: [integration-matrix-guard, integration-scenarios-contract-guard] runs-on: ubuntu-latest + env: + GOCACHE: /tmp/gocache + GOMODCACHE: /tmp/gomodcache strategy: fail-fast: false matrix: - backend: [null, sync, workerpool, redis, mysql, postgres, sqlite, nats, sqs, rabbitmq] + backend: ["null", "sync", "workerpool", "redis", "mysql", "postgres", "sqlite", "nats", "sqs", "rabbitmq"] steps: - name: Checkout uses: actions/checkout@v4 @@ -101,9 +211,67 @@ jobs: go-version: "stable" - name: Install dependencies + working-directory: integration run: go mod download - - name: Run integration tests + - name: Run integration tests with coverage env: INTEGRATION_BACKEND: ${{ matrix.backend }} - run: go test -p=1 -tags integration ./integration/... + run: ./scripts/coverage-codecov.sh integration + + - name: Preserve integration coverage + uses: actions/upload-artifact@v4 + with: + name: coverage-integration-${{ matrix.backend }} + path: coverage/coverage-integration-${{ matrix.backend }}.out + if-no-files-found: error + overwrite: true + retention-days: 1 + + coverage: + needs: [unit, integration-all] + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: "stable" + + - name: Download coverage artifacts + uses: actions/download-artifact@v4 + with: + pattern: coverage-* + path: coverage-artifacts + merge-multiple: false + + - name: Verify complete coverage fan-in + id: coverage-files + run: | + set -euo pipefail + .github/scripts/check_codecov_coverage.sh coverage-artifacts + profiles="$(find coverage-artifacts -type f -name '*.out' -print | LC_ALL=C sort | paste -sd, -)" + test -n "$profiles" + echo "files=$profiles" >> "$GITHUB_OUTPUT" + + - name: Upload complete coverage to Codecov + uses: codecov/codecov-action@v5 + with: + files: ${{ steps.coverage-files.outputs.files }} + disable_search: true + fail_ci_if_error: true + name: complete-test-suite-${{ github.run_id }}-${{ github.run_attempt }} + override_build: ${{ github.run_id }}-${{ github.run_attempt }} + override_commit: ${{ github.event.pull_request.head.sha || github.sha }} + verbose: true + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + - name: Wait for the exact Codecov report + env: + CODECOV_BASE_SHA: ${{ github.event.pull_request.base.sha }} + CODECOV_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + CODECOV_UPLOAD_NAME: complete-test-suite-${{ github.run_id }}-${{ github.run_attempt }} + run: .github/scripts/wait_codecov_report.sh diff --git a/.gitignore b/.gitignore index a681cbc..e1e146f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ .idea coverage.txt -.artifacts \ No newline at end of file +coverage/ +coverage-artifacts/ +.artifacts diff --git a/README.md b/README.md index fe97403..f350d9b 100644 --- a/README.md +++ b/README.md @@ -10,12 +10,12 @@ Go Reference License: MIT Go Test - Go version + Go version Latest tag - Unit tests (executed count) - Integration tests (executed count) + Unit tests (executed count) + Integration tests (executed count)

@@ -25,6 +25,10 @@ go get github.com/goforj/queue ``` +Existing deployments upgrading to application-type direct delivery must replace workers before switching producers. Follow the [direct delivery migration guide](docs/direct-delivery-migration.md), including its SQL schema step and backend-specific rollback constraints. + +The root queue module and non-PostgreSQL driver modules require Go 1.24.4 or newer. The PostgreSQL driver requires Go 1.25 or newer so it can use pgx 5.9.2, the first release containing the [GO-2026-5004 security fix](https://pkg.go.dev/vuln/GO-2026-5004). + ## Quick Start ```go @@ -64,19 +68,21 @@ func main() { | Driver / Backend | Mode | Notes | Durable | Async | Delay | Unique | Backoff | Timeout | Native Stats | Queue Admin | | ---: | :--- | :--- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | -| Null | Drop-only | Discards dispatched jobs; useful for disabled queue modes and smoke tests. | - | - | - | - | - | - | - | - | -| Sync | Inline (caller) | Deterministic local execution with no external infra. | - | - | - | ✓ | - | ✓ | - | - | -| Workerpool | In-process pool | Local async behavior without external broker/database. | - | ✓ | ✓ | ✓ | ✓ | ✓ | - | - | -| MySQL | SQL durable queue | MySQL driver module (`driver/mysqlqueue`) built on shared SQL queue core. | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | - | -| Postgres | SQL durable queue | Postgres driver module (`driver/postgresqueue`) built on shared SQL queue core. | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | - | -| SQLite | SQL durable queue | SQLite driver module (`driver/sqlitequeue`) built on shared SQL queue core. | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | ✓ | - | -| Redis | Redis/Asynq | Production Redis backend (Asynq semantics). | ✓ | ✓ | ✓ | ✓ | - | ✓ | ✓ | ✓ | -| NATS | Broker target | NATS transport with queue-subject routing. | - | ✓ | ✓ | ✓ | ✓ | ✓ | - | - | -| SQS | Broker target | AWS SQS transport with endpoint overrides for localstack/testing. | - | ✓ | ✓ | ✓ | ✓ | ✓ | - | - | -| RabbitMQ | Broker target | RabbitMQ transport and worker consumption. | - | ✓ | ✓ | ✓ | ✓ | ✓ | - | - | +| Null | Drop-only | Discards dispatched jobs; useful for disabled queue modes and smoke tests. | - | - | - | Instance | - | - | - | - | +| Sync | Inline (caller) | Deterministic local execution with no external infra. | - | - | - | Instance | - | ✓ | - | - | +| Workerpool | In-process pool | Local async behavior without external broker/database. | - | ✓ | ✓ | Instance | ✓ | ✓ | - | - | +| MySQL | SQL durable queue | MySQL driver module (`driver/mysqlqueue`) built on shared SQL queue core. | ✓ | ✓ | ✓ | Backend | ✓ | ✓ | ✓ | - | +| Postgres | SQL durable queue | Postgres driver module (`driver/postgresqueue`) built on shared SQL queue core. | ✓ | ✓ | ✓ | Backend | ✓ | ✓ | ✓ | - | +| SQLite | SQL durable queue | SQLite driver module (`driver/sqlitequeue`) built on shared SQL queue core. | ✓ | ✓ | ✓ | Backend | ✓ | ✓ | ✓ | - | +| Redis | Redis/Asynq | Production Redis backend (Asynq semantics). | ✓ | ✓ | ✓ | Backend | - | ✓ | ✓ | ✓ | +| NATS | Ephemeral broker | Core NATS subject routing; every plain subscription can receive a broadcast copy. | - | ✓ | ✓ | Instance | ✓ | ✓ | - | - | +| SQS | Broker target | AWS SQS transport with endpoint overrides for localstack/testing. | - | ✓ | ✓ | Instance | ✓ | ✓ | - | - | +| RabbitMQ | Broker target | RabbitMQ transport and worker consumption. | - | ✓ | ✓ | Instance | ✓ | ✓ | - | - | > SQL-backed queues (`sqlite`, `mysql`, `postgres`) are durable and convenient, but they trade throughput for operational simplicity. They default to `1` worker, and increasing concurrency may require DB tuning (indexes, connection pool, lock contention). Prefer broker-backed drivers for higher-throughput workloads. > +> **Unique scope:** `Instance` suppresses duplicates only within one queue runtime instance; `Backend` shares claims through the configured database or Redis service. Identity is the effective queue, logical application job type, and canonical serialized payload. Absent, zero-byte, and exact JSON `null` payloads share one absence identity; generated workflow IDs and delivery options do not change it. See [`docs/backend-guarantees.md`](docs/backend-guarantees.md) for acceptance, rollout, and failure-boundary details. +> > **Queue Admin status:** the cross-driver admin contract is defined in core (`ListJobs`, `RetryJob`, `CancelJob`, `DeleteJob`, `ClearQueue`, `QueueHistory`), but **full queue admin operations are currently implemented only for Redis**. Other drivers return `ErrQueueAdminUnsupported` for unsupported admin actions. ### Driver constructor quick examples @@ -218,7 +224,9 @@ _, _ = q.Chain( _, _ = q.Batch( queue.NewJob("emails:send"), queue.NewJob("sms:send"), -).Then(queue.NewJob("notifications:done")).Dispatch(context.Background()) +).Then(func(context.Context, queue.BatchState) error { + return nil +}).Dispatch(context.Background()) ``` **Middleware**: Cross-cutting execution policy. @@ -385,7 +393,7 @@ Use `queue.Observer` implementations to capture normalized runtime events across collector := queue.NewStatsCollector() observer := queue.MultiObserver( collector, - queue.ObserverFunc(func(event queue.Event) { + queue.ObserverFunc(func(_ context.Context, event queue.Event) { _ = event.Kind }), ) @@ -416,7 +424,7 @@ observer := queue.MultiObserver( Events: events, DropIfFull: true, }, - queue.ObserverFunc(func(e queue.Event) { + queue.ObserverFunc(func(_ context.Context, e queue.Event) { _ = e }), ) @@ -428,89 +436,58 @@ q, _ := queue.New(queue.Config{ _ = q ``` -### Kitchen sink event logging (runtime + workflow) +### Kitchen sink event logging Runnable example: `examples/observeall/main.go` ```go logger := slog.New(slog.NewJSONHandler(os.Stdout, nil)) -runtimeObserver := queue.ObserverFunc(func(event queue.Event) { - attemptInfo := fmt.Sprintf("attempt=%d/%d", event.Attempt, event.MaxRetry+1) - jobInfo := fmt.Sprintf("job=%s key=%s queue=%s driver=%s", event.JobType, event.JobKey, event.Queue, event.Driver) - - switch event.Kind { - case queue.EventEnqueueAccepted: - logger.Info("Accepted dispatch", "msg", fmt.Sprintf("Accepted %s", jobInfo), "scheduled", event.Scheduled, "at", event.Time.Format(time.RFC3339Nano)) - case queue.EventEnqueueRejected: - logger.Error("Dispatch failed", "msg", fmt.Sprintf("Rejected %s", jobInfo), "error", event.Err) - case queue.EventEnqueueDuplicate: - logger.Warn("Skipped duplicate job", "msg", fmt.Sprintf("Duplicate %s", jobInfo)) - case queue.EventEnqueueCanceled: - logger.Warn("Canceled dispatch", "msg", fmt.Sprintf("Canceled %s", jobInfo), "error", event.Err) - case queue.EventProcessStarted: - logger.Info("Started processing job", "msg", fmt.Sprintf("Started %s (%s)", jobInfo, attemptInfo), "at", event.Time.Format(time.RFC3339Nano)) - case queue.EventProcessSucceeded: - logger.Info("Processed job", "msg", fmt.Sprintf("Processed %s in %s (%s)", jobInfo, event.Duration, attemptInfo)) - case queue.EventProcessFailed: - logger.Error("Processing failed", "msg", fmt.Sprintf("Failed %s after %s (%s)", jobInfo, event.Duration, attemptInfo), "error", event.Err) - case queue.EventProcessRetried: - logger.Warn("Retrying job", "msg", fmt.Sprintf("Retry scheduled for %s (%s)", jobInfo, attemptInfo), "error", event.Err) - case queue.EventProcessArchived: - logger.Error("Archived failed job", "msg", fmt.Sprintf("Archived %s after final failure (%s)", jobInfo, attemptInfo), "error", event.Err) - case queue.EventQueuePaused: - logger.Info("Paused queue", "msg", fmt.Sprintf("Paused queue=%s driver=%s", event.Queue, event.Driver)) - case queue.EventQueueResumed: - logger.Info("Resumed queue", "msg", fmt.Sprintf("Resumed queue=%s driver=%s", event.Queue, event.Driver)) - default: - logger.Info("Queue event", "msg", fmt.Sprintf("kind=%s %s", event.Kind, jobInfo)) - } -}) -workflowObserver := queue.WorkflowObserverFunc(func(event queue.WorkflowEvent) { - logger.Info("workflow event", +observer := queue.ObserverFunc(func(ctx context.Context, event queue.Event) { + logger.InfoContext(ctx, "queue event", + "layer", event.Layer, "kind", event.Kind, + "driver", event.Driver, + "queue", event.Queue, "dispatch_id", event.DispatchID, "job_id", event.JobID, "chain_id", event.ChainID, "batch_id", event.BatchID, "job_type", event.JobType, - "queue", event.Queue, "attempt", event.Attempt, + "max_retry", event.MaxRetry, "duration", event.Duration, "err", event.Err, ) }) q, _ := queue.New( - queue.Config{ - Driver: queue.DriverSync, - Observer: runtimeObserver, - }, - queue.WithObserver(workflowObserver), + queue.Config{Driver: queue.DriverSync}, + queue.WithObserver(observer), ) _ = q ``` ### Events reference -| Type | EventKind | Meaning | +| Layer | EventKind | Meaning | | ---: | --- | --- | +| **queue** | dispatch_started | Public dispatch began. | +| **queue** | dispatch_succeeded | Backend acceptance completed; synchronous execution may still return an application error. | +| **queue** | dispatch_failed | Public dispatch failed before backend acceptance. | | **queue** | enqueue_accepted | Job accepted by driver for enqueue. | | **queue** | enqueue_rejected | Job enqueue failed. | | **queue** | enqueue_duplicate | Duplicate job rejected due to uniqueness key. | | **queue** | enqueue_canceled | Context cancellation prevented enqueue. | -| **queue** | process_started | Worker began processing job. | -| **queue** | process_succeeded | Handler returned success. | -| **queue** | process_failed | Handler returned error. | -| **queue** | process_retried | Driver scheduled retry attempt. | -| **queue** | process_archived | Job moved to terminal failure state. | +| **worker** | process_started | Worker began processing job. | +| **worker** | process_succeeded | Handler returned success. | +| **worker** | process_failed | Handler returned an error or panicked. | +| **worker** | process_retried | A numbered application retry attempt began; infrastructure redelivery may repeat the fact. | +| **worker** | process_archived | Driver confirmed terminal settlement; unsupported paths omit this fact. | | **queue** | queue_paused | Queue was paused (driver supports pause). | | **queue** | queue_resumed | Queue was resumed. | -| **workflow** | dispatch_started | Workflow runtime accepted a dispatch request and created a dispatch record. | -| **workflow** | dispatch_succeeded | Dispatch was successfully enqueued to the underlying queue runtime. | -| **workflow** | dispatch_failed | Dispatch failed before job execution could start. | | **workflow** | job_started | A workflow job handler started execution. | | **workflow** | job_succeeded | A workflow job handler completed successfully. | -| **workflow** | job_failed | A workflow job handler returned an error. | +| **workflow** | job_failed | A logical job reached permanent or exhausted failure. | | **workflow** | chain_started | A chain workflow was created and started. | | **workflow** | chain_advanced | Chain progressed from one node to the next node. | | **workflow** | chain_completed | Chain reached terminal success. | @@ -524,6 +501,8 @@ _ = q | **workflow** | callback_succeeded | Chain/batch callback completed successfully. | | **workflow** | callback_failed | Chain/batch callback returned an error. | +Handler panics now emit `process_failed` before the original panic value is rethrown. This adds truthful failure telemetry without changing backend panic recovery or retry behavior. + ## Examples Runnable examples live in the separate `examples` module ([`./examples`](./examples)). @@ -546,13 +525,13 @@ The API section below is autogenerated; do not edit between the markers. | Group | Functions | |------:|:-----------| | **Admin** | [CancelJob](#queue-canceljob) · [Queue.CancelJob](#queue-queue-canceljob) · [ClearQueue](#queue-clearqueue) · [Queue.ClearQueue](#queue-queue-clearqueue) · [DeleteJob](#queue-deletejob) · [Queue.DeleteJob](#queue-queue-deletejob) · [History](#queue-queue-history) · [ListJobs](#queue-listjobs) · [Queue.ListJobs](#queue-queue-listjobs) · [Normalize](#queue-listjobsoptions-normalize) · [QueueHistory](#queue-queuehistory) · [RetryJob](#queue-retryjob) · [Queue.RetryJob](#queue-queue-retryjob) · [SinglePointHistory](#queue-singlepointhistory) · [SupportsQueueAdmin](#queue-supportsqueueadmin) · [TimelineHistoryFromSnapshot](#queue-timelinehistoryfromsnapshot) | -| **Constructors** | [New](#queue-new) · [NewNull](#queue-newnull) · [NewStatsCollector](#queue-newstatscollector) · [NewSync](#queue-newsync) · [NewWorkerpool](#queue-newworkerpool) | +| **Constructors** | [New](#queue-new) · [NewMemoryStore](#queue-newmemorystore) · [NewMessage](#queue-newmessage) · [NewNull](#queue-newnull) · [NewSQLStore](#queue-newsqlstore) · [NewSQLStoreWithManagedSchema](#queue-newsqlstorewithmanagedschema) · [NewStatsCollector](#queue-newstatscollector) · [NewSync](#queue-newsync) · [NewWorkerpool](#queue-newworkerpool) | | **Job** | [Backoff](#queue-job-backoff) · [Bind](#queue-job-bind) · [Delay](#queue-job-delay) · [NewJob](#queue-newjob) · [OnQueue](#queue-job-onqueue) · [Payload](#queue-job-payload) · [PayloadBytes](#queue-job-payloadbytes) · [PayloadJSON](#queue-job-payloadjson) · [Retry](#queue-job-retry) · [Timeout](#queue-job-timeout) · [UniqueFor](#queue-job-uniquefor) | | **Observability** | [Active](#queue-statssnapshot-active) · [Archived](#queue-statssnapshot-archived) · [Failed](#queue-statssnapshot-failed) · [MultiObserver](#queue-multiobserver) · [ChannelObserver.Observe](#queue-channelobserver-observe) · [Observer.Observe](#queue-observer-observe) · [ObserverFunc.Observe](#queue-observerfunc-observe) · [StatsCollector.Observe](#queue-statscollector-observe) · [Pause](#queue-pause) · [Paused](#queue-statssnapshot-paused) · [Pending](#queue-statssnapshot-pending) · [Processed](#queue-statssnapshot-processed) · [Queue](#queue-statssnapshot-queue) · [Queues](#queue-statssnapshot-queues) · [Ready](#queue-ready) · [Resume](#queue-resume) · [RetryCount](#queue-statssnapshot-retrycount) · [SafeObserve](#queue-safeobserve) · [Scheduled](#queue-statssnapshot-scheduled) · [Snapshot](#queue-snapshot) · [StatsCollector.Snapshot](#queue-statscollector-snapshot) · [SupportsNativeStats](#queue-supportsnativestats) · [SupportsPause](#queue-supportspause) · [Throughput](#queue-statssnapshot-throughput) | -| **Other** | [PhysicalQueueName](#queue-physicalqueuename) · [PhysicalQueueWeights](#queue-physicalqueueweights) · [ResolveObservedJobType](#queue-resolveobservedjobtype) | -| **Queue** | [Batch](#queue-queue-batch) · [Chain](#queue-queue-chain) · [Dispatch](#queue-queue-dispatch) · [Driver](#queue-queue-driver) · [FindBatch](#queue-queue-findbatch) · [FindChain](#queue-queue-findchain) · [Pause](#queue-queue-pause) · [Prune](#queue-queue-prune) · [Ready](#queue-queue-ready) · [Register](#queue-queue-register) · [Resume](#queue-queue-resume) · [Run](#queue-queue-run) · [Shutdown](#queue-queue-shutdown) · [StartWorkers](#queue-queue-startworkers) · [Stats](#queue-queue-stats) · [WithClock](#queue-withclock) · [WithContext](#queue-queue-withcontext) · [WithHandlerContextDecorator](#queue-withhandlercontextdecorator) · [WithMiddleware](#queue-withmiddleware) · [WithObserver](#queue-withobserver) · [WithStore](#queue-withstore) · [WithWorkers](#queue-withworkers) · [Queue.WithWorkers](#queue-queue-withworkers) | +| **Other** | [Acquire](#queue-locker-acquire) · [AdvanceChain](#queue-workflowstore-advancechain) · [Allow](#queue-ratelimiter-allow) · [AllowFailures](#queue-batchbuilder-allowfailures) · [CancelBatch](#queue-workflowstore-cancelbatch) · [BatchBuilder.Catch](#queue-batchbuilder-catch) · [ChainBuilder.Catch](#queue-chainbuilder-catch) · [CreateBatch](#queue-workflowstore-createbatch) · [CreateChain](#queue-workflowstore-createchain) · [BatchBuilder.Dispatch](#queue-batchbuilder-dispatch) · [ChainBuilder.Dispatch](#queue-chainbuilder-dispatch) · [FailChain](#queue-workflowstore-failchain) · [FailChainNode](#queue-workflowoutcomestore-failchainnode) · [BatchBuilder.Finally](#queue-batchbuilder-finally) · [ChainBuilder.Finally](#queue-chainbuilder-finally) · [GetBatch](#queue-workflowstore-getbatch) · [GetChain](#queue-workflowstore-getchain) · [FailOnError.Handle](#queue-failonerror-handle) · [Middleware.Handle](#queue-middleware-handle) · [MiddlewareFunc.Handle](#queue-middlewarefunc-handle) · [RateLimit.Handle](#queue-ratelimit-handle) · [RetryPolicy.Handle](#queue-retrypolicy-handle) · [SkipWhen.Handle](#queue-skipwhen-handle) · [WithoutOverlapping.Handle](#queue-withoutoverlapping-handle) · [MarkBatchJobFailed](#queue-workflowstore-markbatchjobfailed) · [MarkBatchJobStarted](#queue-workflowstore-markbatchjobstarted) · [MarkBatchJobSucceeded](#queue-workflowstore-markbatchjobsucceeded) · [MarkCallbackInvoked](#queue-workflowstore-markcallbackinvoked) · [Name](#queue-batchbuilder-name) · [BatchBuilder.OnQueue](#queue-batchbuilder-onqueue) · [ChainBuilder.OnQueue](#queue-chainbuilder-onqueue) · [PhysicalQueueName](#queue-physicalqueuename) · [PhysicalQueueWeights](#queue-physicalqueueweights) · [Progress](#queue-batchbuilder-progress) · [Prune](#queue-workflowstore-prune) · [Release](#queue-lock-release) · [ResolveObservedJobType](#queue-resolveobservedjobtype) · [SettleBatchJob](#queue-workflowoutcomestore-settlebatchjob) · [Then](#queue-batchbuilder-then) | +| **Queue** | [Batch](#queue-queue-batch) · [Bind](#queue-message-bind) · [Chain](#queue-queue-chain) · [Dispatch](#queue-queue-dispatch) · [Driver](#queue-queue-driver) · [FindBatch](#queue-queue-findbatch) · [FindChain](#queue-queue-findchain) · [IsPermanent](#queue-ispermanent) · [Pause](#queue-queue-pause) · [PayloadBytes](#queue-message-payloadbytes) · [Permanent](#queue-permanent) · [Prune](#queue-queue-prune) · [Ready](#queue-queue-ready) · [Register](#queue-queue-register) · [Resume](#queue-queue-resume) · [Run](#queue-queue-run) · [Shutdown](#queue-queue-shutdown) · [StartWorkers](#queue-queue-startworkers) · [Stats](#queue-queue-stats) · [WithClock](#queue-withclock) · [WithContext](#queue-queue-withcontext) · [WithHandlerContextDecorator](#queue-withhandlercontextdecorator) · [WithLegacyDirectEnvelope](#queue-withlegacydirectenvelope) · [WithMiddleware](#queue-withmiddleware) · [WithObserver](#queue-withobserver) · [WithStore](#queue-withstore) · [WithWorkers](#queue-withworkers) · [Queue.WithWorkers](#queue-queue-withworkers) | | **Driver Constructors** | [mysqlqueue.New](#mysqlqueue-new) · [mysqlqueue.NewWithConfig](#mysqlqueue-newwithconfig) · [natsqueue.New](#natsqueue-new) · [natsqueue.NewWithConfig](#natsqueue-newwithconfig) · [postgresqueue.New](#postgresqueue-new) · [postgresqueue.NewWithConfig](#postgresqueue-newwithconfig) · [rabbitmqqueue.New](#rabbitmqqueue-new) · [rabbitmqqueue.NewWithConfig](#rabbitmqqueue-newwithconfig) · [redisqueue.New](#redisqueue-new) · [redisqueue.NewWithConfig](#redisqueue-newwithconfig) · [sqlitequeue.New](#sqlitequeue-new) · [sqlitequeue.NewWithConfig](#sqlitequeue-newwithconfig) · [sqsqueue.New](#sqsqueue-new) · [sqsqueue.NewWithConfig](#sqsqueue-newwithconfig) | -| **Testing** | [AssertBatchCount](#queuefake-fake-assertbatchcount) · [AssertBatched](#queuefake-fake-assertbatched) · [AssertChained](#queuefake-fake-assertchained) · [AssertCount](#queuefake-fake-assertcount) · [AssertDispatched](#queuefake-fake-assertdispatched) · [AssertDispatchedOn](#queuefake-fake-assertdispatchedon) · [AssertDispatchedTimes](#queuefake-fake-assertdispatchedtimes) · [AssertNotDispatched](#queuefake-fake-assertnotdispatched) · [AssertNothingBatched](#queuefake-fake-assertnothingbatched) · [AssertNothingDispatched](#queuefake-fake-assertnothingdispatched) · [AssertNothingWorkflowDispatched](#queuefake-fake-assertnothingworkflowdispatched) · [AssertWorkflowDispatched](#queuefake-fake-assertworkflowdispatched) · [AssertWorkflowDispatchedOn](#queuefake-fake-assertworkflowdispatchedon) · [AssertWorkflowDispatchedTimes](#queuefake-fake-assertworkflowdispatchedtimes) · [AssertWorkflowNotDispatched](#queuefake-fake-assertworkflownotdispatched) · [Count](#queuefake-fake-count) · [CountJob](#queuefake-fake-countjob) · [CountOn](#queuefake-fake-counton) · [New](#queuefake-new) · [Queue](#queuefake-fake-queue) · [Records](#queuefake-fake-records) · [Reset](#queuefake-fake-reset) · [Workflow](#queuefake-fake-workflow) | +| **Testing** | [AssertBatchCount](#queue-fakequeue-assertbatchcount) · [AssertBatched](#queue-fakequeue-assertbatched) · [AssertChained](#queue-fakequeue-assertchained) · [AssertCount](#queue-fakequeue-assertcount) · [AssertDispatched](#queue-fakequeue-assertdispatched) · [AssertDispatchedOn](#queue-fakequeue-assertdispatchedon) · [AssertDispatchedTimes](#queue-fakequeue-assertdispatchedtimes) · [AssertNotDispatched](#queue-fakequeue-assertnotdispatched) · [AssertNothingBatched](#queue-fakequeue-assertnothingbatched) · [AssertNothingDispatched](#queue-fakequeue-assertnothingdispatched) · [Batch](#queue-fakequeue-batch) · [BatchRecords](#queue-fakequeue-batchrecords) · [Chain](#queue-fakequeue-chain) · [ChainRecords](#queue-fakequeue-chainrecords) · [Dispatch](#queue-fakequeue-dispatch) · [Driver](#queue-fakequeue-driver) · [FindBatch](#queue-fakequeue-findbatch) · [FindChain](#queue-fakequeue-findchain) · [NewFake](#queue-newfake) · [Prune](#queue-fakequeue-prune) · [Ready](#queue-fakequeue-ready) · [Records](#queue-fakequeue-records) · [Register](#queue-fakequeue-register) · [Reset](#queue-fakequeue-reset) · [Shutdown](#queue-fakequeue-shutdown) · [StartWorkers](#queue-fakequeue-startworkers) · [WithContext](#queue-fakequeue-withcontext) · [Workers](#queue-fakequeue-workers) | @@ -821,6 +800,16 @@ _, _ = q.Dispatch( ) ``` +#### NewMemoryStore + +NewMemoryStore creates an in-memory workflow state store. It copies chain +nodes and payload bytes on creation and return so callers retain independent ownership. + +#### NewMessage + +NewMessage creates a logical queue message from an application job type and exact payload bytes. +The payload is copied so callers can safely reuse or mutate their input buffer. + #### NewNull NewNull creates a Queue on the null backend. @@ -832,6 +821,16 @@ if err != nil { } ``` +#### NewSQLStore + +NewSQLStore creates a SQL-backed workflow state store. + +#### NewSQLStoreWithManagedSchema + +NewSQLStoreWithManagedSchema creates a SQL-backed workflow state store +without executing schema DDL. The supplied database must already contain the +dialect-correct workflow tables, including transition receipts. + #### NewStatsCollector NewStatsCollector creates an event collector for queue counters. @@ -872,7 +871,7 @@ Backoff sets delay between retries. job := queue.NewJob("emails:send").Backoff(500 * time.Millisecond) ``` -#### Bind +#### Job.Bind Bind unmarshals job payload JSON into dst. @@ -908,7 +907,7 @@ NewJob creates a job value with a required job type. job := queue.NewJob("emails:send") ``` -#### OnQueue +#### Job.OnQueue OnQueue sets the target queue name. @@ -954,7 +953,7 @@ jobMap := queue.NewJob("emails:send").Payload(map[string]any{ }) ``` -#### PayloadBytes +#### Job.PayloadBytes PayloadBytes returns a copy of job payload bytes. @@ -1175,7 +1174,7 @@ fmt.Println(snapshot.Processed("default")) // Output: 11 ``` -#### StatsSnapshot.Queue +#### Queue Queue returns queue counters for a queue name. @@ -1359,6 +1358,137 @@ fmt.Printf("ok=%v hour=%+v day=%+v week=%+v\n", ok, throughput.Hour, throughput. #### Other +#### Acquire + +Acquire attempts to hold key for ttl. + +#### AdvanceChain + +AdvanceChain atomically claims completedNode and returns the current successor. +Repeating the same (chainID, completedNode) claim must not advance again. +When done is true, GetChain must immediately expose Completed or Failed state. + +#### Allow + +Allow returns whether key may execute and any suggested retry delay. + +#### AllowFailures + +AllowFailures keeps remaining members active after a terminal member failure. + +#### CancelBatch + +CancelBatch commits aggregate batch cancellation. + +#### BatchBuilder.Catch + +Catch registers the explicitly ephemeral batch failure callback. + +#### ChainBuilder.Catch + +Catch registers the explicitly ephemeral chain failure callback. + +#### CreateBatch + +CreateBatch persists a newly accepted batch. BatchID and every JobID must +be non-empty, Jobs must contain at least one entry, and JobIDs must be unique. + +#### CreateChain + +CreateChain persists a newly accepted chain. ChainID and every NodeID must +be non-empty, Nodes must contain at least one entry, and NodeIDs must be unique. + +#### BatchBuilder.Dispatch + +Dispatch persists and starts the batch workflow. + +#### ChainBuilder.Dispatch + +Dispatch persists and starts the chain workflow. + +#### FailChain + +FailChain commits terminal failure without replacing completed state. + +#### FailChainNode + +FailChainNode commits failure only while nodeID is the current unsettled node. +owned remains true on replay while that node's failure owns the chain. + +#### BatchBuilder.Finally + +Finally registers the explicitly ephemeral batch terminal callback. + +#### ChainBuilder.Finally + +Finally registers the explicitly ephemeral chain terminal callback. + +#### GetBatch + +GetBatch returns current batch state. + +#### GetChain + +GetChain returns current chain state. + +#### FailOnError.Handle + +Handle wraps matched errors as fatal errors to stop retries. + +#### Middleware.Handle + +Handle wraps the remaining middleware and handler chain. + +#### MiddlewareFunc.Handle + +Handle calls the wrapped middleware function. + +#### RateLimit.Handle + +Handle applies limiter checks before executing the next handler. + +#### RetryPolicy.Handle + +Handle passes execution through without modification. + +#### SkipWhen.Handle + +Handle skips job execution when Predicate returns true. + +#### WithoutOverlapping.Handle + +Handle acquires a lock and prevents concurrent overlap for the same key. + +#### MarkBatchJobFailed + +MarkBatchJobFailed commits the first outcome for (batchID, jobID). +Duplicate outcomes must return current state without changing counters. + +#### MarkBatchJobStarted + +MarkBatchJobStarted records that one batch member began execution. + +#### MarkBatchJobSucceeded + +MarkBatchJobSucceeded commits the first outcome for (batchID, jobID). +Duplicate outcomes must return current state without changing counters. + +#### MarkCallbackInvoked + +MarkCallbackInvoked atomically claims one callback idempotency key. + +#### Name + +Name assigns an application-facing label to the batch. + +#### BatchBuilder.OnQueue + +OnQueue applies a default queue to batch jobs without an explicit target. + +#### ChainBuilder.OnQueue + +OnQueue applies a default queue to chain jobs without an explicit target. + #### PhysicalQueueName PhysicalQueueName maps a logical queue name into the physical name used by the backing queue driver. @@ -1367,17 +1497,39 @@ PhysicalQueueName maps a logical queue name into the physical name used by the b PhysicalQueueWeights maps logical weighted queue names into their physical backend names. +#### Progress + +Progress registers the explicitly ephemeral batch progress callback. + +#### WorkflowStore.Prune + +Prune removes terminal workflow state older than before. + +#### Release + +Release relinquishes the acquired lease. + #### ResolveObservedJobType ResolveObservedJobType returns the effective application job type that should -be emitted to observers. External workers may process internal bus wrapper -jobs (for example, "bus:job") whose payload embeds the real application job +be emitted to observers. External workers may process private workflow delivery +envelopes (for example, "bus:job") whose payload embeds the real application job type. When possible, this helper unwraps that payload so dashboards and metrics reflect the user-facing job type instead of the transport wrapper. +#### SettleBatchJob + +SettleBatchJob returns the first committed outcome for one batch member. +owned remains true on same-outcome replay and false when the opposite outcome won. +Ownership covers the outcome category; BatchState does not retain a per-member cause. + +#### Then + +Then registers the explicitly ephemeral batch success callback. + #### Queue -#### Batch +#### Queue.Batch Batch creates a batch builder for fan-out workflow execution. @@ -1387,13 +1539,21 @@ if err != nil { return } q.Register("emails:send", func(ctx context.Context, m queue.Message) error { return nil }) +if err := q.StartWorkers(context.Background()); err != nil { + return +} +defer q.Shutdown(context.Background()) _, _ = q.Batch( queue.NewJob("emails:send").Payload(map[string]any{"id": 1}), queue.NewJob("emails:send").Payload(map[string]any{"id": 2}), ).Name("send-emails").OnQueue("default").Dispatch(context.Background()) ``` -#### Chain +#### Message.Bind + +Bind unmarshals the raw job payload into dst. + +#### Queue.Chain Chain creates a chain builder for sequential workflow execution. @@ -1404,6 +1564,10 @@ if err != nil { } q.Register("first", func(ctx context.Context, m queue.Message) error { return nil }) q.Register("second", func(ctx context.Context, m queue.Message) error { return nil }) +if err := q.StartWorkers(context.Background()); err != nil { + return +} +defer q.Shutdown(context.Background()) _, _ = q.Chain( queue.NewJob("first"), queue.NewJob("second"), @@ -1412,7 +1576,8 @@ _, _ = q.Chain( #### Queue.Dispatch -Dispatch enqueues a high-level job using the queue's bound context. +Dispatch enqueues a high-level job using its application type and exact +payload bytes together with the queue's bound context. ```go q, err := queue.NewSync() @@ -1420,6 +1585,10 @@ if err != nil { return } q.Register("emails:send", func(ctx context.Context, m queue.Message) error { return nil }) +if err := q.StartWorkers(context.Background()); err != nil { + return +} +defer q.Shutdown(context.Background()) job := queue.NewJob("emails:send").Payload(map[string]any{"id": 1}).OnQueue("default") _, _ = q.Dispatch(job) ``` @@ -1437,7 +1606,7 @@ fmt.Println(q.Driver()) // Output: sync ``` -#### FindBatch +#### Queue.FindBatch FindBatch returns current batch state by ID. @@ -1454,7 +1623,7 @@ if err != nil { _, _ = q.FindBatch(context.Background(), batchID) ``` -#### FindChain +#### Queue.FindChain FindChain returns current chain state by ID. @@ -1471,6 +1640,10 @@ if err != nil { _, _ = q.FindChain(context.Background(), chainID) ``` +#### IsPermanent + +IsPermanent reports whether an error requests terminal application settlement. + #### Queue.Pause Pause pauses consumption for a queue when supported by the underlying driver. @@ -1487,7 +1660,15 @@ if queue.SupportsPause(q) { } ``` -#### Prune +#### Message.PayloadBytes + +PayloadBytes returns an isolated copy of the raw job payload. + +#### Permanent + +Permanent marks an error as terminal so workers do not spend the remaining application retry budget on it. + +#### Queue.Prune Prune deletes old workflow state records. @@ -1641,6 +1822,15 @@ if err != nil { } ``` +#### WithLegacyDirectEnvelope + +WithLegacyDirectEnvelope keeps ordinary dispatches on the version-one +`bus:job` wire route during a workers-first migration. Remove this option only +after every consumer can process canonical direct deliveries. See the +[direct delivery migration guide] for backend-specific rollout and rollback. + +[direct delivery migration guide]: https://github.com/goforj/queue/blob/main/docs/direct-delivery-migration.md + #### WithMiddleware WithMiddleware appends queue workflow middleware. @@ -1657,10 +1847,10 @@ if err != nil { #### WithObserver -WithObserver installs a workflow lifecycle observer. +WithObserver installs one observer for queue, worker, and workflow lifecycle events. ```go -observer := queue.WorkflowObserverFunc(func(_ context.Context, event queue.WorkflowEvent) { +observer := queue.ObserverFunc(func(_ context.Context, event queue.Event) { _ = event.Kind }) q, err := queue.New(queue.Config{Driver: queue.DriverSync}, queue.WithObserver(observer)) @@ -1707,184 +1897,6 @@ if err != nil { q.WithWorkers(4) // optional; default: runtime.NumCPU() (min 1) ``` -#### Testing - -#### FakeQueue.AssertCount - -AssertCount fails when dispatch count is not expected. - -```go -fake := queue.NewFake() -_ = fake.Dispatch(queue.NewJob("emails:send")) -fake.AssertCount(t, 1) -``` - -#### FakeQueue.AssertDispatched - -AssertDispatched fails when jobType was not dispatched. - -```go -fake := queue.NewFake() -_ = fake.Dispatch(queue.NewJob("emails:send")) -fake.AssertDispatched(t, "emails:send") -``` - -#### FakeQueue.AssertDispatchedOn - -AssertDispatchedOn fails when jobType was not dispatched on queueName. - -```go -fake := queue.NewFake() -_ = fake.Dispatch( - queue.NewJob("emails:send"). - OnQueue("critical"), -) -fake.AssertDispatchedOn(t, "critical", "emails:send") -``` - -#### FakeQueue.AssertDispatchedTimes - -AssertDispatchedTimes fails when jobType dispatch count does not match expected. - -```go -fake := queue.NewFake() -_ = fake.Dispatch(queue.NewJob("emails:send")) -_ = fake.Dispatch(queue.NewJob("emails:send")) -fake.AssertDispatchedTimes(t, "emails:send", 2) -``` - -#### FakeQueue.AssertNotDispatched - -AssertNotDispatched fails when jobType was dispatched. - -```go -fake := queue.NewFake() -_ = fake.Dispatch(queue.NewJob("emails:send")) -fake.AssertNotDispatched(t, "emails:cancel") -``` - -#### FakeQueue.AssertNothingDispatched - -AssertNothingDispatched fails when any dispatch was recorded. - -```go -fake := queue.NewFake() -fake.AssertNothingDispatched(t) -``` - -#### FakeQueue.Dispatch - -Dispatch records a typed job payload in-memory using the fake default queue. - -```go -fake := queue.NewFake() -err := fake.Dispatch(queue.NewJob("emails:send").OnQueue("default")) -``` - -#### FakeQueue.Driver - -Driver returns the active queue driver. - -```go -fake := queue.NewFake() -driver := fake.Driver() -``` - -#### NewFake - -NewFake creates a queue fake that records dispatches and provides assertions. - -```go -fake := queue.NewFake() -_ = fake.Dispatch( - queue.NewJob("emails:send"). - Payload(map[string]any{"id": 1}). - OnQueue("critical"), -) -records := fake.Records() -fmt.Println(len(records), records[0].Queue, records[0].Job.Type) -// Output: 1 critical emails:send -``` - -#### FakeQueue.Ready - -Ready validates fake queue readiness. - -```go -fake := queue.NewFake() -fmt.Println(fake.Ready(context.Background()) == nil) -// true -``` - -#### FakeQueue.Records - -Records returns a copy of all dispatch records. - -```go -fake := queue.NewFake() -_ = fake.Dispatch(queue.NewJob("emails:send").OnQueue("default")) -records := fake.Records() -fmt.Println(len(records), records[0].Job.Type) -// Output: 1 emails:send -``` - -#### FakeQueue.Register - -Register associates a handler with a job type. - -```go -fake := queue.NewFake() -fake.Register("emails:send", func(context.Context, queue.Job) error { return nil }) -``` - -#### FakeQueue.Reset - -Reset clears all recorded dispatches. - -```go -fake := queue.NewFake() -_ = fake.Dispatch(queue.NewJob("emails:send").OnQueue("default")) -fmt.Println(len(fake.Records())) -fake.Reset() -fmt.Println(len(fake.Records())) -// Output: -// 1 -// 0 -``` - -#### FakeQueue.Shutdown - -Shutdown drains running work and releases resources. - -```go -fake := queue.NewFake() -err := fake.Shutdown(context.Background()) -``` - -#### FakeQueue.StartWorkers - -StartWorkers starts worker execution. - -```go -fake := queue.NewFake() -err := fake.StartWorkers(context.Background()) -``` - -#### FakeQueue.WithContext - -WithContext returns a derived fake queue handle bound to ctx. - -#### FakeQueue.Workers - -Workers sets desired worker concurrency before StartWorkers. - -```go -fake := queue.NewFake() -q := fake.Workers(4) -fmt.Println(q != nil) -// Output: true -``` - ## Driver Constructors @@ -1917,6 +1929,7 @@ q, err := mysqlqueue.NewWithConfig( }, DB: nil, // optional; provide *sql.DB instead of DSN DSN: "user:pass@tcp(127.0.0.1:3306)/queue?parseTime=true", // optional if DB is set + DisableAutoMigrate: false, // set true when schema migrations are managed externally ProcessingRecoveryGrace: 2 * time.Second, // default if <=0: 2s ProcessingLeaseNoTimeout: 5 * time.Minute, // default if <=0: 5m }, @@ -1994,6 +2007,7 @@ q, err := postgresqueue.NewWithConfig( }, DB: nil, // optional; provide *sql.DB instead of DSN DSN: "postgres://user:pass@127.0.0.1:5432/queue?sslmode=disable", // optional if DB is set + DisableAutoMigrate: false, // set true when schema migrations are managed externally ProcessingRecoveryGrace: 2 * time.Second, // default if <=0: 2s ProcessingLeaseNoTimeout: 5 * time.Minute, // default if <=0: 5m }, @@ -2112,6 +2126,7 @@ q, err := sqlitequeue.NewWithConfig( }, DB: nil, // optional; provide *sql.DB instead of DSN DSN: "file:queue.db?_busy_timeout=5000", // optional if DB is set + DisableAutoMigrate: false, // set true when schema migrations are managed externally ProcessingRecoveryGrace: 2 * time.Second, // default if <=0: 2s ProcessingLeaseNoTimeout: 5 * time.Minute, // default if <=0: 5m }, @@ -2165,250 +2180,279 @@ if err != nil { ## Testing API +`queue.NewFake` is a recording fake with its established `Dispatch(any) error` surface. Inject it where `*queue.FakeQueue` or that recording contract is accepted; it is not a drop-in `*queue.Queue`. + Examples in this section assume they are used inside tests and `t` is a `*testing.T` (or `testing.TB`). -#### Fake.AssertBatchCount +#### FakeQueue.AssertBatchCount -AssertBatchCount fails if total recorded workflow batch count does not match n. +AssertBatchCount fails unless the accepted batch count equals expected. ```go -f := queuefake.New() -_, _ = f.Workflow().Batch(bus.NewJob("a", nil)).Dispatch(nil) -f.AssertBatchCount(t, 1) +fake := queue.NewFake() +_, _ = fake.Batch(queue.NewJob("emails:send")).Dispatch(context.Background()) +fake.AssertBatchCount(t, 1) ``` -#### Fake.AssertBatched +#### FakeQueue.AssertBatched -AssertBatched fails unless at least one recorded workflow batch matches predicate. +AssertBatched fails unless an accepted canonical batch matches predicate. +The predicate runs outside the recorder lock so it may safely inspect the fake. ```go -f := queuefake.New() -_, _ = f.Workflow().Batch(bus.NewJob("a", nil), bus.NewJob("b", nil)).Dispatch(nil) -f.AssertBatched(t, func(spec bus.BatchSpec) bool { return len(spec.JobTypes) == 2 }) +fake := queue.NewFake() +_, _ = fake.Batch(queue.NewJob("emails:send")).Name("nightly").Dispatch(context.Background()) +fake.AssertBatched(t, func(record queue.BatchRecord) bool { return record.Name == "nightly" }) ``` -#### Fake.AssertChained +#### FakeQueue.AssertChained -AssertChained fails if no recorded workflow chain matches expected job type order. +AssertChained fails unless an accepted chain has the expected ordered job types. ```go -f := queuefake.New() -_, _ = f.Workflow().Chain(bus.NewJob("a", nil), bus.NewJob("b", nil)).Dispatch(nil) -f.AssertChained(t, []string{"a", "b"}) +fake := queue.NewFake() +_, _ = fake.Chain( + queue.NewJob("reports:build"), + queue.NewJob("reports:publish"), +).Dispatch(context.Background()) +fake.AssertChained(t, []string{"reports:build", "reports:publish"}) ``` -#### Fake.AssertCount +#### FakeQueue.AssertCount -AssertCount fails when total dispatch count is not expected. +AssertCount fails when the direct dispatch count is not expected. ```go -f := queuefake.New() -q := f.Queue() -_ = q.Dispatch(queue.NewJob("a")) -_ = q.Dispatch(queue.NewJob("b")) -f.AssertCount(t, 2) +fake := queue.NewFake() +_ = fake.Dispatch(queue.NewJob("emails:send")) +fake.AssertCount(t, 1) ``` -#### Fake.AssertDispatched +#### FakeQueue.AssertDispatched AssertDispatched fails when jobType was not dispatched. ```go -f := queuefake.New() -_ = f.Queue().Dispatch(queue.NewJob("emails:send")) -f.AssertDispatched(t, "emails:send") +fake := queue.NewFake() +_ = fake.Dispatch(queue.NewJob("emails:send")) +fake.AssertDispatched(t, "emails:send") ``` -#### Fake.AssertDispatchedOn +#### FakeQueue.AssertDispatchedOn AssertDispatchedOn fails when jobType was not dispatched on queueName. ```go -f := queuefake.New() -_ = f.Queue().Dispatch(queue.NewJob("emails:send").OnQueue("critical")) -f.AssertDispatchedOn(t, "critical", "emails:send") +fake := queue.NewFake() +_ = fake.Dispatch( + queue.NewJob("emails:send"). + OnQueue("critical"), +) +fake.AssertDispatchedOn(t, "critical", "emails:send") ``` -#### Fake.AssertDispatchedTimes +#### FakeQueue.AssertDispatchedTimes AssertDispatchedTimes fails when jobType dispatch count does not match expected. ```go -f := queuefake.New() -q := f.Queue() -_ = q.Dispatch(queue.NewJob("emails:send")) -_ = q.Dispatch(queue.NewJob("emails:send")) -f.AssertDispatchedTimes(t, "emails:send", 2) +fake := queue.NewFake() +_ = fake.Dispatch(queue.NewJob("emails:send")) +_ = fake.Dispatch(queue.NewJob("emails:send")) +fake.AssertDispatchedTimes(t, "emails:send", 2) ``` -#### Fake.AssertNotDispatched +#### FakeQueue.AssertNotDispatched AssertNotDispatched fails when jobType was dispatched. ```go -f := queuefake.New() -f.AssertNotDispatched(t, "emails:send") +fake := queue.NewFake() +_ = fake.Dispatch(queue.NewJob("emails:send")) +fake.AssertNotDispatched(t, "emails:cancel") ``` -#### Fake.AssertNothingBatched - -AssertNothingBatched fails if any workflow batch was recorded. +#### FakeQueue.AssertNothingBatched -```go -f := queuefake.New() -f.AssertNothingBatched(t) -``` +AssertNothingBatched fails when any accepted batch was recorded. -#### Fake.AssertNothingDispatched +#### FakeQueue.AssertNothingDispatched -AssertNothingDispatched fails when any dispatch was recorded. +AssertNothingDispatched fails when any direct dispatch was recorded. ```go -f := queuefake.New() -f.AssertNothingDispatched(t) +fake := queue.NewFake() +fake.AssertNothingDispatched(t) ``` -#### Fake.AssertNothingWorkflowDispatched +#### FakeQueue.Batch + +Batch creates a fake batch backed by the production workflow builder and +records it only when Dispatch accepts all initial member deliveries. Fluent +function callbacks are accepted for compatibility but are not retained in +fake runtime state or executed. + +#### FakeQueue.BatchRecords -AssertNothingWorkflowDispatched fails when any workflow dispatch was recorded. +BatchRecords returns isolated creation records for accepted fake batches. ```go -f := queuefake.New() -f.AssertNothingWorkflowDispatched(t) +fake := queue.NewFake() +_, _ = fake.Batch( + queue.NewJob("emails:first"), + queue.NewJob("emails:second"), +).Name("nightly").AllowFailures().Dispatch(context.Background()) +record := fake.BatchRecords()[0] +fmt.Println(record.Name, len(record.Jobs), record.AllowFailed) +// Output: nightly 2 true ``` -#### Fake.AssertWorkflowDispatched +#### FakeQueue.Chain + +Chain creates a fake chain backed by the production workflow builder and +records it only when Dispatch accepts its initial delivery. Fluent function +callbacks are accepted for compatibility but are not retained in fake runtime +state or executed. + +#### FakeQueue.ChainRecords -AssertWorkflowDispatched fails when jobType was not workflow-dispatched. +ChainRecords returns isolated creation records for accepted fake chains. ```go -f := queuefake.New() -_, _ = f.Workflow().Chain(bus.NewJob("a", nil)).Dispatch(nil) -f.AssertWorkflowDispatched(t, "a") +fake := queue.NewFake() +_, _ = fake.Chain( + queue.NewJob("reports:build"), + queue.NewJob("reports:publish"), +).OnQueue("workflow").Dispatch(context.Background()) +record := fake.ChainRecords()[0] +fmt.Println(len(record.Nodes), record.Queue) +// Output: 2 workflow ``` -#### Fake.AssertWorkflowDispatchedOn +#### FakeQueue.Dispatch -AssertWorkflowDispatchedOn fails when jobType was not workflow-dispatched on queueName. +Dispatch records a typed job payload in-memory using the fake default queue. ```go -f := queuefake.New() -_, _ = f.Workflow().Chain(bus.NewJob("a", nil)).OnQueue("critical").Dispatch(nil) -f.AssertWorkflowDispatchedOn(t, "critical", "a") +fake := queue.NewFake() +err := fake.Dispatch(queue.NewJob("emails:send").OnQueue("default")) ``` -#### Fake.AssertWorkflowDispatchedTimes +#### FakeQueue.Driver -AssertWorkflowDispatchedTimes fails when workflow dispatch count for jobType does not match expected. +Driver returns the active queue driver. ```go -f := queuefake.New() -wf := f.Workflow() -_, _ = wf.Chain(bus.NewJob("a", nil)).Dispatch(nil) -_, _ = wf.Chain(bus.NewJob("a", nil)).Dispatch(nil) -f.AssertWorkflowDispatchedTimes(t, "a", 2) +fake := queue.NewFake() +driver := fake.Driver() ``` -#### Fake.AssertWorkflowNotDispatched +#### FakeQueue.FindBatch -AssertWorkflowNotDispatched fails when jobType was workflow-dispatched. +FindBatch returns workflow state created by the fake's production engine. -```go -f := queuefake.New() -f.AssertWorkflowNotDispatched(t, "emails:send") -``` +#### FakeQueue.FindChain + +FindChain returns workflow state created by the fake's production engine. -#### Fake.Count +#### NewFake -Count returns the total number of recorded dispatches. +NewFake creates the canonical fake used directly and by deprecated testing adapters. ```go -f := queuefake.New() -q := f.Queue() -_ = q.Dispatch(queue.NewJob("a")) -_ = q.Dispatch(queue.NewJob("b")) -_ = f.Count() +fake := queue.NewFake() +_ = fake.Dispatch( + queue.NewJob("emails:send"). + Payload(map[string]any{"id": 1}). + OnQueue("critical"), +) +records := fake.Records() +fmt.Println(len(records), records[0].Queue, records[0].Job.Type) +// Output: 1 critical emails:send ``` -#### Fake.CountJob +#### FakeQueue.Prune + +Prune removes terminal workflow state while retaining fake dispatch records. + +#### FakeQueue.Ready -CountJob returns how many times a job type was dispatched. +Ready validates fake queue readiness. ```go -f := queuefake.New() -q := f.Queue() -_ = q.Dispatch(queue.NewJob("emails:send")) -_ = q.Dispatch(queue.NewJob("emails:send")) -_ = f.CountJob("emails:send") +fake := queue.NewFake() +fmt.Println(fake.Ready(context.Background()) == nil) +// Output: true ``` -#### Fake.CountOn +#### FakeQueue.Records -CountOn returns how many times a job type was dispatched on a queue. +Records returns isolated records for accepted direct dispatches. +Chain and batch creation is available through ChainRecords and BatchRecords. ```go -f := queuefake.New() -q := f.Queue() -_ = q.Dispatch(queue.NewJob("emails:send").OnQueue("critical")) -_ = f.CountOn("critical", "emails:send") +fake := queue.NewFake() +_ = fake.Dispatch(queue.NewJob("emails:send").OnQueue("default")) +records := fake.Records() +fmt.Println(len(records), records[0].Job.Type) +// Output: 1 emails:send ``` -#### queuefake.New +#### FakeQueue.Register -New creates a fake queue harness backed by queue.NewFake(). +Register is a compatibility no-op because the recording fake never executes handlers. ```go -f := queuefake.New() -q := f.Queue() -_ = q.Dispatch(queue.NewJob("emails:send").OnQueue("default")) -f.AssertDispatched(t, "emails:send") -f.AssertCount(t, 1) +fake := queue.NewFake() +fake.Register("emails:send", func(context.Context, queue.Job) error { return nil }) ``` -#### Fake.Queue +#### FakeQueue.Reset -Queue returns the queue fake to inject into code under test. +Reset clears direct dispatches and all workflow records through every fake view. ```go -f := queuefake.New() -q := f.Queue() -_ = q.Dispatch(queue.NewJob("emails:send").OnQueue("default")) +fake := queue.NewFake() +_ = fake.Dispatch(queue.NewJob("emails:send").OnQueue("default")) +fmt.Println(len(fake.Records())) +fake.Reset() +fmt.Println(len(fake.Records())) +// Output: +// 1 +// 0 ``` -#### Fake.Records +#### FakeQueue.Shutdown -Records returns a copy of recorded dispatches. +Shutdown is a compatibility no-op because the recording fake owns no worker resources. ```go -f := queuefake.New() -_ = f.Queue().Dispatch(queue.NewJob("emails:send")) -records := f.Records() +fake := queue.NewFake() +err := fake.Shutdown(context.Background()) ``` -#### Fake.Reset +#### FakeQueue.StartWorkers -Reset clears recorded dispatches. +StartWorkers is a compatibility no-op because the recording fake owns no workers. ```go -f := queuefake.New() -q := f.Queue() -_ = q.Dispatch(queue.NewJob("emails:send")) -f.Reset() -f.AssertNothingDispatched(t) +fake := queue.NewFake() +err := fake.StartWorkers(context.Background()) ``` -#### Fake.Workflow +#### FakeQueue.WithContext + +WithContext returns a derived fake queue handle bound to ctx. + +#### FakeQueue.Workers -Workflow returns the workflow/orchestration fake for chain/batch assertions. +Workers preserves fluent lifecycle compatibility without creating workers. ```go -f := queuefake.New() -wf := f.Workflow() -_, _ = wf.Chain( - bus.NewJob("a", nil), - bus.NewJob("b", nil), -).Dispatch(context.Background()) -f.AssertChained(t, []string{"a", "b"}) +fake := queue.NewFake() +q := fake.Workers(4) +fmt.Println(q != nil) +// Output: true ``` diff --git a/admin.go b/admin.go index d82089e..4c4bbfb 100644 --- a/admin.go +++ b/admin.go @@ -262,14 +262,22 @@ func resolveQueueAdmin(v any) QueueAdmin { return nil } if admin, ok := rt.common.inner.(QueueAdmin); ok { - return queueAdminWithNamespace{admin: admin, common: rt.common} + return queueAdminWithNamespace{ + admin: admin, + common: rt.common, + lease: func(ctx context.Context) (func(), error) { return rt.acquireOperation(ctx, false) }, + } } case *externalQueueRuntime: if rt == nil || rt.common == nil { return nil } if admin, ok := rt.common.inner.(QueueAdmin); ok { - return queueAdminWithNamespace{admin: admin, common: rt.common} + return queueAdminWithNamespace{ + admin: admin, + common: rt.common, + lease: func(ctx context.Context) (func(), error) { return rt.acquireOperation(ctx, false) }, + } } default: if admin, ok := raw.(QueueAdmin); ok { @@ -290,14 +298,22 @@ func resolveQueueHistory(v any) QueueHistoryProvider { return nil } if history, ok := rt.common.inner.(QueueHistoryProvider); ok { - return queueHistoryWithNamespace{history: history, common: rt.common} + return queueHistoryWithNamespace{ + history: history, + common: rt.common, + lease: func(ctx context.Context) (func(), error) { return rt.acquireOperation(ctx, false) }, + } } case *externalQueueRuntime: if rt == nil || rt.common == nil { return nil } if history, ok := rt.common.inner.(QueueHistoryProvider); ok { - return queueHistoryWithNamespace{history: history, common: rt.common} + return queueHistoryWithNamespace{ + history: history, + common: rt.common, + lease: func(ctx context.Context) (func(), error) { return rt.acquireOperation(ctx, false) }, + } } default: if history, ok := raw.(QueueHistoryProvider); ok { @@ -310,42 +326,95 @@ func resolveQueueHistory(v any) QueueHistoryProvider { type queueAdminWithNamespace struct { admin QueueAdmin common *queueCommon + lease func(context.Context) (func(), error) } func (a queueAdminWithNamespace) ListJobs(ctx context.Context, opts ListJobsOptions) (ListJobsResult, error) { + release, err := a.acquire(ctx) + if err != nil { + return ListJobsResult{}, err + } + defer release() opts.Queue = a.common.physicalQueueNameOrDefault(opts.Queue) return a.admin.ListJobs(ctx, opts) } func (a queueAdminWithNamespace) RetryJob(ctx context.Context, queueName, jobID string) error { + release, err := a.acquire(ctx) + if err != nil { + return err + } + defer release() return a.admin.RetryJob(ctx, a.common.physicalQueueNameOrDefault(queueName), jobID) } func (a queueAdminWithNamespace) CancelJob(ctx context.Context, jobID string) error { + release, err := a.acquire(ctx) + if err != nil { + return err + } + defer release() return a.admin.CancelJob(ctx, jobID) } func (a queueAdminWithNamespace) DeleteJob(ctx context.Context, queueName, jobID string) error { + release, err := a.acquire(ctx) + if err != nil { + return err + } + defer release() return a.admin.DeleteJob(ctx, a.common.physicalQueueNameOrDefault(queueName), jobID) } func (a queueAdminWithNamespace) ClearQueue(ctx context.Context, queueName string) error { + release, err := a.acquire(ctx) + if err != nil { + return err + } + defer release() return a.admin.ClearQueue(ctx, a.common.physicalQueueNameOrDefault(queueName)) } func (a queueAdminWithNamespace) History(ctx context.Context, queueName string, window QueueHistoryWindow) ([]QueueHistoryPoint, error) { + release, err := a.acquire(ctx) + if err != nil { + return nil, err + } + defer release() return a.admin.History(ctx, a.common.physicalQueueNameOrDefault(queueName), window) } +// acquire reserves runtime resources for one administrative operation. +func (a queueAdminWithNamespace) acquire(ctx context.Context) (func(), error) { + if a.lease == nil { + return func() {}, nil + } + return a.lease(ctx) +} + type queueHistoryWithNamespace struct { history QueueHistoryProvider common *queueCommon + lease func(context.Context) (func(), error) } func (h queueHistoryWithNamespace) History(ctx context.Context, queueName string, window QueueHistoryWindow) ([]QueueHistoryPoint, error) { + release, err := h.acquire(ctx) + if err != nil { + return nil, err + } + defer release() return h.history.History(ctx, h.common.physicalQueueNameOrDefault(queueName), window) } +// acquire reserves runtime resources for one history operation. +func (h queueHistoryWithNamespace) acquire(ctx context.Context) (func(), error) { + if h.lease == nil { + return func() {}, nil + } + return h.lease(ctx) +} + // ListJobs lists jobs via queue admin capability when supported. // @group Admin // diff --git a/admin_lifecycle_test.go b/admin_lifecycle_test.go new file mode 100644 index 0000000..dda6fbc --- /dev/null +++ b/admin_lifecycle_test.go @@ -0,0 +1,251 @@ +package queue + +import ( + "context" + "errors" + "testing" +) + +// lifecycleAdminBackend records whether lifecycle-gated calls reach driver resources. +type lifecycleAdminBackend struct { + activeLeases *int + callErr error + calls int + leaseViolated bool + queues []string +} + +// recordCall verifies driver resources are used only while the runtime lease is active. +func (b *lifecycleAdminBackend) recordCall() error { + b.calls++ + if b.activeLeases != nil && *b.activeLeases != 1 { + b.leaseViolated = true + } + return b.callErr +} + +// recordQueueCall retains the physical queue while applying the common lease assertion. +func (b *lifecycleAdminBackend) recordQueueCall(queueName string) error { + b.queues = append(b.queues, queueName) + return b.recordCall() +} + +// Driver identifies the backend used by the admin wrapper tests. +func (b *lifecycleAdminBackend) Driver() Driver { return DriverRedis } + +// Dispatch satisfies queueBackend without affecting admin call accounting. +func (b *lifecycleAdminBackend) Dispatch(context.Context, Job) error { return nil } + +// Shutdown satisfies queueBackend without affecting admin call accounting. +func (b *lifecycleAdminBackend) Shutdown(context.Context) error { return nil } + +// ListJobs records the physical queue passed through the namespace wrapper. +func (b *lifecycleAdminBackend) ListJobs(_ context.Context, opts ListJobsOptions) (ListJobsResult, error) { + return ListJobsResult{}, b.recordQueueCall(opts.Queue) +} + +// RetryJob records the physical queue passed through the namespace wrapper. +func (b *lifecycleAdminBackend) RetryJob(_ context.Context, queueName, _ string) error { + return b.recordQueueCall(queueName) +} + +// CancelJob records that the queue-independent operation reached the backend. +func (b *lifecycleAdminBackend) CancelJob(context.Context, string) error { + return b.recordCall() +} + +// DeleteJob records the physical queue passed through the namespace wrapper. +func (b *lifecycleAdminBackend) DeleteJob(_ context.Context, queueName, _ string) error { + return b.recordQueueCall(queueName) +} + +// ClearQueue records the physical queue passed through the namespace wrapper. +func (b *lifecycleAdminBackend) ClearQueue(_ context.Context, queueName string) error { + return b.recordQueueCall(queueName) +} + +// History records the physical queue passed through the namespace wrapper. +func (b *lifecycleAdminBackend) History(_ context.Context, queueName string, _ QueueHistoryWindow) ([]QueueHistoryPoint, error) { + return []QueueHistoryPoint{{Processed: 1}}, b.recordQueueCall(queueName) +} + +// TestQueueAdminNamespaceLeaseCoversEveryOperation verifies backend access remains leased through all admin calls. +func TestQueueAdminNamespaceLeaseCoversEveryOperation(t *testing.T) { + activeLeases := 0 + backend := &lifecycleAdminBackend{activeLeases: &activeLeases} + leases := 0 + releases := 0 + admin := queueAdminWithNamespace{ + admin: backend, + common: &queueCommon{cfg: Config{DefaultQueue: "billing_default"}}, + lease: func(context.Context) (func(), error) { + leases++ + activeLeases++ + return func() { + activeLeases-- + releases++ + }, nil + }, + } + + if _, err := admin.ListJobs(context.Background(), ListJobsOptions{Queue: "reports"}); err != nil { + t.Fatalf("ListJobs: %v", err) + } + if err := admin.RetryJob(context.Background(), "reports", "job-1"); err != nil { + t.Fatalf("RetryJob: %v", err) + } + if err := admin.CancelJob(context.Background(), "job-1"); err != nil { + t.Fatalf("CancelJob: %v", err) + } + if err := admin.DeleteJob(context.Background(), "reports", "job-1"); err != nil { + t.Fatalf("DeleteJob: %v", err) + } + if err := admin.ClearQueue(context.Background(), "reports"); err != nil { + t.Fatalf("ClearQueue: %v", err) + } + if _, err := admin.History(context.Background(), "reports", QueueHistoryHour); err != nil { + t.Fatalf("History: %v", err) + } + + if leases != 6 || releases != 6 || activeLeases != 0 || backend.calls != 6 { + t.Fatalf("leases/releases/active/backend calls = %d/%d/%d/%d, want 6/6/0/6", leases, releases, activeLeases, backend.calls) + } + if backend.leaseViolated { + t.Fatal("backend operation ran without exactly one active runtime lease") + } + if len(backend.queues) != 5 { + t.Fatalf("physical queue calls = %v, want five queue-scoped calls", backend.queues) + } + for _, queueName := range backend.queues { + if queueName != "billing_reports" { + t.Fatalf("physical queue = %q, want billing_reports", queueName) + } + } +} + +// TestQueueAdminNamespaceReleasesLeaseAfterBackendFailure verifies deferred +// release is not conditional on a successful driver operation. +func TestQueueAdminNamespaceReleasesLeaseAfterBackendFailure(t *testing.T) { + activeLeases := 0 + callErr := errors.New("admin backend unavailable") + backend := &lifecycleAdminBackend{activeLeases: &activeLeases, callErr: callErr} + releases := 0 + admin := queueAdminWithNamespace{ + admin: backend, + common: &queueCommon{cfg: Config{DefaultQueue: "default"}}, + lease: func(context.Context) (func(), error) { + activeLeases++ + return func() { + activeLeases-- + releases++ + }, nil + }, + } + + if err := admin.CancelJob(context.Background(), "job-1"); !errors.Is(err, callErr) { + t.Fatalf("CancelJob error = %v, want %v", err, callErr) + } + if backend.leaseViolated || activeLeases != 0 || releases != 1 || backend.calls != 1 { + t.Fatalf("failure lease state = violated:%t active:%d releases:%d calls:%d", backend.leaseViolated, activeLeases, releases, backend.calls) + } +} + +// TestQueueAdminNamespaceLeaseFailureRejectsEveryOperation verifies shutdown rejection cannot reach driver resources. +func TestQueueAdminNamespaceLeaseFailureRejectsEveryOperation(t *testing.T) { + backend := &lifecycleAdminBackend{} + leaseErr := errors.New("runtime is draining") + admin := queueAdminWithNamespace{ + admin: backend, + common: &queueCommon{cfg: Config{DefaultQueue: "default"}}, + lease: func(context.Context) (func(), error) { return nil, leaseErr }, + } + + tests := []struct { + name string + call func() error + }{ + { + name: "list jobs", + call: func() error { + _, err := admin.ListJobs(context.Background(), ListJobsOptions{}) + return err + }, + }, + {name: "retry job", call: func() error { return admin.RetryJob(context.Background(), "default", "job-1") }}, + {name: "cancel job", call: func() error { return admin.CancelJob(context.Background(), "job-1") }}, + {name: "delete job", call: func() error { return admin.DeleteJob(context.Background(), "default", "job-1") }}, + {name: "clear queue", call: func() error { return admin.ClearQueue(context.Background(), "default") }}, + { + name: "history", + call: func() error { + _, err := admin.History(context.Background(), "default", QueueHistoryHour) + return err + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if err := test.call(); !errors.Is(err, leaseErr) { + t.Fatalf("error = %v, want %v", err, leaseErr) + } + }) + } + if backend.calls != 0 { + t.Fatalf("backend calls = %d, want none after lease rejection", backend.calls) + } + + history := queueHistoryWithNamespace{ + history: backend, + common: &queueCommon{cfg: Config{DefaultQueue: "default"}}, + lease: func(context.Context) (func(), error) { return nil, leaseErr }, + } + if _, err := history.History(context.Background(), "default", QueueHistoryHour); !errors.Is(err, leaseErr) { + t.Fatalf("history-only error = %v, want %v", err, leaseErr) + } + if backend.calls != 0 { + t.Fatalf("backend calls = %d, want none after history lease rejection", backend.calls) + } +} + +// TestQueueAdminNamespaceOptionalLeasePreservesCompatibility verifies wrappers constructed without lifecycle wiring still delegate safely. +func TestQueueAdminNamespaceOptionalLeasePreservesCompatibility(t *testing.T) { + backend := &lifecycleAdminBackend{} + common := &queueCommon{cfg: Config{DefaultQueue: "billing_default"}} + admin := queueAdminWithNamespace{admin: backend, common: common} + if err := admin.CancelJob(context.Background(), "job-1"); err != nil { + t.Fatalf("CancelJob without lease: %v", err) + } + + history := queueHistoryWithNamespace{history: backend, common: common} + if _, err := history.History(context.Background(), "reports", QueueHistoryHour); err != nil { + t.Fatalf("History without lease: %v", err) + } + if backend.calls != 2 { + t.Fatalf("backend calls = %d, want 2", backend.calls) + } +} + +// TestResolvedNativeQueueAdminRejectsOperationsAfterShutdown verifies resolver-installed leases honor the runtime lifecycle gate. +func TestResolvedNativeQueueAdminRejectsOperationsAfterShutdown(t *testing.T) { + backend := &lifecycleAdminBackend{} + runtime := &nativeQueueRuntime{ + common: &queueCommon{ + inner: backend, + cfg: Config{DefaultQueue: "default"}, + }, + nativeQueueRuntimeState: &nativeQueueRuntimeState{ + registered: make(map[string]Handler), + closed: true, + }, + } + admin := resolveQueueAdmin(runtime) + if admin == nil { + t.Fatal("resolved native admin is nil") + } + if err := admin.CancelJob(context.Background(), "job-1"); !errors.Is(err, ErrQueuerShuttingDown) { + t.Fatalf("CancelJob error = %v, want %v", err, ErrQueuerShuttingDown) + } + if backend.calls != 0 { + t.Fatalf("backend calls = %d, want none after shutdown", backend.calls) + } +} diff --git a/admin_test.go b/admin_test.go index d532814..20ff6db 100644 --- a/admin_test.go +++ b/admin_test.go @@ -114,6 +114,9 @@ func TestQueueAdminHelpersPhysicalizeTargetQueues(t *testing.T) { cfg: Config{DefaultQueue: "billing_default"}, driver: DriverRedis, }, + externalQueueRuntimeState: &externalQueueRuntimeState{ + registered: map[string]Handler{}, + }, }, } diff --git a/bus/batch.go b/bus/batch.go deleted file mode 100644 index 6b38b65..0000000 --- a/bus/batch.go +++ /dev/null @@ -1,416 +0,0 @@ -package bus - -import ( - "context" - "errors" - - "github.com/goforj/queue/busruntime" -) - -type BatchBuilder interface { - // Name sets a display name for the batch. - // @group Batching - // - // Example: set batch name - // - // batchID, _ := b.Batch(bus.NewJob("a", nil)).Name("nightly").Dispatch(context.Background()) - // _ = batchID - Name(name string) BatchBuilder - // OnQueue applies a default queue to batch jobs that do not set one. - // @group Batching - // - // Example: set batch queue - // - // batchID, _ := b.Batch(bus.NewJob("a", nil)).OnQueue("critical").Dispatch(context.Background()) - // _ = batchID - OnQueue(queue string) BatchBuilder - // AllowFailures keeps the batch running when individual jobs fail. - // @group Batching - // - // Example: allow failures - // - // batchID, _ := b.Batch(bus.NewJob("a", nil)).AllowFailures().Dispatch(context.Background()) - // _ = batchID - AllowFailures() BatchBuilder - // Progress registers a callback invoked as jobs complete. - // @group Batching - // - // Example: progress callback - // - // batchID, _ := b.Batch(bus.NewJob("a", nil)). - // Progress(func(context.Context, bus.BatchState) error { return nil }). - // Dispatch(context.Background()) - // _ = batchID - Progress(fn func(ctx context.Context, st BatchState) error) BatchBuilder - // Then registers a callback invoked once when batch succeeds. - // @group Batching - // - // Example: then callback - // - // batchID, _ := b.Batch(bus.NewJob("a", nil)). - // Then(func(context.Context, bus.BatchState) error { return nil }). - // Dispatch(context.Background()) - // _ = batchID - Then(fn func(ctx context.Context, st BatchState) error) BatchBuilder - // Catch registers a callback invoked when batch encounters a failure. - // @group Batching - // - // Example: catch callback - // - // batchID, _ := b.Batch(bus.NewJob("a", nil)). - // Catch(func(context.Context, bus.BatchState, error) error { return nil }). - // Dispatch(context.Background()) - // _ = batchID - Catch(fn func(ctx context.Context, st BatchState, err error) error) BatchBuilder - // Finally registers a callback invoked once when batch reaches terminal state. - // @group Batching - // - // Example: finally callback - // - // batchID, _ := b.Batch(bus.NewJob("a", nil)). - // Finally(func(context.Context, bus.BatchState) error { return nil }). - // Dispatch(context.Background()) - // _ = batchID - Finally(fn func(ctx context.Context, st BatchState) error) BatchBuilder - // Dispatch creates and starts the batch workflow. - // @group Batching - // - // Example: dispatch batch - // - // batchID, _ := b.Batch(bus.NewJob("a", nil), bus.NewJob("b", nil)).Dispatch(context.Background()) - // _ = batchID - Dispatch(ctx context.Context) (string, error) -} - -type batchBuilder struct { - r *runtime - jobs []Job - name string - queue string - allowFailed bool - progress func(ctx context.Context, st BatchState) error - then func(ctx context.Context, st BatchState) error - catch func(ctx context.Context, st BatchState, err error) error - finally func(ctx context.Context, st BatchState) error -} - -func (b *batchBuilder) Name(name string) BatchBuilder { b.name = name; return b } -func (b *batchBuilder) OnQueue(queue string) BatchBuilder { - b.queue = queue - return b -} -func (b *batchBuilder) AllowFailures() BatchBuilder { - b.allowFailed = true - return b -} -func (b *batchBuilder) Progress(fn func(ctx context.Context, st BatchState) error) BatchBuilder { - b.progress = fn - return b -} -func (b *batchBuilder) Then(fn func(ctx context.Context, st BatchState) error) BatchBuilder { - b.then = fn - return b -} -func (b *batchBuilder) Catch(fn func(ctx context.Context, st BatchState, err error) error) BatchBuilder { - b.catch = fn - return b -} -func (b *batchBuilder) Finally(fn func(ctx context.Context, st BatchState) error) BatchBuilder { - b.finally = fn - return b -} - -func (b *batchBuilder) Dispatch(ctx context.Context) (string, error) { - if len(b.jobs) == 0 { - return "", errors.New("batch requires at least one job") - } - batchID := newID("bat") - dispatchID := newID("dsp") - jobs := make([]BatchJob, 0, len(b.jobs)) - for _, job := range b.jobs { - wj, err := toWireJob(job) - if err != nil { - return "", err - } - if b.queue != "" && wj.Options.Queue == "" { - wj.Options.Queue = b.queue - } - jobs = append(jobs, BatchJob{ - JobID: newID("job"), - Job: wj, - }) - } - if err := b.r.store.CreateBatch(ctx, BatchRecord{ - BatchID: batchID, - DispatchID: dispatchID, - Name: b.name, - Queue: b.queue, - AllowFailed: b.allowFailed, - Jobs: jobs, - CreatedAt: b.r.now(), - }); err != nil { - return "", err - } - - b.r.mu.Lock() - b.r.batchCallbacks[batchID] = batchCallbacks{ - progress: b.progress, - then: b.then, - catch: b.catch, - finally: b.finally, - } - b.r.mu.Unlock() - - b.r.emit(ctx, Event{SchemaVersion: schemaVersion, EventID: newID("evt"), Kind: EventBatchStarted, DispatchID: dispatchID, BatchID: batchID, Queue: b.queue, Time: b.r.now()}) - for _, job := range jobs { - if err := b.r.dispatchEnvelope(ctx, internalJobBatchJob, envelope{ - SchemaVersion: schemaVersion, - DispatchID: dispatchID, - Kind: "batch_job", - BatchID: batchID, - JobID: job.JobID, - Job: job.Job, - }); err != nil { - if st, stErr := b.r.store.GetBatch(ctx, batchID); stErr == nil && (st.Completed || st.Processed > 0 || st.Failed > 0) { - return batchID, err - } - _ = b.r.store.CancelBatch(ctx, batchID) - st, stErr := b.r.store.GetBatch(ctx, batchID) - if stErr == nil { - _ = b.r.invokeBatchCatch(ctx, st, err) - _ = b.r.invokeBatchFinally(ctx, st) - } - b.r.emit(ctx, Event{SchemaVersion: schemaVersion, EventID: newID("evt"), Kind: EventBatchFailed, DispatchID: dispatchID, BatchID: batchID, Time: b.r.now(), Err: err}) - b.r.emit(ctx, Event{SchemaVersion: schemaVersion, EventID: newID("evt"), Kind: EventBatchCancelled, DispatchID: dispatchID, BatchID: batchID, Time: b.r.now()}) - return batchID, err - } - } - return batchID, nil -} - -type batchCallbacks struct { - progress func(ctx context.Context, st BatchState) error - then func(ctx context.Context, st BatchState) error - catch func(ctx context.Context, st BatchState, err error) error - finally func(ctx context.Context, st BatchState) error -} - -func (r *runtime) handleInternalBatchJob(ctx context.Context, job busruntime.InboundJob) error { - var env envelope - if err := job.Bind(&env); err != nil { - return err - } - _ = r.store.MarkBatchJobStarted(ctx, env.BatchID, env.JobID) - - err := r.executeWireJob(ctx, env) - if err != nil { - st, done, markErr := r.store.MarkBatchJobFailed(ctx, env.BatchID, env.JobID, err) - if markErr != nil { - return markErr - } - r.emit(ctx, Event{SchemaVersion: schemaVersion, EventID: newID("evt"), Kind: EventBatchFailed, DispatchID: env.DispatchID, BatchID: env.BatchID, JobID: env.JobID, JobType: env.Job.Type, Queue: env.Job.Options.Queue, Time: r.now(), Err: err}) - if st.Cancelled { - r.emit(ctx, Event{SchemaVersion: schemaVersion, EventID: newID("evt"), Kind: EventBatchCancelled, DispatchID: env.DispatchID, BatchID: env.BatchID, Time: r.now()}) - } - _ = r.dispatchCallback(ctx, env, "batch_catch", err) - r.invokeBatchProgress(ctx, st) - if done { - _ = r.dispatchCallback(ctx, env, "batch_finally", nil) - } - return err - } - st, done, markErr := r.store.MarkBatchJobSucceeded(ctx, env.BatchID, env.JobID) - if markErr != nil { - return markErr - } - r.emit(ctx, Event{SchemaVersion: schemaVersion, EventID: newID("evt"), Kind: EventBatchProgressed, DispatchID: env.DispatchID, BatchID: env.BatchID, JobID: env.JobID, JobType: env.Job.Type, Queue: env.Job.Options.Queue, Time: r.now()}) - r.invokeBatchProgress(ctx, st) - if done { - r.emit(ctx, Event{SchemaVersion: schemaVersion, EventID: newID("evt"), Kind: EventBatchCompleted, DispatchID: env.DispatchID, BatchID: env.BatchID, Time: r.now()}) - _ = r.dispatchCallback(ctx, env, "batch_then", nil) - _ = r.dispatchCallback(ctx, env, "batch_finally", nil) - } - return nil -} - -func (r *runtime) invokeBatchProgress(ctx context.Context, st BatchState) { - r.mu.RLock() - cb := r.batchCallbacks[st.BatchID] - r.mu.RUnlock() - if cb.progress != nil { - _ = cb.progress(ctx, st) - } -} - -func (r *runtime) invokeBatchThen(ctx context.Context, st BatchState) error { - key := "batch_then:" + st.BatchID - ok, onceErr := r.callbackOnce(ctx, key) - if onceErr != nil { - return onceErr - } - if !ok { - return nil - } - r.mu.RLock() - cb := r.batchCallbacks[st.BatchID] - r.mu.RUnlock() - if cb.then != nil { - _ = cb.then(ctx, st) - } - return nil -} - -func (r *runtime) invokeBatchCatch(ctx context.Context, st BatchState, err error) error { - key := "batch_catch:" + st.BatchID - ok, onceErr := r.callbackOnce(ctx, key) - if onceErr != nil { - return onceErr - } - if !ok { - return nil - } - r.mu.RLock() - cb := r.batchCallbacks[st.BatchID] - r.mu.RUnlock() - if cb.catch != nil { - _ = cb.catch(ctx, st, err) - } - return nil -} - -func (r *runtime) invokeBatchFinally(ctx context.Context, st BatchState) error { - key := "batch_finally:" + st.BatchID - ok, onceErr := r.callbackOnce(ctx, key) - if onceErr != nil { - return onceErr - } - if !ok { - return nil - } - r.mu.RLock() - cb := r.batchCallbacks[st.BatchID] - r.mu.RUnlock() - if cb.finally != nil { - _ = cb.finally(ctx, st) - } - r.mu.Lock() - delete(r.batchCallbacks, st.BatchID) - r.mu.Unlock() - return nil -} - -func (r *runtime) callbackOnce(ctx context.Context, key string) (bool, error) { - return r.store.MarkCallbackInvoked(ctx, key) -} - -func (r *runtime) handleInternalCallback(ctx context.Context, job busruntime.InboundJob) error { - var env envelope - if err := job.Bind(&env); err != nil { - return err - } - cbErr := error(nil) - if env.Error != "" { - cbErr = errors.New(env.Error) - } - start := r.now() - r.emit(ctx, Event{ - SchemaVersion: schemaVersion, - EventID: newID("evt"), - Kind: EventCallbackStarted, - DispatchID: env.DispatchID, - JobID: env.JobID, - ChainID: env.ChainID, - BatchID: env.BatchID, - Queue: env.Job.Options.Queue, - Time: r.now(), - }) - var err error - switch env.CallbackKind { - case "chain_catch": - if env.ChainID == "" { - err = errors.New("chain callback requires chain_id") - break - } - st, stErr := r.store.GetChain(ctx, env.ChainID) - if stErr != nil { - err = stErr - break - } - err = r.invokeChainCatch(ctx, st, cbErr) - case "chain_finally": - if env.ChainID == "" { - err = errors.New("chain callback requires chain_id") - break - } - st, stErr := r.store.GetChain(ctx, env.ChainID) - if stErr != nil { - err = stErr - break - } - err = r.invokeChainFinally(ctx, st) - case "batch_catch": - if env.BatchID == "" { - err = errors.New("batch callback requires batch_id") - break - } - st, stErr := r.store.GetBatch(ctx, env.BatchID) - if stErr != nil { - err = stErr - break - } - err = r.invokeBatchCatch(ctx, st, cbErr) - case "batch_then": - if env.BatchID == "" { - err = errors.New("batch callback requires batch_id") - break - } - st, stErr := r.store.GetBatch(ctx, env.BatchID) - if stErr != nil { - err = stErr - break - } - err = r.invokeBatchThen(ctx, st) - case "batch_finally": - if env.BatchID == "" { - err = errors.New("batch callback requires batch_id") - break - } - st, stErr := r.store.GetBatch(ctx, env.BatchID) - if stErr != nil { - err = stErr - break - } - err = r.invokeBatchFinally(ctx, st) - default: - err = errors.New("unknown callback kind") - } - if err != nil { - r.emit(ctx, Event{ - SchemaVersion: schemaVersion, - EventID: newID("evt"), - Kind: EventCallbackFailed, - DispatchID: env.DispatchID, - JobID: env.JobID, - ChainID: env.ChainID, - BatchID: env.BatchID, - Queue: env.Job.Options.Queue, - Duration: r.now().Sub(start), - Time: r.now(), - Err: err, - }) - return err - } - r.emit(ctx, Event{ - SchemaVersion: schemaVersion, - EventID: newID("evt"), - Kind: EventCallbackSucceeded, - DispatchID: env.DispatchID, - JobID: env.JobID, - ChainID: env.ChainID, - BatchID: env.BatchID, - Queue: env.Job.Options.Queue, - Duration: r.now().Sub(start), - Time: r.now(), - }) - return nil -} diff --git a/bus/bus.go b/bus/bus.go index 5a4fa66..8d12b68 100644 --- a/bus/bus.go +++ b/bus/bus.go @@ -2,483 +2,649 @@ package bus import ( "context" - "crypto/rand" - "encoding/hex" "encoding/json" "errors" "fmt" - "sync" "time" - "github.com/goforj/queue/busruntime" + "github.com/goforj/queue" + "github.com/goforj/queue/internal/workflow" ) -const ( - schemaVersion = 1 - - internalJob = "bus:job" - internalJobChainNode = "bus:chain:node" - internalJobBatchJob = "bus:batch:job" - internalJobCallback = "bus:callback" -) +// ErrQueueOptionsUnsupported indicates that bus construction options cannot be +// retrofitted onto an already configured queue.Queue. +var ErrQueueOptionsUnsupported = errors.New("bus options cannot configure an existing queue.Queue") +// Bus is the legacy workflow orchestration contract. +// +// Deprecated: use queue.Queue. type Bus interface { + // Register binds a legacy workflow handler to a job type. Register(jobType string, handler Handler) - + // Dispatch submits one legacy workflow job. Dispatch(ctx context.Context, job Job) (DispatchResult, error) + // Chain creates a sequential workflow builder. Chain(jobs ...Job) ChainBuilder + // Batch creates an aggregate workflow builder. Batch(jobs ...Job) BatchBuilder - + // StartWorkers starts the underlying queue worker runtime. StartWorkers(ctx context.Context) error + // Shutdown stops the underlying queue worker runtime. Shutdown(ctx context.Context) error - + // FindBatch returns persisted batch state. FindBatch(ctx context.Context, batchID string) (BatchState, error) + // FindChain returns persisted chain state. FindChain(ctx context.Context, chainID string) (ChainState, error) + // Prune removes terminal workflow state older than the supplied time. Prune(ctx context.Context, before time.Time) error } -type Option func(*runtime) +// ChainBuilder configures and dispatches a sequential workflow. +// +// Deprecated: use queue.ChainBuilder. +type ChainBuilder interface { + // OnQueue applies a default queue to chain jobs without an explicit target. + OnQueue(queue string) ChainBuilder + // Catch registers the explicitly ephemeral chain failure callback. + Catch(fn func(ctx context.Context, st ChainState, err error) error) ChainBuilder + // Finally registers the explicitly ephemeral chain terminal callback. + Finally(fn func(ctx context.Context, st ChainState) error) ChainBuilder + // Dispatch persists and starts the chain workflow. + Dispatch(ctx context.Context) (string, error) +} + +// BatchBuilder configures and dispatches an aggregate workflow. +// +// Deprecated: use queue.BatchBuilder. +type BatchBuilder interface { + // Name assigns an application-facing label to the batch. + Name(name string) BatchBuilder + // OnQueue applies a default queue to batch jobs without an explicit target. + OnQueue(queue string) BatchBuilder + // AllowFailures keeps remaining members active after a terminal member failure. + AllowFailures() BatchBuilder + // Progress registers the explicitly ephemeral batch progress callback. + Progress(fn func(ctx context.Context, st BatchState) error) BatchBuilder + // Then registers the explicitly ephemeral batch success callback. + Then(fn func(ctx context.Context, st BatchState) error) BatchBuilder + // Catch registers the explicitly ephemeral batch failure callback. + Catch(fn func(ctx context.Context, st BatchState, err error) error) BatchBuilder + // Finally registers the explicitly ephemeral batch terminal callback. + Finally(fn func(ctx context.Context, st BatchState) error) BatchBuilder + // Dispatch persists and starts the batch workflow. + Dispatch(ctx context.Context) (string, error) +} -// WithObserver installs an event observer for dispatch/job/chain/batch lifecycle hooks. -// @group Options +// Option configures the legacy raw-runtime construction route. // -// Example: attach observer +// Deprecated: configure queue.Queue directly. Options are rejected when New +// receives an already constructed queue.Queue because its engine already exists. +type Option func(*optionConfig) + +type optionConfig struct { + observer Observer + store Store + clock func() time.Time + middlewares []Middleware +} + +// WithObserver installs a legacy workflow observer on a raw-runtime bus. // -// observer := bus.ObserverFunc(func(event bus.Event) { -// _ = event.Kind -// }) -// b, _ := bus.New(q, bus.WithObserver(observer)) -// _ = b +// Deprecated: use queue.WithObserver. func WithObserver(observer Observer) Option { - return func(r *runtime) { - r.observer = observer + return func(config *optionConfig) { + config.observer = observer } } -// WithStore overrides the orchestration store used for chain/batch/callback state. -// @group Options +// WithStore selects the workflow store on a raw-runtime bus. // -// Example: custom store -// -// store := bus.NewMemoryStore() -// b, _ := bus.New(q, bus.WithStore(store)) -// _ = b +// Deprecated: use queue.WithStore. func WithStore(store Store) Option { - return func(r *runtime) { + return func(config *optionConfig) { if store != nil { - r.store = store + config.store = store } } } -// WithClock overrides the runtime clock used for event/state timestamps. -// @group Options -// -// Example: fixed clock +// WithClock selects the workflow clock on a raw-runtime bus. // -// fixed := time.Date(2026, time.January, 1, 0, 0, 0, 0, time.UTC) -// b, _ := bus.New(q, bus.WithClock(func() time.Time { return fixed })) -// _ = b +// Deprecated: use queue.WithClock. func WithClock(clock func() time.Time) Option { - return func(r *runtime) { + return func(config *optionConfig) { if clock != nil { - r.now = clock + config.clock = clock } } } -// WithMiddleware appends middleware to the runtime execution chain. -// @group Options +// WithMiddleware appends middleware to a raw-runtime bus. // -// Example: add middleware -// -// audit := bus.MiddlewareFunc(func(ctx context.Context, jc bus.Context, next bus.Next) error { -// return next(ctx, jc) -// }) -// skipHealth := bus.SkipWhen{ -// Predicate: func(_ context.Context, jc bus.Context) bool { return jc.JobType == "health:ping" }, -// } -// fatalize := bus.FailOnError{ -// When: func(err error) bool { return err != nil }, -// } -// b, _ := bus.New(q, bus.WithMiddleware(audit, skipHealth, fatalize)) -// _ = b +// Deprecated: use queue.WithMiddleware. func WithMiddleware(middlewares ...Middleware) Option { - return func(r *runtime) { - for _, m := range middlewares { - if m != nil { - r.middlewares = append(r.middlewares, m) + return func(config *optionConfig) { + for _, middleware := range middlewares { + if middleware != nil { + config.middlewares = append(config.middlewares, middleware) } } } } -// New creates a bus runtime using an in-memory orchestration store. -// @group Constructors -// -// Example: new bus runtime +// New returns a compatibility view over queue.Queue or constructs the retained +// low-level route when q implements busruntime.Runtime. // -// q, _ := queue.NewSync() -// b, _ := bus.New(q) -// b.Register("monitor:poll", func(context.Context, bus.Context) error { return nil }) -// _ = b.StartWorkers(context.Background()) -// defer b.Shutdown(context.Background()) -// type PollPayload struct { -// URL string `json:"url"` -// } -// _, _ = b.Dispatch(context.Background(), bus.NewJob("monitor:poll", PollPayload{ -// URL: "https://goforj.dev/health", -// })) +// Deprecated: construct and use queue.Queue directly. func New(q any, opts ...Option) (Bus, error) { - return NewWithStore(q, NewMemoryStore(), opts...) + if existing, ok := q.(*queue.Queue); ok { + if existing == nil { + return nil, errors.New("queue is required") + } + if hasConstructionOptions(opts) { + return nil, fmt.Errorf("%w: pass options to queue.New instead", ErrQueueOptionsUnsupported) + } + return &queueAdapter{queue: existing}, nil + } + return newRawRuntimeAdapter(q, nil, opts...) } -// NewWithStore creates a bus runtime with a custom orchestration store. -// @group Constructors +// NewWithStore constructs the retained low-level bus route with an explicit +// store. An existing queue.Queue must instead receive queue.WithStore when built. // -// Example: new bus with store -// -// q, _ := queue.NewSync() -// store := bus.NewMemoryStore() -// b, _ := bus.NewWithStore(q, store) -// _ = b +// Deprecated: use queue.New with queue.WithStore. func NewWithStore(q any, store Store, opts ...Option) (Bus, error) { - if q == nil { - return nil, errors.New("queue is required") + if existing, ok := q.(*queue.Queue); ok { + if existing == nil { + return nil, errors.New("queue is required") + } + return nil, fmt.Errorf("%w: pass queue.WithStore to queue.New instead", ErrQueueOptionsUnsupported) } - qr, err := asRuntime(q) - if err != nil { - return nil, err + return newRawRuntimeAdapter(q, store, opts...) +} + +// hasConstructionOptions distinguishes an option-free facade request from an +// attempt to mutate an engine that queue.Queue has already configured. +func hasConstructionOptions(opts []Option) bool { + for _, option := range opts { + if option != nil { + return true + } + } + return false +} + +// newRawRuntimeAdapter preserves the advanced busruntime.Runtime construction +// seam while delegating all orchestration behavior to the single internal engine. +func newRawRuntimeAdapter(q any, store Store, opts ...Option) (Bus, error) { + config := optionConfig{store: store} + for _, option := range opts { + if option != nil { + option(&config) + } } - if store == nil { - store = NewMemoryStore() + engineOptions := make([]workflow.Option, 0, 3) + if config.observer != nil { + engineOptions = append(engineOptions, workflow.WithObserver(legacyObserverAdapter{observer: config.observer})) } - r := &runtime{ - q: qr, - store: store, - now: time.Now, - handlers: make(map[string]Handler), - chainCallbacks: make(map[string]chainCallbacks), - batchCallbacks: make(map[string]batchCallbacks), + if config.clock != nil { + engineOptions = append(engineOptions, workflow.WithClock(config.clock)) } - for _, opt := range opts { - opt(r) + if len(config.middlewares) > 0 { + engineOptions = append(engineOptions, workflow.WithMiddleware(toWorkflowMiddlewares(config.middlewares)...)) } + engine, err := workflow.NewWithStore(q, toWorkflowStore(config.store), engineOptions...) + if err != nil { + return nil, err + } + return &runtimeAdapter{engine: engine}, nil +} - qr.BusRegister(internalJob, r.handleInternalJob) - qr.BusRegister(internalJobChainNode, r.handleInternalChainNode) - qr.BusRegister(internalJobBatchJob, r.handleInternalBatchJob) - qr.BusRegister(internalJobCallback, r.handleInternalCallback) - return r, nil +type legacyObserverAdapter struct { + observer Observer } -func asRuntime(v any) (busruntime.Runtime, error) { - if v == nil { - return nil, errors.New("queue is required") +// Observe translates the canonical engine event into the frozen legacy event shape. +func (a legacyObserverAdapter) Observe(ctx context.Context, event workflow.Event) { + safeObserve(ctx, a.observer, Event{ + SchemaVersion: event.SchemaVersion, + EventID: event.EventID, + Kind: EventKind(event.Kind), + DispatchID: event.DispatchID, + JobID: event.JobID, + ChainID: event.ChainID, + BatchID: event.BatchID, + Attempt: event.Attempt, + JobType: event.JobType, + JobKey: event.JobKey, + Queue: event.Queue, + Duration: event.Duration, + Time: event.Time, + Err: event.Err, + }) +} + +type runtimeAdapter struct { + engine workflow.Engine +} + +var _ Bus = (*runtimeAdapter)(nil) + +// Register adapts a legacy handler to the internal engine message contract. +func (a *runtimeAdapter) Register(jobType string, handler Handler) { + if handler == nil { + return } - if q, ok := v.(busruntime.Runtime); ok { - return q, nil + a.engine.Register(jobType, func(ctx context.Context, message workflow.Context) error { + return handler(ctx, toQueueMessage(message)) + }) +} + +// Dispatch converts the legacy boundary DTO without changing when payload JSON is encoded. +func (a *runtimeAdapter) Dispatch(ctx context.Context, job Job) (DispatchResult, error) { + result, err := a.engine.Dispatch(ctx, toWorkflowJob(job)) + return toQueueDispatchResult(result), err +} + +// Chain converts legacy job DTOs and wraps the engine's self-returning builder interface. +func (a *runtimeAdapter) Chain(jobs ...Job) ChainBuilder { + converted := make([]workflow.Job, 0, len(jobs)) + for _, job := range jobs { + converted = append(converted, toWorkflowJob(job)) } - return nil, fmt.Errorf("queue does not support bus runtime adapter") + return &runtimeChainBuilder{inner: a.engine.Chain(converted...)} } -type runtime struct { - q busruntime.Runtime - store Store - now func() time.Time +// Batch converts legacy job DTOs and wraps the engine's self-returning builder interface. +func (a *runtimeAdapter) Batch(jobs ...Job) BatchBuilder { + converted := make([]workflow.Job, 0, len(jobs)) + for _, job := range jobs { + converted = append(converted, toWorkflowJob(job)) + } + return &runtimeBatchBuilder{inner: a.engine.Batch(converted...)} +} - observer Observer +// StartWorkers forwards worker startup to the raw runtime engine. +func (a *runtimeAdapter) StartWorkers(ctx context.Context) error { + return a.engine.StartWorkers(ctx) +} - mu sync.RWMutex - handlers map[string]Handler - chainCallbacks map[string]chainCallbacks - batchCallbacks map[string]batchCallbacks - middlewares []Middleware +// Shutdown forwards worker shutdown to the raw runtime engine. +func (a *runtimeAdapter) Shutdown(ctx context.Context) error { + return a.engine.Shutdown(ctx) } -var _ Bus = (*runtime)(nil) +// FindBatch forwards persisted batch lookup to the internal engine. +func (a *runtimeAdapter) FindBatch(ctx context.Context, batchID string) (BatchState, error) { + state, err := a.engine.FindBatch(ctx, batchID) + return toQueueBatchState(state), err +} -// Register binds a job type to a handler. -// @group Runtime -// -// Example: register handler -// -// b.Register("emails:send", func(ctx context.Context, jc bus.Context) error { return nil }) -func (r *runtime) Register(jobType string, handler Handler) { - r.mu.Lock() - defer r.mu.Unlock() - r.handlers[jobType] = handler +// FindChain forwards persisted chain lookup to the internal engine. +func (a *runtimeAdapter) FindChain(ctx context.Context, chainID string) (ChainState, error) { + state, err := a.engine.FindChain(ctx, chainID) + return toQueueChainState(state), err } -// Dispatch enqueues one job for execution. -// @group Runtime -// -// Example: dispatch one job -// -// _, _ = b.Dispatch(context.Background(), bus.NewJob("emails:send", map[string]any{"id": 1})) -func (r *runtime) Dispatch(ctx context.Context, job Job) (DispatchResult, error) { - wj, err := toWireJob(job) - if err != nil { - return DispatchResult{}, err +// Prune forwards workflow retention to the internal engine. +func (a *runtimeAdapter) Prune(ctx context.Context, before time.Time) error { + return a.engine.Prune(ctx, before) +} + +// toWorkflowJob maps the legacy mutable DTO into the engine model without +// encoding Payload, preserving the legacy Dispatch-time failure boundary. +func toWorkflowJob(job Job) workflow.Job { + return workflow.Job{ + Type: job.Type, + Payload: job.Payload, + Options: workflow.JobOptions{ + Queue: job.Options.Queue, + Delay: job.Options.Delay, + Timeout: job.Options.Timeout, + Retry: job.Options.Retry, + Backoff: job.Options.Backoff, + UniqueFor: job.Options.UniqueFor, + }, } - dispatchID := newID("dsp") - env := envelope{ - SchemaVersion: schemaVersion, - DispatchID: dispatchID, - Kind: "job", - JobID: newID("job"), - Job: wj, +} + +type runtimeChainBuilder struct { + inner workflow.ChainBuilder +} + +// OnQueue applies a default queue to jobs without an explicit target. +func (b *runtimeChainBuilder) OnQueue(queueName string) ChainBuilder { + b.inner = b.inner.OnQueue(queueName) + return b +} + +// Catch registers the legacy failure callback on the internal builder. +func (b *runtimeChainBuilder) Catch(callback func(context.Context, ChainState, error) error) ChainBuilder { + if callback == nil { + b.inner = b.inner.Catch(nil) + return b } - r.emit(ctx, Event{SchemaVersion: schemaVersion, EventID: newID("evt"), Kind: EventDispatchStarted, DispatchID: dispatchID, JobID: env.JobID, JobType: wj.Type, Queue: wj.Options.Queue, Time: r.now()}) - if err := r.dispatchEnvelope(ctx, internalJob, env); err != nil { - r.emit(ctx, Event{SchemaVersion: schemaVersion, EventID: newID("evt"), Kind: EventDispatchFailed, DispatchID: dispatchID, JobID: env.JobID, JobType: wj.Type, Queue: wj.Options.Queue, Time: r.now(), Err: err}) - return DispatchResult{DispatchID: dispatchID}, err + b.inner = b.inner.Catch(func(ctx context.Context, state workflow.ChainState, err error) error { + return callback(ctx, toQueueChainState(state), err) + }) + return b +} + +// Finally registers the legacy terminal callback on the internal builder. +func (b *runtimeChainBuilder) Finally(callback func(context.Context, ChainState) error) ChainBuilder { + if callback == nil { + b.inner = b.inner.Finally(nil) + return b } - r.emit(ctx, Event{SchemaVersion: schemaVersion, EventID: newID("evt"), Kind: EventDispatchSucceeded, DispatchID: dispatchID, JobID: env.JobID, JobType: wj.Type, Queue: wj.Options.Queue, Time: r.now()}) - return DispatchResult{DispatchID: dispatchID}, nil + b.inner = b.inner.Finally(func(ctx context.Context, state workflow.ChainState) error { + return callback(ctx, toQueueChainState(state)) + }) + return b } -// Chain creates a sequential workflow where each job runs only after the prior job succeeds. -// @group Chaining -// -// Example: dispatch chain -// -// type PollPayload struct { -// URL string `json:"url"` -// } -// type DownsamplePayload struct { -// Window string `json:"window"` -// } -// type AlertPayload struct { -// Severity string `json:"severity"` -// } -// chainID, _ := b.Chain( -// bus.NewJob("monitor:poll", PollPayload{URL: "https://goforj.dev/health"}), -// bus.NewJob("monitor:downsample", DownsamplePayload{Window: "5m"}), -// bus.NewJob("monitor:alert", AlertPayload{Severity: "critical"}), -// ).OnQueue("monitor-critical"). -// Catch(func(context.Context, bus.ChainState, error) error { return nil }). -// Finally(func(context.Context, bus.ChainState) error { return nil }). -// Dispatch(context.Background()) -// _ = chainID -func (r *runtime) Chain(jobs ...Job) ChainBuilder { - return &chainBuilder{r: r, jobs: append([]Job(nil), jobs...)} -} - -// Batch creates a parallel workflow and tracks aggregate completion state. -// @group Batching -// -// Example: dispatch batch -// -// type PollPayload struct { -// URL string `json:"url"` -// } -// batchID, _ := b.Batch( -// bus.NewJob("monitor:poll", PollPayload{URL: "https://a"}), -// bus.NewJob("monitor:poll", PollPayload{URL: "https://b"}), -// ).Name("monitor sweep"). -// OnQueue("monitor-scan"). -// AllowFailures(). -// Progress(func(context.Context, bus.BatchState) error { return nil }). -// Then(func(context.Context, bus.BatchState) error { return nil }). -// Catch(func(context.Context, bus.BatchState, error) error { return nil }). -// Finally(func(context.Context, bus.BatchState) error { return nil }). -// Dispatch(context.Background()) -// _ = batchID -func (r *runtime) Batch(jobs ...Job) BatchBuilder { - return &batchBuilder{r: r, jobs: append([]Job(nil), jobs...)} -} - -// StartWorkers starts the underlying queue worker runtime. -// @group Runtime -// -// Example: start workers -// -// _ = b.StartWorkers(context.Background()) -func (r *runtime) StartWorkers(ctx context.Context) error { return r.q.StartWorkers(ctx) } +// Dispatch creates and starts the internal chain workflow. +func (b *runtimeChainBuilder) Dispatch(ctx context.Context) (string, error) { + return b.inner.Dispatch(ctx) +} -// Shutdown stops the underlying queue worker runtime. -// @group Runtime -// -// Example: shutdown workers -// -// _ = b.Shutdown(context.Background()) -func (r *runtime) Shutdown(ctx context.Context) error { return r.q.Shutdown(ctx) } +type runtimeBatchBuilder struct { + inner workflow.BatchBuilder +} -// FindBatch returns persisted batch state by id. -// @group Runtime -// -// Example: find batch state -// -// st, _ := b.FindBatch(context.Background(), "bat_123") -// _ = st -func (r *runtime) FindBatch(ctx context.Context, batchID string) (BatchState, error) { - return r.store.GetBatch(ctx, batchID) +// Name sets the display name on the internal batch builder. +func (b *runtimeBatchBuilder) Name(name string) BatchBuilder { + b.inner = b.inner.Name(name) + return b } -// FindChain returns persisted chain state by id. -// @group Runtime -// -// Example: find chain state -// -// st, _ := b.FindChain(context.Background(), "chn_123") -// _ = st -func (r *runtime) FindChain(ctx context.Context, chainID string) (ChainState, error) { - return r.store.GetChain(ctx, chainID) +// OnQueue applies a default queue to jobs without an explicit target. +func (b *runtimeBatchBuilder) OnQueue(queueName string) BatchBuilder { + b.inner = b.inner.OnQueue(queueName) + return b } -// Prune removes terminal orchestration records older than before. -// @group Runtime -// -// Example: prune old state -// -// _ = b.Prune(context.Background(), time.Now().Add(-24*time.Hour)) -func (r *runtime) Prune(ctx context.Context, before time.Time) error { - return r.store.Prune(ctx, before) +// AllowFailures keeps remaining batch jobs active after one member fails. +func (b *runtimeBatchBuilder) AllowFailures() BatchBuilder { + b.inner = b.inner.AllowFailures() + return b } -func (r *runtime) dispatchEnvelope(ctx context.Context, jobType string, env envelope) error { - payload, err := json.Marshal(env) - if err != nil { - return err - } - return r.q.BusDispatch(ctx, jobType, payload, busruntime.JobOptions{ - Queue: env.Job.Options.Queue, - Delay: env.Job.Options.Delay, - Timeout: env.Job.Options.Timeout, - Retry: env.Job.Options.Retry, - Backoff: env.Job.Options.Backoff, - UniqueFor: env.Job.Options.UniqueFor, +// Progress registers the legacy progress callback on the internal builder. +func (b *runtimeBatchBuilder) Progress(callback func(context.Context, BatchState) error) BatchBuilder { + if callback == nil { + b.inner = b.inner.Progress(nil) + return b + } + b.inner = b.inner.Progress(func(ctx context.Context, state workflow.BatchState) error { + return callback(ctx, toQueueBatchState(state)) }) + return b } -func (r *runtime) dispatchCallback(ctx context.Context, base envelope, kind string, err error) error { - cbEnv := envelope{ - SchemaVersion: schemaVersion, - DispatchID: base.DispatchID, - Kind: "callback", - JobID: newID("job"), - ChainID: base.ChainID, - BatchID: base.BatchID, - CallbackKind: kind, - Job: wireJob{ - Options: JobOptions{ - Queue: base.Job.Options.Queue, - }, - }, +// Then registers the legacy success callback on the internal builder. +func (b *runtimeBatchBuilder) Then(callback func(context.Context, BatchState) error) BatchBuilder { + if callback == nil { + b.inner = b.inner.Then(nil) + return b + } + b.inner = b.inner.Then(func(ctx context.Context, state workflow.BatchState) error { + return callback(ctx, toQueueBatchState(state)) + }) + return b +} + +// Catch registers the legacy failure callback on the internal builder. +func (b *runtimeBatchBuilder) Catch(callback func(context.Context, BatchState, error) error) BatchBuilder { + if callback == nil { + b.inner = b.inner.Catch(nil) + return b + } + b.inner = b.inner.Catch(func(ctx context.Context, state workflow.BatchState, err error) error { + return callback(ctx, toQueueBatchState(state), err) + }) + return b +} + +// Finally registers the legacy terminal callback on the internal builder. +func (b *runtimeBatchBuilder) Finally(callback func(context.Context, BatchState) error) BatchBuilder { + if callback == nil { + b.inner = b.inner.Finally(nil) + return b } + b.inner = b.inner.Finally(func(ctx context.Context, state workflow.BatchState) error { + return callback(ctx, toQueueBatchState(state)) + }) + return b +} + +// Dispatch creates and starts the internal batch workflow. +func (b *runtimeBatchBuilder) Dispatch(ctx context.Context) (string, error) { + return b.inner.Dispatch(ctx) +} + +type queueAdapter struct { + queue *queue.Queue +} + +var _ Bus = (*queueAdapter)(nil) + +// Register forwards a legacy handler to the already configured root queue. +func (a *queueAdapter) Register(jobType string, handler Handler) { + a.queue.Register(jobType, handler) +} + +// Dispatch binds ctx to the root queue for this call and preserves legacy payload JSON semantics. +func (a *queueAdapter) Dispatch(ctx context.Context, job Job) (DispatchResult, error) { + converted, err := toQueueJob(job) if err != nil { - cbEnv.Error = err.Error() + return DispatchResult{}, err } - return r.dispatchEnvelope(ctx, internalJobCallback, cbEnv) + return a.queue.WithContext(ctx).Dispatch(converted) } -func (r *runtime) handleInternalJob(ctx context.Context, job busruntime.InboundJob) error { - var env envelope - if err := job.Bind(&env); err != nil { - return err +// Chain snapshots legacy job values while preserving Dispatch-time payload encoding. +func (a *queueAdapter) Chain(jobs ...Job) ChainBuilder { + return &queueChainBuilder{ + queue: a.queue, + jobs: append([]Job(nil), jobs...), } - return r.executeWireJob(ctx, env) } -func (r *runtime) executeWireJob(ctx context.Context, env envelope) error { - start := r.now() - r.emit(ctx, Event{ - SchemaVersion: schemaVersion, - EventID: newID("evt"), - Kind: EventJobStarted, - DispatchID: env.DispatchID, - JobID: env.JobID, - ChainID: env.ChainID, - BatchID: env.BatchID, - Attempt: env.Attempt, - JobType: env.Job.Type, - Queue: env.Job.Options.Queue, - Time: start, - }) - handler, ok := r.lookupHandler(env.Job.Type) - if !ok { - err := fmt.Errorf("bus handler not registered for %q", env.Job.Type) - r.emit(ctx, Event{SchemaVersion: schemaVersion, EventID: newID("evt"), Kind: EventJobFailed, DispatchID: env.DispatchID, JobID: env.JobID, ChainID: env.ChainID, BatchID: env.BatchID, Attempt: env.Attempt, JobType: env.Job.Type, Queue: env.Job.Options.Queue, Duration: r.now().Sub(start), Time: r.now(), Err: err}) - return err - } - jc := Context{ - SchemaVersion: schemaVersion, - DispatchID: env.DispatchID, - JobID: env.JobID, - ChainID: env.ChainID, - BatchID: env.BatchID, - Attempt: env.Attempt, - JobType: env.Job.Type, - payload: append([]byte(nil), env.Job.Payload...), - } - err := chainMiddleware(r.middlewareSnapshot(), func(ctx context.Context, c Context) error { - return handler(ctx, c) - })(ctx, jc) +// Batch snapshots legacy job values while preserving Dispatch-time payload encoding. +func (a *queueAdapter) Batch(jobs ...Job) BatchBuilder { + return &queueBatchBuilder{ + queue: a.queue, + jobs: append([]Job(nil), jobs...), + } +} + +// StartWorkers starts the existing root queue runtime. +func (a *queueAdapter) StartWorkers(ctx context.Context) error { + return a.queue.StartWorkers(ctx) +} + +// Shutdown stops the existing root queue runtime. +func (a *queueAdapter) Shutdown(ctx context.Context) error { + return a.queue.Shutdown(ctx) +} + +// FindBatch reads batch state from the root queue's configured store. +func (a *queueAdapter) FindBatch(ctx context.Context, batchID string) (BatchState, error) { + return a.queue.FindBatch(ctx, batchID) +} + +// FindChain reads chain state from the root queue's configured store. +func (a *queueAdapter) FindChain(ctx context.Context, chainID string) (ChainState, error) { + return a.queue.FindChain(ctx, chainID) +} + +// Prune applies retention through the root queue's configured store. +func (a *queueAdapter) Prune(ctx context.Context, before time.Time) error { + return a.queue.Prune(ctx, before) +} + +// queueWorkflowTarget is the canonical root builder surface shared by the +// production facade and its recording fake. +type queueWorkflowTarget interface { + // Chain creates a canonical sequential workflow builder. + Chain(jobs ...queue.Job) queue.ChainBuilder + // Batch creates a canonical aggregate workflow builder. + Batch(jobs ...queue.Job) queue.BatchBuilder +} + +type queueChainBuilder struct { + queue queueWorkflowTarget + jobs []Job + queueName string + catch func(context.Context, ChainState, error) error + finally func(context.Context, ChainState) error +} + +// OnQueue forwards queue selection while retaining the legacy fluent return type. +func (b *queueChainBuilder) OnQueue(queueName string) ChainBuilder { + b.queueName = queueName + return b +} + +// Catch forwards the legacy failure callback to the root builder. +func (b *queueChainBuilder) Catch(callback func(context.Context, ChainState, error) error) ChainBuilder { + b.catch = callback + return b +} + +// Finally forwards the legacy terminal callback to the root builder. +func (b *queueChainBuilder) Finally(callback func(context.Context, ChainState) error) ChainBuilder { + b.finally = callback + return b +} + +// Dispatch converts the shallow legacy job snapshot at the historical dispatch boundary. +func (b *queueChainBuilder) Dispatch(ctx context.Context) (string, error) { + converted, err := toQueueJobs(b.jobs) if err != nil { - r.emit(ctx, Event{SchemaVersion: schemaVersion, EventID: newID("evt"), Kind: EventJobFailed, DispatchID: env.DispatchID, JobID: env.JobID, ChainID: env.ChainID, BatchID: env.BatchID, Attempt: env.Attempt, JobType: env.Job.Type, Queue: env.Job.Options.Queue, Duration: r.now().Sub(start), Time: r.now(), Err: err}) - return err + return "", err } - r.emit(ctx, Event{SchemaVersion: schemaVersion, EventID: newID("evt"), Kind: EventJobSucceeded, DispatchID: env.DispatchID, JobID: env.JobID, ChainID: env.ChainID, BatchID: env.BatchID, Attempt: env.Attempt, JobType: env.Job.Type, Queue: env.Job.Options.Queue, Duration: r.now().Sub(start), Time: r.now()}) - return nil + return b.queue.Chain(converted...). + OnQueue(b.queueName). + Catch(b.catch). + Finally(b.finally). + Dispatch(ctx) +} + +type queueBatchBuilder struct { + queue queueWorkflowTarget + jobs []Job + name string + queueName string + allowFailures bool + progress func(context.Context, BatchState) error + then func(context.Context, BatchState) error + catch func(context.Context, BatchState, error) error + finally func(context.Context, BatchState) error +} + +// Name forwards the application-facing batch label while retaining the legacy fluent return type. +func (b *queueBatchBuilder) Name(name string) BatchBuilder { + b.name = name + return b } -func (r *runtime) middlewareSnapshot() []Middleware { - r.mu.RLock() - defer r.mu.RUnlock() - out := make([]Middleware, len(r.middlewares)) - copy(out, r.middlewares) - return out +// OnQueue forwards queue selection while retaining the legacy fluent return type. +func (b *queueBatchBuilder) OnQueue(queueName string) BatchBuilder { + b.queueName = queueName + return b } -func (r *runtime) lookupHandler(jobType string) (Handler, bool) { - r.mu.RLock() - defer r.mu.RUnlock() - handler, ok := r.handlers[jobType] - return handler, ok +// AllowFailures forwards fail-soft behavior while retaining the legacy fluent return type. +func (b *queueBatchBuilder) AllowFailures() BatchBuilder { + b.allowFailures = true + return b } -func (r *runtime) emit(ctx context.Context, event Event) { - safeObserve(ctx, r.observer, event) +// Progress forwards the legacy progress callback to the root builder. +func (b *queueBatchBuilder) Progress(callback func(context.Context, BatchState) error) BatchBuilder { + b.progress = callback + return b } -type wireJob struct { - Type string `json:"type"` - Payload []byte `json:"payload"` - Options JobOptions `json:"options"` +// Then forwards the legacy success callback to the root builder. +func (b *queueBatchBuilder) Then(callback func(context.Context, BatchState) error) BatchBuilder { + b.then = callback + return b } -func toWireJob(job Job) (wireJob, error) { +// Catch forwards the legacy failure callback to the root builder. +func (b *queueBatchBuilder) Catch(callback func(context.Context, BatchState, error) error) BatchBuilder { + b.catch = callback + return b +} + +// Finally forwards the legacy terminal callback to the root builder. +func (b *queueBatchBuilder) Finally(callback func(context.Context, BatchState) error) BatchBuilder { + b.finally = callback + return b +} + +// Dispatch converts the shallow legacy job snapshot at the historical dispatch boundary. +func (b *queueBatchBuilder) Dispatch(ctx context.Context) (string, error) { + converted, err := toQueueJobs(b.jobs) + if err != nil { + return "", err + } + builder := b.queue.Batch(converted...). + Name(b.name). + OnQueue(b.queueName) + if b.allowFailures { + builder = builder.AllowFailures() + } + return builder. + Progress(b.progress). + Then(b.then). + Catch(b.catch). + Finally(b.finally). + Dispatch(ctx) +} + +// toQueueJobs converts a legacy workflow job slice while retaining the first conversion error. +func toQueueJobs(jobs []Job) ([]queue.Job, error) { + converted := make([]queue.Job, 0, len(jobs)) + for _, job := range jobs { + convertedJob, err := toQueueJob(job) + if err != nil { + return nil, err + } + converted = append(converted, convertedJob) + } + return converted, nil +} + +// toQueueJob freezes the legacy DTO's json.Marshal result as raw canonical +// payload bytes so strings, byte slices, RawMessage, nil, and custom marshalers +// retain their historical wire representation. +func toQueueJob(job Job) (queue.Job, error) { if job.Type == "" { - return wireJob{}, errors.New("bus job type is required") + return queue.Job{}, errors.New("bus job type is required") } payload, err := json.Marshal(job.Payload) if err != nil { - return wireJob{}, err + return queue.Job{}, err } - return wireJob{ - Type: job.Type, - Payload: payload, - Options: job.Options, - }, nil -} - -type envelope struct { - SchemaVersion int `json:"schema_version"` - DispatchID string `json:"dispatch_id"` - Kind string `json:"kind"` - JobID string `json:"job_id"` - ChainID string `json:"chain_id,omitempty"` - BatchID string `json:"batch_id,omitempty"` - NodeID string `json:"node_id,omitempty"` - Attempt int `json:"attempt"` - Job wireJob `json:"job"` - CallbackKind string `json:"callback_kind,omitempty"` - Error string `json:"error,omitempty"` -} - -func newID(prefix string) string { - var b [8]byte - _, _ = rand.Read(b[:]) - return prefix + "_" + hex.EncodeToString(b[:]) + converted := queue.NewJob(job.Type).Payload(json.RawMessage(payload)) + if job.Options.Queue != "" { + converted = converted.OnQueue(job.Options.Queue) + } + if job.Options.Delay != 0 { + converted = converted.Delay(job.Options.Delay) + } + if job.Options.Timeout != 0 { + converted = converted.Timeout(job.Options.Timeout) + } + converted = converted.Retry(job.Options.Retry) + if job.Options.Backoff != 0 { + converted = converted.Backoff(job.Options.Backoff) + } + if job.Options.UniqueFor != 0 { + converted = converted.UniqueFor(job.Options.UniqueFor) + } + return converted, nil } diff --git a/bus/chain.go b/bus/chain.go deleted file mode 100644 index a02e460..0000000 --- a/bus/chain.go +++ /dev/null @@ -1,217 +0,0 @@ -package bus - -import ( - "context" - "errors" - - "github.com/goforj/queue/busruntime" -) - -type ChainBuilder interface { - // OnQueue applies a default queue to chain jobs that do not set one. - // @group Chaining - // - // Example: set chain queue - // - // chainID, _ := b.Chain( - // bus.NewJob("a", nil), - // bus.NewJob("b", nil), - // ).OnQueue("critical").Dispatch(context.Background()) - // _ = chainID - OnQueue(queue string) ChainBuilder - // Catch registers a callback invoked when chain execution fails. - // @group Chaining - // - // Example: chain catch callback - // - // chainID, _ := b.Chain(bus.NewJob("a", nil)). - // Catch(func(context.Context, bus.ChainState, error) error { return nil }). - // Dispatch(context.Background()) - // _ = chainID - Catch(fn func(ctx context.Context, st ChainState, err error) error) ChainBuilder - // Finally registers a callback invoked once when chain execution finishes. - // @group Chaining - // - // Example: chain finally callback - // - // chainID, _ := b.Chain(bus.NewJob("a", nil)). - // Finally(func(context.Context, bus.ChainState) error { return nil }). - // Dispatch(context.Background()) - // _ = chainID - Finally(fn func(ctx context.Context, st ChainState) error) ChainBuilder - // Dispatch creates and starts the chain workflow. - // @group Chaining - // - // Example: dispatch chain - // - // chainID, _ := b.Chain(bus.NewJob("a", nil), bus.NewJob("b", nil)).Dispatch(context.Background()) - // _ = chainID - Dispatch(ctx context.Context) (string, error) -} - -type chainBuilder struct { - r *runtime - jobs []Job - queue string - catch func(ctx context.Context, st ChainState, err error) error - done func(ctx context.Context, st ChainState) error -} - -func (b *chainBuilder) OnQueue(queue string) ChainBuilder { - b.queue = queue - return b -} - -func (b *chainBuilder) Catch(fn func(ctx context.Context, st ChainState, err error) error) ChainBuilder { - b.catch = fn - return b -} - -func (b *chainBuilder) Finally(fn func(ctx context.Context, st ChainState) error) ChainBuilder { - b.done = fn - return b -} - -func (b *chainBuilder) Dispatch(ctx context.Context) (string, error) { - if len(b.jobs) == 0 { - return "", errors.New("chain requires at least one job") - } - chainID := newID("chn") - dispatchID := newID("dsp") - nodes := make([]ChainNode, 0, len(b.jobs)) - for i, job := range b.jobs { - wj, err := toWireJob(job) - if err != nil { - return "", err - } - if b.queue != "" && wj.Options.Queue == "" { - wj.Options.Queue = b.queue - } - nodes = append(nodes, ChainNode{ - NodeID: nodeID(chainID, i), - Job: wj, - }) - } - if err := b.r.store.CreateChain(ctx, ChainRecord{ - ChainID: chainID, - DispatchID: dispatchID, - Queue: b.queue, - Nodes: nodes, - CreatedAt: b.r.now(), - }); err != nil { - return "", err - } - b.r.mu.Lock() - b.r.chainCallbacks[chainID] = chainCallbacks{ - catch: b.catch, - finally: b.done, - } - b.r.mu.Unlock() - - b.r.emit(ctx, Event{SchemaVersion: schemaVersion, EventID: newID("evt"), Kind: EventChainStarted, DispatchID: dispatchID, ChainID: chainID, Queue: b.queue, Time: b.r.now()}) - first := nodes[0] - if err := b.r.dispatchEnvelope(ctx, internalJobChainNode, envelope{ - SchemaVersion: schemaVersion, - DispatchID: dispatchID, - Kind: "chain_node", - ChainID: chainID, - NodeID: first.NodeID, - JobID: newID("job"), - Job: first.Job, - }); err != nil { - if st, stErr := b.r.store.GetChain(ctx, chainID); stErr == nil && (st.Failed || st.Completed || st.NextIndex > 0) { - return chainID, err - } - _ = b.r.store.FailChain(ctx, chainID, err) - st, stErr := b.r.store.GetChain(ctx, chainID) - if stErr == nil { - _ = b.r.invokeChainCatch(ctx, st, err) - _ = b.r.invokeChainFinally(ctx, st) - } - b.r.emit(ctx, Event{SchemaVersion: schemaVersion, EventID: newID("evt"), Kind: EventChainFailed, DispatchID: dispatchID, ChainID: chainID, Time: b.r.now(), Err: err}) - return chainID, err - } - return chainID, nil -} - -type chainCallbacks struct { - catch func(ctx context.Context, st ChainState, err error) error - finally func(ctx context.Context, st ChainState) error -} - -func nodeID(chainID string, idx int) string { - return chainID + "_" + newID("n") -} - -func (r *runtime) handleInternalChainNode(ctx context.Context, job busruntime.InboundJob) error { - var env envelope - if err := job.Bind(&env); err != nil { - return err - } - err := r.executeWireJob(ctx, env) - if err != nil { - _ = r.store.FailChain(ctx, env.ChainID, err) - r.emit(ctx, Event{SchemaVersion: schemaVersion, EventID: newID("evt"), Kind: EventChainFailed, DispatchID: env.DispatchID, ChainID: env.ChainID, JobID: env.JobID, JobType: env.Job.Type, Queue: env.Job.Options.Queue, Time: r.now(), Err: err}) - _ = r.dispatchCallback(ctx, env, "chain_catch", err) - _ = r.dispatchCallback(ctx, env, "chain_finally", nil) - return err - } - next, done, advErr := r.store.AdvanceChain(ctx, env.ChainID, env.NodeID) - if advErr != nil { - return advErr - } - if done { - r.emit(ctx, Event{SchemaVersion: schemaVersion, EventID: newID("evt"), Kind: EventChainCompleted, DispatchID: env.DispatchID, ChainID: env.ChainID, Time: r.now()}) - _ = r.dispatchCallback(ctx, env, "chain_finally", nil) - return nil - } - r.emit(ctx, Event{SchemaVersion: schemaVersion, EventID: newID("evt"), Kind: EventChainAdvanced, DispatchID: env.DispatchID, ChainID: env.ChainID, Time: r.now()}) - return r.dispatchEnvelope(ctx, internalJobChainNode, envelope{ - SchemaVersion: schemaVersion, - DispatchID: env.DispatchID, - Kind: "chain_node", - ChainID: env.ChainID, - NodeID: next.NodeID, - JobID: newID("job"), - Job: next.Job, - }) -} - -func (r *runtime) invokeChainCatch(ctx context.Context, st ChainState, err error) error { - key := "chain_catch:" + st.ChainID - ok, onceErr := r.callbackOnce(ctx, key) - if onceErr != nil { - return onceErr - } - if !ok { - return nil - } - r.mu.RLock() - cb := r.chainCallbacks[st.ChainID] - r.mu.RUnlock() - if cb.catch != nil { - _ = cb.catch(ctx, st, err) - } - return nil -} - -func (r *runtime) invokeChainFinally(ctx context.Context, st ChainState) error { - key := "chain_finally:" + st.ChainID - ok, onceErr := r.callbackOnce(ctx, key) - if onceErr != nil { - return onceErr - } - if !ok { - return nil - } - r.mu.RLock() - cb := r.chainCallbacks[st.ChainID] - r.mu.RUnlock() - if cb.finally != nil { - _ = cb.finally(ctx, st) - } - r.mu.Lock() - delete(r.chainCallbacks, st.ChainID) - r.mu.Unlock() - return nil -} diff --git a/bus/construction_compat_test.go b/bus/construction_compat_test.go new file mode 100644 index 0000000..ab9df73 --- /dev/null +++ b/bus/construction_compat_test.go @@ -0,0 +1,185 @@ +package bus_test + +import ( + "context" + "encoding/json" + "errors" + "sync/atomic" + "testing" + "time" + + "github.com/goforj/queue" + "github.com/goforj/queue/bus" +) + +type facadeContextKey struct{} + +// TestBusNewWithQueueSharesCanonicalEngine proves compatibility wrappers do not construct or register a second workflow engine. +func TestBusNewWithQueueSharesCanonicalEngine(t *testing.T) { + root, err := queue.NewSync() + if err != nil { + t.Fatalf("new sync queue: %v", err) + } + first, err := bus.New(root) + if err != nil { + t.Fatalf("new first compatibility facade: %v", err) + } + second, err := bus.New(root, nil) + if err != nil { + t.Fatalf("new second compatibility facade: %v", err) + } + if err := first.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers through facade: %v", err) + } + t.Cleanup(func() { + if shutdownErr := root.Shutdown(context.Background()); shutdownErr != nil { + t.Errorf("shutdown root queue: %v", shutdownErr) + } + }) + + var handled atomic.Int32 + first.Register("compat:shared", func(ctx context.Context, message bus.Context) error { + if ctx.Value(facadeContextKey{}) != "legacy-context" { + return errors.New("legacy dispatch context was not forwarded") + } + var payload struct { + ID int `json:"id"` + } + if err := message.Bind(&payload); err != nil { + return err + } + if payload.ID != 7 && payload.ID != 8 { + return errors.New("unexpected shared handler payload") + } + handled.Add(1) + return nil + }) + ctx := context.WithValue(context.Background(), facadeContextKey{}, "legacy-context") + if _, err := second.Dispatch(ctx, bus.NewJob("compat:shared", map[string]int{"id": 7})); err != nil { + t.Fatalf("dispatch through second facade: %v", err) + } + if _, err := root.WithContext(ctx).Dispatch(queue.NewJob("compat:shared").PayloadJSON(map[string]int{"id": 8})); err != nil { + t.Fatalf("dispatch through root after facade registration: %v", err) + } + if handled.Load() != 2 { + t.Fatalf("shared handler calls = %d, want 2", handled.Load()) + } + + first.Register("compat:step", func(context.Context, bus.Context) error { return nil }) + chainID, err := second.Chain( + bus.NewJob("compat:step", nil), + bus.NewJob("compat:step", nil), + ).Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch legacy chain through root engine: %v", err) + } + chainState, err := root.FindChain(context.Background(), chainID) + if err != nil { + t.Fatalf("find legacy chain through root: %v", err) + } + if !chainState.Completed || chainState.NextIndex != 2 { + t.Fatalf("shared chain state = %+v, want completed two-node chain", chainState) + } + + batchID, err := root.Batch( + queue.NewJob("compat:step"), + queue.NewJob("compat:step"), + ).Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch root batch: %v", err) + } + batchState, err := first.FindBatch(context.Background(), batchID) + if err != nil { + t.Fatalf("find root batch through facade: %v", err) + } + if !batchState.Completed || batchState.Processed != 2 { + t.Fatalf("shared batch state = %+v, want completed two-job batch", batchState) + } + + if _, err := first.FindChain(context.Background(), "missing-chain"); !errors.Is(err, bus.ErrNotFound) || !errors.Is(err, queue.ErrWorkflowNotFound) { + t.Fatalf("shared not-found identity = %v", err) + } +} + +// TestBusNewWithQueueRejectsConstructionOptions makes already-applied queue configuration explicit instead of silently ignoring it. +func TestBusNewWithQueueRejectsConstructionOptions(t *testing.T) { + root, err := queue.NewSync() + if err != nil { + t.Fatalf("new sync queue: %v", err) + } + tests := []struct { + name string + option bus.Option + }{ + {name: "observer", option: bus.WithObserver(bus.ObserverFunc(func(context.Context, bus.Event) {}))}, + {name: "store", option: bus.WithStore(bus.NewMemoryStore())}, + {name: "clock", option: bus.WithClock(time.Now)}, + {name: "middleware", option: bus.WithMiddleware(bus.RetryPolicy{})}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if _, err := bus.New(root, test.option); !errors.Is(err, bus.ErrQueueOptionsUnsupported) { + t.Fatalf("bus.New option error = %v, want ErrQueueOptionsUnsupported", err) + } + }) + } + if _, err := bus.NewWithStore(root, bus.NewMemoryStore()); !errors.Is(err, bus.ErrQueueOptionsUnsupported) { + t.Fatalf("bus.NewWithStore error = %v, want ErrQueueOptionsUnsupported", err) + } + if _, err := bus.New((*queue.Queue)(nil)); err == nil || err.Error() != "queue is required" { + t.Fatalf("typed nil queue error = %v, want queue is required", err) + } +} + +// TestBusQueueFacadePreservesLegacyPayloadEncoding proves the new canonical route does not reinterpret compatibility DTO payloads. +func TestBusQueueFacadePreservesLegacyPayloadEncoding(t *testing.T) { + root, err := queue.NewSync() + if err != nil { + t.Fatalf("new sync queue: %v", err) + } + compatibility, err := bus.New(root) + if err != nil { + t.Fatalf("new compatibility facade: %v", err) + } + if err := compatibility.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + t.Cleanup(func() { + if shutdownErr := compatibility.Shutdown(context.Background()); shutdownErr != nil { + t.Errorf("shutdown compatibility facade: %v", shutdownErr) + } + }) + + var got []byte + compatibility.Register("compat:payload", func(_ context.Context, message bus.Context) error { + got = message.PayloadBytes() + return nil + }) + tests := []struct { + name string + payload any + want string + }{ + {name: "nil", payload: nil, want: "null"}, + {name: "map", payload: map[string]bool{"ready": true}, want: `{"ready":true}`}, + {name: "string", payload: "raw", want: `"raw"`}, + {name: "bytes", payload: []byte{0, 1, 2}, want: `"AAEC"`}, + {name: "raw message", payload: json.RawMessage(`{"raw":true}`), want: `{"raw":true}`}, + {name: "custom marshaler", payload: fixedJSONPayload{}, want: `{"custom":true}`}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got = nil + if _, err := compatibility.Dispatch(context.Background(), bus.NewJob("compat:payload", test.payload)); err != nil { + t.Fatalf("dispatch legacy payload: %v", err) + } + if string(got) != test.want { + t.Fatalf("handler payload = %q, want %q", got, test.want) + } + }) + } + + if _, err := compatibility.Dispatch(context.Background(), bus.NewJob("compat:payload", failingJSONPayload{})); err == nil || err.Error() != "json: error calling MarshalJSON for type bus_test.failingJSONPayload: compat marshal failure" { + t.Fatalf("marshal failure = %v, want legacy deferred error", err) + } +} diff --git a/bus/doc.go b/bus/doc.go index 0c51f00..f83a1bc 100644 --- a/bus/doc.go +++ b/bus/doc.go @@ -1,9 +1,8 @@ -// Package bus provides the workflow orchestration engine used by queue. +// Package bus preserves the legacy workflow API as a compatibility facade over queue. // -// Most applications should use the top-level queue package (`queue.New(...)`, -// `Queue.Dispatch`, `Queue.Chain`, `Queue.Batch`) rather than importing bus -// directly. -// -// This package remains available for advanced/internal orchestration plumbing, -// custom workflow integration, and lower-level testing. +// Deprecated: use the top-level queue package. The low-level raw-runtime route +// remains temporarily available for existing integrations, but all orchestration +// behavior is owned by queue's internal workflow engine. Compatible model aliases +// resolve to physical root queue types; legacy boundary DTOs and interfaces remain +// here only where aliasing would change source behavior. package bus diff --git a/bus/events.go b/bus/events.go index 788b1e0..c05913e 100644 --- a/bus/events.go +++ b/bus/events.go @@ -5,29 +5,54 @@ import ( "time" ) +// EventKind identifies one legacy workflow lifecycle fact. +// +// Deprecated: use queue.EventKind. type EventKind string const ( - EventDispatchStarted EventKind = "dispatch_started" + // EventDispatchStarted identifies the start of a legacy dispatch operation. + EventDispatchStarted EventKind = "dispatch_started" + // EventDispatchSucceeded identifies an accepted legacy dispatch operation. EventDispatchSucceeded EventKind = "dispatch_succeeded" - EventDispatchFailed EventKind = "dispatch_failed" - EventJobStarted EventKind = "job_started" - EventJobSucceeded EventKind = "job_succeeded" - EventJobFailed EventKind = "job_failed" - EventChainStarted EventKind = "chain_started" - EventChainAdvanced EventKind = "chain_advanced" - EventChainCompleted EventKind = "chain_completed" - EventChainFailed EventKind = "chain_failed" - EventBatchStarted EventKind = "batch_started" - EventBatchProgressed EventKind = "batch_progressed" - EventBatchCompleted EventKind = "batch_completed" - EventBatchFailed EventKind = "batch_failed" - EventBatchCancelled EventKind = "batch_cancelled" - EventCallbackStarted EventKind = "callback_started" + // EventDispatchFailed identifies a rejected legacy dispatch operation. + EventDispatchFailed EventKind = "dispatch_failed" + // EventJobStarted identifies the start of logical workflow job execution. + EventJobStarted EventKind = "job_started" + // EventJobSucceeded identifies committed logical workflow job success. + EventJobSucceeded EventKind = "job_succeeded" + // EventJobFailed identifies terminal logical workflow job failure. + EventJobFailed EventKind = "job_failed" + // EventChainStarted identifies creation of a chain workflow. + EventChainStarted EventKind = "chain_started" + // EventChainAdvanced identifies committed advancement of a chain workflow. + EventChainAdvanced EventKind = "chain_advanced" + // EventChainCompleted identifies successful completion of a chain workflow. + EventChainCompleted EventKind = "chain_completed" + // EventChainFailed identifies terminal failure of a chain workflow. + EventChainFailed EventKind = "chain_failed" + // EventBatchStarted identifies creation of a batch workflow. + EventBatchStarted EventKind = "batch_started" + // EventBatchProgressed identifies committed progress of a batch workflow. + EventBatchProgressed EventKind = "batch_progressed" + // EventBatchCompleted identifies completion of a batch workflow. + EventBatchCompleted EventKind = "batch_completed" + // EventBatchFailed identifies a failed member of a batch workflow. + EventBatchFailed EventKind = "batch_failed" + // EventBatchCancelled identifies cancellation of a batch workflow. + EventBatchCancelled EventKind = "batch_cancelled" + // EventCallbackStarted identifies the start of an ephemeral callback. + EventCallbackStarted EventKind = "callback_started" + // EventCallbackSucceeded identifies successful completion of an ephemeral callback. EventCallbackSucceeded EventKind = "callback_succeeded" - EventCallbackFailed EventKind = "callback_failed" + // EventCallbackFailed identifies failure of an ephemeral callback. + EventCallbackFailed EventKind = "callback_failed" ) +// Event carries the legacy bus workflow event shape. +// +// Deprecated: use queue.Event. This shape remains available only at the bus +// compatibility boundary and is translated from the canonical producer. type Event struct { SchemaVersion int EventID string @@ -38,41 +63,33 @@ type Event struct { BatchID string Attempt int JobType string + JobKey string Queue string Duration time.Duration Time time.Time Err error } +// Observer receives legacy bus workflow events. +// +// Deprecated: use queue.Observer. type Observer interface { Observe(ctx context.Context, event Event) } +// ObserverFunc adapts a function to Observer. +// +// Deprecated: use queue.ObserverFunc. type ObserverFunc func(ctx context.Context, event Event) // Observe calls the wrapped observer function. -// @group Events -// -// Example: observer func -// -// observer := bus.ObserverFunc(func(ctx context.Context, event bus.Event) { -// _ = event.Kind -// }) -// observer.Observe(context.Background(), bus.Event{Kind: bus.EventDispatchStarted}) func (f ObserverFunc) Observe(ctx context.Context, event Event) { f(ctx, event) } -// MultiObserver fans out one event to multiple observers. -// @group Events -// -// Example: fan out observers +// MultiObserver fans out one legacy event while isolating observer panics. // -// observer := bus.MultiObserver( -// bus.ObserverFunc(func(context.Context, bus.Event) {}), -// bus.ObserverFunc(func(context.Context, bus.Event) {}), -// ) -// observer.Observe(context.Background(), bus.Event{Kind: bus.EventDispatchStarted}) +// Deprecated: use queue.MultiObserver. func MultiObserver(observers ...Observer) Observer { filtered := make([]Observer, 0, len(observers)) for _, observer := range observers { @@ -85,12 +102,14 @@ func MultiObserver(observers ...Observer) Observer { type multiObserver []Observer +// Observe forwards the unchanged legacy event to each configured observer. func (m multiObserver) Observe(ctx context.Context, event Event) { for _, observer := range m { safeObserve(ctx, observer, event) } } +// safeObserve prevents optional telemetry from changing workflow execution. func safeObserve(ctx context.Context, observer Observer, event Event) { if observer == nil { return diff --git a/bus/events_test.go b/bus/events_test.go deleted file mode 100644 index 006f2b8..0000000 --- a/bus/events_test.go +++ /dev/null @@ -1,276 +0,0 @@ -package bus - -import ( - "context" - "errors" - "testing" - - "github.com/goforj/queue/busruntime" -) - -type failingDispatchQueue struct { - err error - handlers map[string]busruntime.Handler - workerCnt int -} - -func (q *failingDispatchQueue) StartWorkers(context.Context) error { return nil } -func (q *failingDispatchQueue) Shutdown(context.Context) error { return nil } - -func (q *failingDispatchQueue) BusRegister(jobType string, handler busruntime.Handler) { - if q.handlers == nil { - q.handlers = make(map[string]busruntime.Handler) - } - q.handlers[jobType] = handler -} - -func (q *failingDispatchQueue) BusDispatch(context.Context, string, []byte, busruntime.JobOptions) error { - return q.err -} - -func TestDispatchEnqueueFailureEmitsStartedThenFailed(t *testing.T) { - q := &failingDispatchQueue{err: errors.New("enqueue failed")} - var kinds []EventKind - b, err := NewWithStore(q, NewMemoryStore(), WithObserver(ObserverFunc(func(_ context.Context, e Event) { - kinds = append(kinds, e.Kind) - }))) - if err != nil { - t.Fatalf("new bus: %v", err) - } - - res, err := b.Dispatch(context.Background(), NewJob("monitor:poll", nil)) - if err == nil { - t.Fatal("expected dispatch enqueue failure") - } - if res.DispatchID == "" { - t.Fatal("expected non-empty dispatch id on enqueue failure") - } - if len(kinds) != 2 { - t.Fatalf("expected 2 events, got %d (%v)", len(kinds), kinds) - } - if kinds[0] != EventDispatchStarted || kinds[1] != EventDispatchFailed { - t.Fatalf("expected started then failed, got %v", kinds) - } -} - -func TestUnknownCallbackKindEmitsCallbackFailed(t *testing.T) { - q := newSyncTestRuntime() - var started int - var failed int - b, err := New(q, WithObserver(ObserverFunc(func(_ context.Context, e Event) { - if e.Kind == EventCallbackStarted { - started++ - } - if e.Kind == EventCallbackFailed { - failed++ - } - }))) - if err != nil { - t.Fatalf("new bus: %v", err) - } - if err := b.StartWorkers(context.Background()); err != nil { - t.Fatalf("start workers: %v", err) - } - - payload := map[string]any{ - "schema_version": 1, - "dispatch_id": "d1", - "kind": "callback", - "job_id": "j1", - "callback_kind": "unknown_kind", - } - if err := q.DispatchJSON(context.Background(), internalJobCallback, payload); err == nil { - t.Fatal("expected unknown callback kind error") - } - if started != 1 { - t.Fatalf("expected callback started once, got %d", started) - } - if failed != 1 { - t.Fatalf("expected callback failed once, got %d", failed) - } -} - -func TestCallbackMissingRequiredIDsEmitsCallbackFailed(t *testing.T) { - q := newSyncTestRuntime() - var failed int - b, err := New(q, WithObserver(ObserverFunc(func(_ context.Context, e Event) { - if e.Kind == EventCallbackFailed { - failed++ - } - }))) - if err != nil { - t.Fatalf("new bus: %v", err) - } - if err := b.StartWorkers(context.Background()); err != nil { - t.Fatalf("start workers: %v", err) - } - - tests := []map[string]any{ - { - "schema_version": 1, - "dispatch_id": "d1", - "kind": "callback", - "job_id": "j1", - "callback_kind": "chain_catch", - // missing chain_id - }, - { - "schema_version": 1, - "dispatch_id": "d2", - "kind": "callback", - "job_id": "j2", - "callback_kind": "batch_then", - // missing batch_id - }, - } - - for i, payloadMap := range tests { - if err := q.DispatchJSON(context.Background(), internalJobCallback, payloadMap); err == nil { - t.Fatalf("expected callback validation error for case %d", i) - } - } - - if failed != len(tests) { - t.Fatalf("expected %d callback failed events, got %d", len(tests), failed) - } -} - -func TestMultiObserverPanicsAreIsolated(t *testing.T) { - var called int - observer := MultiObserver( - ObserverFunc(func(context.Context, Event) { panic("boom") }), - ObserverFunc(func(context.Context, Event) { called++ }), - ) - observer.Observe(context.Background(), Event{Kind: EventDispatchStarted}) - if called != 1 { - t.Fatalf("expected second observer called once despite panic, got %d", called) - } -} - -func TestChainEnqueueFailureInvokesCatchAndFinally(t *testing.T) { - q := &failingDispatchQueue{err: errors.New("enqueue failed")} - bi, err := NewWithStore(q, NewMemoryStore()) - if err != nil { - t.Fatalf("new bus: %v", err) - } - b := bi.(*runtime) - - var catchCount int - var finallyCount int - chainID, err := b.Chain(NewJob("monitor:poll", nil)). - Catch(func(context.Context, ChainState, error) error { - catchCount++ - return nil - }). - Finally(func(context.Context, ChainState) error { - finallyCount++ - return nil - }). - Dispatch(context.Background()) - if err == nil { - t.Fatal("expected chain enqueue error") - } - if catchCount != 1 { - t.Fatalf("expected catch once, got %d", catchCount) - } - if finallyCount != 1 { - t.Fatalf("expected finally once, got %d", finallyCount) - } - st, err := b.FindChain(context.Background(), chainID) - if err != nil { - t.Fatalf("find failed chain: %v", err) - } - if !st.Failed { - t.Fatalf("expected chain marked failed, got %+v", st) - } - b.mu.RLock() - cbCount := len(b.chainCallbacks) - b.mu.RUnlock() - if cbCount != 0 { - t.Fatalf("expected chain callbacks cleaned, got %d", cbCount) - } -} - -func TestBatchEnqueueFailureInvokesCatchAndFinally(t *testing.T) { - q := &failingDispatchQueue{err: errors.New("enqueue failed")} - bi, err := NewWithStore(q, NewMemoryStore()) - if err != nil { - t.Fatalf("new bus: %v", err) - } - b := bi.(*runtime) - - var catchCount int - var finallyCount int - batchID, err := b.Batch(NewJob("monitor:poll", nil)). - Catch(func(context.Context, BatchState, error) error { - catchCount++ - return nil - }). - Finally(func(context.Context, BatchState) error { - finallyCount++ - return nil - }). - Dispatch(context.Background()) - if err == nil { - t.Fatal("expected batch enqueue error") - } - if catchCount != 1 { - t.Fatalf("expected catch once, got %d", catchCount) - } - if finallyCount != 1 { - t.Fatalf("expected finally once, got %d", finallyCount) - } - st, err := b.FindBatch(context.Background(), batchID) - if err != nil { - t.Fatalf("find failed batch: %v", err) - } - if !st.Completed || !st.Cancelled { - t.Fatalf("expected batch cancelled+completed, got %+v", st) - } - b.mu.RLock() - cbCount := len(b.batchCallbacks) - b.mu.RUnlock() - if cbCount != 0 { - t.Fatalf("expected batch callbacks cleaned, got %d", cbCount) - } -} - -func TestChainDispatchFailureStillReturnsChainID(t *testing.T) { - q := newSyncTestRuntime() - b, err := New(q) - if err != nil { - t.Fatalf("new bus: %v", err) - } - if err := b.StartWorkers(context.Background()); err != nil { - t.Fatalf("start workers: %v", err) - } - b.Register("monitor:downsample", func(context.Context, Context) error { return errors.New("boom") }) - - chainID, err := b.Chain(NewJob("monitor:downsample", nil)).Dispatch(context.Background()) - if err == nil { - t.Fatal("expected chain dispatch error") - } - if chainID == "" { - t.Fatal("expected non-empty chain id on dispatch error") - } -} - -func TestBatchDispatchFailureStillReturnsBatchID(t *testing.T) { - q := newSyncTestRuntime() - b, err := New(q) - if err != nil { - t.Fatalf("new bus: %v", err) - } - if err := b.StartWorkers(context.Background()); err != nil { - t.Fatalf("start workers: %v", err) - } - b.Register("monitor:downsample", func(context.Context, Context) error { return errors.New("boom") }) - - batchID, err := b.Batch(NewJob("monitor:downsample", nil)).Dispatch(context.Background()) - if err == nil { - t.Fatal("expected batch dispatch error") - } - if batchID == "" { - t.Fatal("expected non-empty batch id on dispatch error") - } -} diff --git a/bus/facade_forwarding_test.go b/bus/facade_forwarding_test.go new file mode 100644 index 0000000..6c5d9eb --- /dev/null +++ b/bus/facade_forwarding_test.go @@ -0,0 +1,370 @@ +package bus_test + +import ( + "context" + "encoding/json" + "errors" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/goforj/queue" + "github.com/goforj/queue/bus" +) + +type facadeDeferredPayload struct { + calls *atomic.Int32 + value int +} + +// MarshalJSON records when the compatibility facade freezes the referenced payload state. +func (p *facadeDeferredPayload) MarshalJSON() ([]byte, error) { + p.calls.Add(1) + return json.Marshal(struct { + Value int `json:"value"` + }{Value: p.value}) +} + +// TestQueueFacadeForwardsFluentBuildersCallbacksAndPrune exercises the retained facade through one shared root engine. +func TestQueueFacadeForwardsFluentBuildersCallbacksAndPrune(t *testing.T) { + root, err := queue.NewSync() + if err != nil { + t.Fatalf("new sync queue: %v", err) + } + compatibility, err := bus.New(root) + if err != nil { + t.Fatalf("new compatibility facade: %v", err) + } + if err := compatibility.StartWorkers(context.Background()); err != nil { + t.Fatalf("start compatibility workers: %v", err) + } + t.Cleanup(func() { + if shutdownErr := compatibility.Shutdown(context.Background()); shutdownErr != nil { + t.Errorf("shutdown compatibility workers: %v", shutdownErr) + } + }) + + cause := errors.New("compatibility handler failed") + compatibility.Register("compat:facade-fail", func(context.Context, bus.Context) error { + return cause + }) + + var chainCatch atomic.Int32 + var chainFinally atomic.Int32 + chainID, err := compatibility.Chain(bus.NewJob("compat:facade-fail", nil)). + OnQueue("critical"). + Catch(func(_ context.Context, state bus.ChainState, callbackErr error) error { + if state.Queue != "critical" || callbackErr == nil || callbackErr.Error() != cause.Error() { + t.Errorf("chain catch state/error = %+v/%v", state, callbackErr) + } + chainCatch.Add(1) + return nil + }). + Finally(func(_ context.Context, state bus.ChainState) error { + if state.Queue != "critical" || !state.Failed { + t.Errorf("chain finally state = %+v", state) + } + chainFinally.Add(1) + return nil + }). + Dispatch(context.Background()) + if !errors.Is(err, cause) { + t.Fatalf("chain error = %v, want handler cause", err) + } + if chainCatch.Load() != 1 || chainFinally.Load() != 1 { + t.Fatalf("chain callback counts = %d/%d, want 1/1", chainCatch.Load(), chainFinally.Load()) + } + chainState, err := compatibility.FindChain(context.Background(), chainID) + if err != nil { + t.Fatalf("find chain: %v", err) + } + if len(chainState.Nodes) != 1 || chainState.Nodes[0].Job.Options.Queue != "critical" { + t.Fatalf("chain routing state = %+v", chainState) + } + + var batchProgress atomic.Int32 + var batchThen atomic.Int32 + var batchCatch atomic.Int32 + var batchFinally atomic.Int32 + batchID, err := compatibility.Batch(bus.NewJob("compat:facade-fail", nil)). + Name("compatibility batch"). + OnQueue("bulk"). + AllowFailures(). + Progress(func(_ context.Context, state bus.BatchState) error { + if state.Name != "compatibility batch" || state.Queue != "bulk" { + t.Errorf("batch progress state = %+v", state) + } + batchProgress.Add(1) + return nil + }). + Then(func(_ context.Context, state bus.BatchState) error { + if !state.Completed || state.Cancelled { + t.Errorf("batch then state = %+v", state) + } + batchThen.Add(1) + return nil + }). + Catch(func(_ context.Context, state bus.BatchState, callbackErr error) error { + if state.Failed != 1 || callbackErr == nil || callbackErr.Error() != cause.Error() { + t.Errorf("batch catch state/error = %+v/%v", state, callbackErr) + } + batchCatch.Add(1) + return nil + }). + Finally(func(_ context.Context, state bus.BatchState) error { + if !state.Completed || state.Queue != "bulk" { + t.Errorf("batch finally state = %+v", state) + } + batchFinally.Add(1) + return nil + }). + Dispatch(context.Background()) + if !errors.Is(err, cause) { + t.Fatalf("batch error = %v, want handler cause", err) + } + if batchProgress.Load() != 1 || batchThen.Load() != 1 || batchCatch.Load() != 1 || batchFinally.Load() != 1 { + t.Fatalf("batch callback counts = %d/%d/%d/%d, want 1/1/1/1", batchProgress.Load(), batchThen.Load(), batchCatch.Load(), batchFinally.Load()) + } + batchState, err := compatibility.FindBatch(context.Background(), batchID) + if err != nil { + t.Fatalf("find batch: %v", err) + } + if batchState.Name != "compatibility batch" || batchState.Queue != "bulk" || !batchState.AllowFailed || !batchState.Completed || batchState.Cancelled { + t.Fatalf("batch state = %+v", batchState) + } + + if err := compatibility.Prune(context.Background(), time.Now().Add(time.Hour)); err != nil { + t.Fatalf("prune compatibility state: %v", err) + } + if _, err := compatibility.FindChain(context.Background(), chainID); !errors.Is(err, bus.ErrNotFound) { + t.Fatalf("find pruned chain error = %v, want ErrNotFound", err) + } + if _, err := compatibility.FindBatch(context.Background(), batchID); !errors.Is(err, bus.ErrNotFound) { + t.Fatalf("find pruned batch error = %v, want ErrNotFound", err) + } +} + +// TestQueueFacadeDefersLegacyConversionFailuresAcrossFluentBuilders verifies errors remain at the legacy Dispatch boundary. +func TestQueueFacadeDefersLegacyConversionFailuresAcrossFluentBuilders(t *testing.T) { + root, err := queue.NewSync() + if err != nil { + t.Fatalf("new sync queue: %v", err) + } + t.Cleanup(func() { + if shutdownErr := root.Shutdown(context.Background()); shutdownErr != nil { + t.Errorf("shutdown root queue: %v", shutdownErr) + } + }) + compatibility, err := bus.New(root) + if err != nil { + t.Fatalf("new compatibility facade: %v", err) + } + + chainCallbackCalled := false + _, err = compatibility.Chain(bus.Job{}). + OnQueue("critical"). + Catch(func(context.Context, bus.ChainState, error) error { + chainCallbackCalled = true + return nil + }). + Finally(func(context.Context, bus.ChainState) error { + chainCallbackCalled = true + return nil + }). + Dispatch(context.Background()) + if err == nil || err.Error() != "bus job type is required" { + t.Fatalf("chain conversion error = %v, want missing legacy type", err) + } + if chainCallbackCalled { + t.Fatal("deferred chain conversion failure invoked callbacks") + } + + batchCallbackCalled := false + _, err = compatibility.Batch(bus.NewJob("compat:marshal-failure", failingJSONPayload{})). + Name("unreachable"). + OnQueue("bulk"). + AllowFailures(). + Progress(func(context.Context, bus.BatchState) error { + batchCallbackCalled = true + return nil + }). + Then(func(context.Context, bus.BatchState) error { + batchCallbackCalled = true + return nil + }). + Catch(func(context.Context, bus.BatchState, error) error { + batchCallbackCalled = true + return nil + }). + Finally(func(context.Context, bus.BatchState) error { + batchCallbackCalled = true + return nil + }). + Dispatch(context.Background()) + if err == nil || err.Error() != "json: error calling MarshalJSON for type bus_test.failingJSONPayload: compat marshal failure" { + t.Fatalf("batch conversion error = %v, want legacy marshal failure", err) + } + if batchCallbackCalled { + t.Fatal("deferred batch conversion failure invoked callbacks") + } +} + +// TestQueueFacadeDefersBuilderEncodingAndKeepsShallowJobSnapshots freezes the legacy builder timing contract. +func TestQueueFacadeDefersBuilderEncodingAndKeepsShallowJobSnapshots(t *testing.T) { + root, err := queue.NewSync() + if err != nil { + t.Fatalf("new sync queue: %v", err) + } + compatibility, err := bus.New(root) + if err != nil { + t.Fatalf("new compatibility facade: %v", err) + } + if err := compatibility.StartWorkers(context.Background()); err != nil { + t.Fatalf("start compatibility workers: %v", err) + } + t.Cleanup(func() { + if shutdownErr := compatibility.Shutdown(context.Background()); shutdownErr != nil { + t.Errorf("shutdown compatibility workers: %v", shutdownErr) + } + }) + + var chainSeen atomic.Int32 + compatibility.Register("compat:deferred-chain", func(_ context.Context, message bus.Context) error { + var payload struct { + Value int `json:"value"` + } + if bindErr := message.Bind(&payload); bindErr != nil { + return bindErr + } + chainSeen.Store(int32(payload.Value)) + return nil + }) + var batchSeen atomic.Int32 + compatibility.Register("compat:deferred-batch", func(_ context.Context, message bus.Context) error { + var payload struct { + Value int `json:"value"` + } + if bindErr := message.Bind(&payload); bindErr != nil { + return bindErr + } + batchSeen.Store(int32(payload.Value)) + return nil + }) + + var chainMarshalCalls atomic.Int32 + chainPayload := &facadeDeferredPayload{calls: &chainMarshalCalls, value: 1} + chainJobs := []bus.Job{bus.NewJob("compat:deferred-chain", chainPayload).OnQueue("chain-job")} + chainBuilder := compatibility.Chain(chainJobs...).OnQueue("chain-default").Catch(nil).Finally(nil) + if chainMarshalCalls.Load() != 0 { + t.Fatalf("chain marshal calls before Dispatch = %d, want 0", chainMarshalCalls.Load()) + } + chainJobs[0].Type = "compat:mutated-chain" + chainJobs[0].Options.Queue = "mutated-chain-job" + chainPayload.value = 2 + chainID, err := chainBuilder.Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch deferred chain: %v", err) + } + if chainMarshalCalls.Load() != 1 || chainSeen.Load() != 2 { + t.Fatalf("chain marshal calls/payload = %d/%d, want 1/2", chainMarshalCalls.Load(), chainSeen.Load()) + } + chainState, err := compatibility.FindChain(context.Background(), chainID) + if err != nil { + t.Fatalf("find deferred chain: %v", err) + } + if len(chainState.Nodes) != 1 || chainState.Nodes[0].Job.Type != "compat:deferred-chain" || chainState.Nodes[0].Job.Options.Queue != "chain-job" { + t.Fatalf("chain shallow snapshot = %+v", chainState) + } + + var batchMarshalCalls atomic.Int32 + batchPayload := &facadeDeferredPayload{calls: &batchMarshalCalls, value: 3} + batchJobs := []bus.Job{bus.NewJob("compat:deferred-batch", batchPayload).OnQueue("batch-job")} + batchBuilder := compatibility.Batch(batchJobs...).Name("deferred batch").OnQueue("batch-default").AllowFailures(). + Progress(nil).Then(nil).Catch(nil).Finally(nil) + if batchMarshalCalls.Load() != 0 { + t.Fatalf("batch marshal calls before Dispatch = %d, want 0", batchMarshalCalls.Load()) + } + batchJobs[0].Type = "compat:mutated-batch" + batchJobs[0].Options.Queue = "mutated-batch-job" + batchPayload.value = 4 + batchID, err := batchBuilder.Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch deferred batch: %v", err) + } + if batchMarshalCalls.Load() != 1 || batchSeen.Load() != 4 { + t.Fatalf("batch marshal calls/payload = %d/%d, want 1/4", batchMarshalCalls.Load(), batchSeen.Load()) + } + batchState, err := compatibility.FindBatch(context.Background(), batchID) + if err != nil { + t.Fatalf("find deferred batch: %v", err) + } + if batchState.Name != "deferred batch" || batchState.Queue != "batch-default" || !batchState.AllowFailed || !batchState.Completed { + t.Fatalf("deferred batch state = %+v", batchState) + } +} + +// TestRawRuntimeFacadeForwardsNilCallbacksAndShutdown covers the retained low-level compatibility lifecycle. +func TestRawRuntimeFacadeForwardsNilCallbacksAndShutdown(t *testing.T) { + runtime, err := newBusTestRuntime(queue.Config{Driver: queue.DriverSync}) + if err != nil { + t.Fatalf("new raw sync runtime: %v", err) + } + compatibility, err := bus.New(runtime) + if err != nil { + t.Fatalf("new raw compatibility facade: %v", err) + } + t.Cleanup(func() { + _ = compatibility.Shutdown(context.Background()) + }) + compatibility.Register("compat:nil-handler", nil) + compatibility.Register("compat:raw-success", func(context.Context, bus.Context) error { return nil }) + if err := compatibility.StartWorkers(context.Background()); err != nil { + t.Fatalf("start raw compatibility workers: %v", err) + } + if _, err := compatibility.Dispatch(context.Background(), bus.NewJob("compat:nil-handler", nil)); err == nil { + t.Fatal("nil compatibility registration accepted a delivery") + } else if !strings.Contains(err.Error(), "handler not registered") { + t.Fatalf("nil compatibility dispatch error = %v, want missing handler", err) + } + + chainID, err := compatibility.Chain(bus.NewJob("compat:raw-success", nil)). + OnQueue("raw-chain"). + Catch(nil). + Finally(nil). + Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch raw chain: %v", err) + } + chainState, err := compatibility.FindChain(context.Background(), chainID) + if err != nil { + t.Fatalf("find raw chain: %v", err) + } + if chainState.Queue != "raw-chain" || !chainState.Completed { + t.Fatalf("raw chain state = %+v", chainState) + } + + batchID, err := compatibility.Batch(bus.NewJob("compat:raw-success", nil)). + Name("raw batch"). + OnQueue("raw-batch"). + AllowFailures(). + Progress(nil). + Then(nil). + Catch(nil). + Finally(nil). + Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch raw batch: %v", err) + } + batchState, err := compatibility.FindBatch(context.Background(), batchID) + if err != nil { + t.Fatalf("find raw batch: %v", err) + } + if batchState.Name != "raw batch" || batchState.Queue != "raw-batch" || !batchState.AllowFailed || !batchState.Completed { + t.Fatalf("raw batch state = %+v", batchState) + } + + if err := compatibility.Shutdown(context.Background()); err != nil { + t.Fatalf("shutdown raw compatibility workers: %v", err) + } +} diff --git a/bus/fake.go b/bus/fake.go index a4d91da..356658e 100644 --- a/bus/fake.go +++ b/bus/fake.go @@ -2,19 +2,26 @@ package bus import ( "context" + "sync" "testing" "time" + + "github.com/goforj/queue" ) +// Fake preserves the legacy bus testing surface as a thin view of the +// concurrency-safe root queue fake. +// +// Deprecated: use queue.NewFake. type Fake struct { - dispatched []Job - chains [][]Job - batches [][]Job + queue *queue.FakeQueue } var _ Bus = (*Fake)(nil) -// NewFake creates a bus fake that records dispatch, chain, and batch calls. +var fakeInitializationMu sync.Mutex + +// NewFake creates a legacy workflow view over one canonical root fake. // @group Constructors // // Example: new bus fake @@ -22,237 +29,179 @@ var _ Bus = (*Fake)(nil) // fake := bus.NewFake() // _, _ = fake.Dispatch(context.Background(), bus.NewJob("monitor:poll", nil)) func NewFake() *Fake { - return &Fake{} + return &Fake{queue: queue.NewFake()} +} + +// Queue returns the canonical root fake shared by this compatibility view. +// @group Testing +func (f *Fake) Queue() *queue.FakeQueue { + return f.canonicalQueue() } +// canonicalQueue lazily initializes the historical zero value under a package +// lock while keeping Fake values safe to copy after construction. +func (f *Fake) canonicalQueue() *queue.FakeQueue { + fakeInitializationMu.Lock() + defer fakeInitializationMu.Unlock() + if f.queue == nil { + f.queue = queue.NewFake() + } + return f.queue +} + +// Register is inert because Fake records accepted intent instead of executing handlers. func (f *Fake) Register(string, Handler) {} -// Dispatch records a dispatched job. +// Dispatch converts and records a legacy job through the canonical fake. // @group Testing // // Example: record dispatch // // fake := bus.NewFake() // _, _ = fake.Dispatch(context.Background(), bus.NewJob("emails:send", nil)) -func (f *Fake) Dispatch(_ context.Context, job Job) (DispatchResult, error) { - f.dispatched = append(f.dispatched, job) +func (f *Fake) Dispatch(ctx context.Context, job Job) (DispatchResult, error) { + converted, err := toQueueJob(job) + if err != nil { + return DispatchResult{}, err + } + if err := f.canonicalQueue().WithContext(ctx).Dispatch(converted); err != nil { + return DispatchResult{}, err + } return DispatchResult{DispatchID: "fake"}, nil } -// Chain records a chain specification. +// Chain snapshots legacy jobs and delegates execution-time conversion to the +// same compatibility builder used by a production queue facade. // @group Testing -// -// Example: record chain -// -// fake := bus.NewFake() -// _, _ = fake.Chain( -// bus.NewJob("a", nil), -// bus.NewJob("b", nil), -// ).Dispatch(context.Background()) func (f *Fake) Chain(jobs ...Job) ChainBuilder { - f.chains = append(f.chains, append([]Job(nil), jobs...)) - return &fakeChain{fake: f} + return &queueChainBuilder{ + queue: f.canonicalQueue(), + jobs: append([]Job(nil), jobs...), + } } -// Batch records a batch specification. +// Batch snapshots legacy jobs and delegates execution-time conversion to the +// same compatibility builder used by a production queue facade. // @group Testing -// -// Example: record batch -// -// fake := bus.NewFake() -// _, _ = fake.Batch( -// bus.NewJob("a", nil), -// bus.NewJob("b", nil), -// ).Dispatch(context.Background()) func (f *Fake) Batch(jobs ...Job) BatchBuilder { - f.batches = append(f.batches, append([]Job(nil), jobs...)) - return &fakeBatch{fake: f} + return &queueBatchBuilder{ + queue: f.canonicalQueue(), + jobs: append([]Job(nil), jobs...), + } +} + +// StartWorkers delegates the inert lifecycle contract to the canonical fake. +func (f *Fake) StartWorkers(ctx context.Context) error { + if f == nil { + return nil + } + return f.canonicalQueue().StartWorkers(ctx) +} + +// Shutdown delegates the inert lifecycle contract to the canonical fake. +func (f *Fake) Shutdown(ctx context.Context) error { + if f == nil { + return nil + } + return f.canonicalQueue().Shutdown(ctx) +} + +// FindBatch returns state created by an accepted fake batch. +func (f *Fake) FindBatch(ctx context.Context, batchID string) (BatchState, error) { + if f == nil { + return BatchState{}, ErrNotFound + } + return f.canonicalQueue().FindBatch(ctx, batchID) } -func (f *Fake) StartWorkers(context.Context) error { return nil } -func (f *Fake) Shutdown(context.Context) error { return nil } -func (f *Fake) FindBatch(context.Context, string) (BatchState, error) { - return BatchState{}, ErrNotFound +// FindChain returns state created by an accepted fake chain. +func (f *Fake) FindChain(ctx context.Context, chainID string) (ChainState, error) { + if f == nil { + return ChainState{}, ErrNotFound + } + return f.canonicalQueue().FindChain(ctx, chainID) } -func (f *Fake) FindChain(context.Context, string) (ChainState, error) { - return ChainState{}, ErrNotFound + +// Prune applies workflow retention to the canonical fake store. +func (f *Fake) Prune(ctx context.Context, before time.Time) error { + if f == nil { + return nil + } + return f.canonicalQueue().Prune(ctx, before) } -func (f *Fake) Prune(context.Context, time.Time) error { return nil } -// AssertNothingDispatched fails if any job was dispatched. +// AssertNothingDispatched fails if any direct job was accepted. // @group Testing -// -// Example: assert no dispatch -// -// fake := bus.NewFake() -// fake.AssertNothingDispatched(nil) func (f *Fake) AssertNothingDispatched(t testing.TB) { t.Helper() - if len(f.dispatched) != 0 { - t.Fatalf("expected no dispatched jobs, got %d", len(f.dispatched)) - } + f.canonicalQueue().AssertNothingDispatched(t) } -// AssertDispatched fails if the given job type was never dispatched. +// AssertDispatched fails if the given job type was never accepted. // @group Testing -// -// Example: assert dispatched -// -// fake := bus.NewFake() -// _, _ = fake.Dispatch(context.Background(), bus.NewJob("emails:send", nil)) -// fake.AssertDispatched(nil, "emails:send") func (f *Fake) AssertDispatched(t testing.TB, jobType string) { t.Helper() - for _, j := range f.dispatched { - if j.Type == jobType { - return - } - } - t.Fatalf("expected dispatched job %q", jobType) + f.canonicalQueue().AssertDispatched(t, jobType) } -// AssertDispatchedTimes fails if dispatched count for job type does not match n. +// AssertDispatchedTimes fails if the accepted count for jobType does not match n. // @group Testing -// -// Example: assert dispatch count by type -// -// fake := bus.NewFake() -// _, _ = fake.Dispatch(context.Background(), bus.NewJob("emails:send", nil)) -// _, _ = fake.Dispatch(context.Background(), bus.NewJob("emails:send", nil)) -// fake.AssertDispatchedTimes(nil, "emails:send", 2) func (f *Fake) AssertDispatchedTimes(t testing.TB, jobType string, n int) { t.Helper() - var got int - for _, j := range f.dispatched { - if j.Type == jobType { - got++ - } - } - if got != n { - t.Fatalf("expected job %q dispatched %d times, got %d", jobType, n, got) - } + f.canonicalQueue().AssertDispatchedTimes(t, jobType, n) } -// AssertNotDispatched fails if the given job type was dispatched. +// AssertNotDispatched fails if the given job type was accepted. // @group Testing -// -// Example: assert not dispatched -// -// fake := bus.NewFake() -// fake.AssertNotDispatched(nil, "emails:send") func (f *Fake) AssertNotDispatched(t testing.TB, jobType string) { t.Helper() - for _, j := range f.dispatched { - if j.Type == jobType { - t.Fatalf("expected job %q not dispatched", jobType) - } - } + f.canonicalQueue().AssertNotDispatched(t, jobType) } -// AssertCount fails if total dispatched count does not match n. +// AssertCount fails if the total direct dispatch count does not match n. // @group Testing -// -// Example: assert dispatch count -// -// fake := bus.NewFake() -// _, _ = fake.Dispatch(context.Background(), bus.NewJob("emails:send", nil)) -// fake.AssertCount(nil, 1) func (f *Fake) AssertCount(t testing.TB, n int) { t.Helper() - if len(f.dispatched) != n { - t.Fatalf("expected dispatched count %d, got %d", n, len(f.dispatched)) - } + f.canonicalQueue().AssertCount(t, n) } -// AssertDispatchedOn fails if a job type was not dispatched on queueName. +// AssertDispatchedOn fails if a job type was not accepted on queueName. // @group Testing -// -// Example: assert dispatched on queue -// -// fake := bus.NewFake() -// _, _ = fake.Dispatch(context.Background(), bus.NewJob("emails:send", nil).OnQueue("critical")) -// fake.AssertDispatchedOn(nil, "critical", "emails:send") func (f *Fake) AssertDispatchedOn(t testing.TB, queueName, jobType string) { t.Helper() - for _, j := range f.dispatched { - if j.Type == jobType && j.Options.Queue == queueName { - return - } - } - t.Fatalf("expected job %q dispatched on queue %q", jobType, queueName) + f.canonicalQueue().AssertDispatchedOn(t, queueName, jobType) } -// AssertChained fails if no recorded chain matches expected job type order. +// AssertChained fails if no accepted chain matches the expected job order. // @group Testing -// -// Example: assert chain -// -// fake := bus.NewFake() -// _, _ = fake.Chain(bus.NewJob("a", nil), bus.NewJob("b", nil)).Dispatch(context.Background()) -// fake.AssertChained(nil, []string{"a", "b"}) func (f *Fake) AssertChained(t testing.TB, expected []string) { t.Helper() - for _, chain := range f.chains { - if len(chain) != len(expected) { - continue - } - ok := true - for i := range chain { - if chain[i].Type != expected[i] { - ok = false - break - } - } - if ok { - return - } - } - t.Fatalf("expected chain %v", expected) + f.canonicalQueue().AssertChained(t, expected) } -// AssertBatchCount fails if total recorded batch count does not match n. +// AssertBatchCount fails if the accepted batch count does not match n. // @group Testing -// -// Example: assert batch count -// -// fake := bus.NewFake() -// _, _ = fake.Batch(bus.NewJob("a", nil)).Dispatch(context.Background()) -// fake.AssertBatchCount(nil, 1) func (f *Fake) AssertBatchCount(t testing.TB, n int) { t.Helper() - if len(f.batches) != n { - t.Fatalf("expected batch count %d, got %d", n, len(f.batches)) - } + f.canonicalQueue().AssertBatchCount(t, n) } -// AssertNothingBatched fails if any batch was recorded. +// AssertNothingBatched fails if any batch was accepted. // @group Testing -// -// Example: assert no batches -// -// fake := bus.NewFake() -// fake.AssertNothingBatched(nil) func (f *Fake) AssertNothingBatched(t testing.TB) { t.Helper() - if len(f.batches) != 0 { - t.Fatalf("expected no batches, got %d", len(f.batches)) - } + f.canonicalQueue().AssertNothingBatched(t) } -// AssertBatched fails unless at least one recorded batch matches predicate. +// AssertBatched fails unless one canonical batch matches the legacy projection. +// The predicate runs outside recorder locks. // @group Testing -// -// Example: assert batch predicate -// -// fake := bus.NewFake() -// _, _ = fake.Batch(bus.NewJob("a", nil), bus.NewJob("b", nil)).Dispatch(context.Background()) -// fake.AssertBatched(nil, func(spec bus.BatchSpec) bool { return len(spec.JobTypes) == 2 }) func (f *Fake) AssertBatched(t testing.TB, predicate func(spec BatchSpec) bool) { t.Helper() - for _, b := range f.batches { - spec := BatchSpec{JobTypes: make([]string, 0, len(b))} - for _, job := range b { - spec.JobTypes = append(spec.JobTypes, job.Type) + for _, record := range f.canonicalQueue().BatchRecords() { + spec := BatchSpec{JobTypes: make([]string, 0, len(record.Jobs))} + for _, job := range record.Jobs { + spec.JobTypes = append(spec.JobTypes, job.Job.Type) } if predicate(spec) { return @@ -261,28 +210,7 @@ func (f *Fake) AssertBatched(t testing.TB, predicate func(spec BatchSpec) bool) t.Fatalf("expected at least one batch to match predicate") } +// BatchSpec is the frozen assertion projection retained for legacy source compatibility. type BatchSpec struct { JobTypes []string } - -type fakeChain struct{ fake *Fake } - -func (f *fakeChain) OnQueue(string) ChainBuilder { return f } -func (f *fakeChain) Catch(func(context.Context, ChainState, error) error) ChainBuilder { - return f -} -func (f *fakeChain) Finally(func(context.Context, ChainState) error) ChainBuilder { return f } -func (f *fakeChain) Dispatch(context.Context) (string, error) { return "fake-chain", nil } - -type fakeBatch struct{ fake *Fake } - -func (f *fakeBatch) Name(string) BatchBuilder { return f } -func (f *fakeBatch) OnQueue(string) BatchBuilder { return f } -func (f *fakeBatch) AllowFailures() BatchBuilder { return f } -func (f *fakeBatch) Progress(func(context.Context, BatchState) error) BatchBuilder { return f } -func (f *fakeBatch) Then(func(context.Context, BatchState) error) BatchBuilder { return f } -func (f *fakeBatch) Catch(func(context.Context, BatchState, error) error) BatchBuilder { - return f -} -func (f *fakeBatch) Finally(func(context.Context, BatchState) error) BatchBuilder { return f } -func (f *fakeBatch) Dispatch(context.Context) (string, error) { return "fake-batch", nil } diff --git a/bus/fake_test.go b/bus/fake_test.go index b5612d7..12d17a2 100644 --- a/bus/fake_test.go +++ b/bus/fake_test.go @@ -3,12 +3,19 @@ package bus_test import ( "context" "errors" + "sync" "testing" "time" + "github.com/goforj/queue" "github.com/goforj/queue/bus" ) +// fakeDeferredPayload proves legacy builders retain Dispatch-time JSON encoding. +type fakeDeferredPayload struct { + Value int `json:"value"` +} + func TestFakeAssertions(t *testing.T) { f := bus.NewFake() f.AssertNothingDispatched(t) @@ -47,6 +54,49 @@ func TestFakeAssertions(t *testing.T) { }) } +// TestFakeZeroValueSharesOneConcurrentRecorder preserves the historical usable +// zero value without allowing concurrent callers to initialize separate state. +func TestFakeZeroValueSharesOneConcurrentRecorder(t *testing.T) { + var fake bus.Fake + queues := make(chan *queue.FakeQueue, 16) + var wait sync.WaitGroup + for i := 0; i < cap(queues); i++ { + wait.Add(1) + go func() { + defer wait.Done() + queues <- fake.Queue() + }() + } + wait.Wait() + close(queues) + var first *queue.FakeQueue + for candidate := range queues { + if first == nil { + first = candidate + continue + } + if candidate != first { + t.Fatal("zero-value Fake initialized more than one canonical queue") + } + } + copied := fake + if copied.Queue() != fake.Queue() { + t.Fatal("copy of initialized Fake lost canonical queue identity") + } + if _, err := fake.Dispatch(context.Background(), bus.NewJob("zero:direct", nil)); err != nil { + t.Fatalf("zero-value direct dispatch: %v", err) + } + if _, err := fake.Chain(bus.NewJob("zero:chain", nil)).Dispatch(context.Background()); err != nil { + t.Fatalf("zero-value chain dispatch: %v", err) + } + if _, err := fake.Batch(bus.NewJob("zero:batch", nil)).Dispatch(context.Background()); err != nil { + t.Fatalf("zero-value batch dispatch: %v", err) + } + fake.AssertDispatched(t, "zero:direct") + fake.AssertChained(t, []string{"zero:chain"}) + fake.AssertBatchCount(t, 1) +} + func TestFakeFindNotFound(t *testing.T) { f := bus.NewFake() _, err := f.FindChain(context.Background(), "missing") @@ -59,10 +109,40 @@ func TestFakeFindNotFound(t *testing.T) { } } -func TestFakePruneNoop(t *testing.T) { +// TestFakeNilReceiverLifecycleCompatibility preserves the historical inert +// lifecycle and missing-state behavior on a nil legacy fake pointer. +func TestFakeNilReceiverLifecycleCompatibility(t *testing.T) { + var fake *bus.Fake + if err := fake.StartWorkers(context.Background()); err != nil { + t.Fatalf("nil StartWorkers: %v", err) + } + if err := fake.Shutdown(context.Background()); err != nil { + t.Fatalf("nil Shutdown: %v", err) + } + if err := fake.Prune(context.Background(), time.Now()); err != nil { + t.Fatalf("nil Prune: %v", err) + } + if _, err := fake.FindChain(context.Background(), "missing"); !errors.Is(err, bus.ErrNotFound) { + t.Fatalf("nil FindChain error = %v, want ErrNotFound", err) + } + if _, err := fake.FindBatch(context.Background(), "missing"); !errors.Is(err, bus.ErrNotFound) { + t.Fatalf("nil FindBatch error = %v, want ErrNotFound", err) + } +} + +// TestFakePrunePreservesActiveWorkflow verifies the compatibility method uses +// canonical retention semantics rather than an unrelated fake no-op. +func TestFakePrunePreservesActiveWorkflow(t *testing.T) { f := bus.NewFake() - if err := f.Prune(context.Background(), time.Now()); err != nil { - t.Fatalf("expected fake prune noop, got %v", err) + chainID, err := f.Chain(bus.NewJob("prune:active", nil)).Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch active chain: %v", err) + } + if err := f.Prune(context.Background(), time.Now().Add(time.Hour)); err != nil { + t.Fatalf("prune active fake state: %v", err) + } + if _, err := f.FindChain(context.Background(), chainID); err != nil { + t.Fatalf("active chain was pruned: %v", err) } } @@ -108,3 +188,175 @@ func TestFakeRuntimeNoopAndFluentBuilders(t *testing.T) { t.Fatal("expected batch id") } } + +// TestFakeSharesCanonicalRootState verifies the legacy surface is only a typed +// conversion and assertion view over queue.FakeQueue. +func TestFakeSharesCanonicalRootState(t *testing.T) { + fake := bus.NewFake() + root := fake.Queue() + if root == nil { + t.Fatal("Queue returned nil canonical fake") + } + if err := root.Dispatch(queue.NewJob("root:dispatch").OnQueue("root")); err != nil { + t.Fatalf("root dispatch: %v", err) + } + result, err := fake.Dispatch(context.Background(), bus.NewJob("bus:dispatch", nil).OnQueue("legacy")) + if err != nil { + t.Fatalf("bus dispatch: %v", err) + } + if result.DispatchID != "fake" { + t.Fatalf("legacy direct fake ID = %q, want fake", result.DispatchID) + } + + fake.AssertDispatched(t, "root:dispatch") + root.AssertDispatched(t, "bus:dispatch") + if got := len(root.Records()); got != 2 { + t.Fatalf("shared direct records = %d, want 2", got) + } +} + +// TestFakeBuildersUseProductionTimingAndOptions verifies deferred legacy +// encoding and fluent workflow policy survive the compatibility adapter. +func TestFakeBuildersUseProductionTimingAndOptions(t *testing.T) { + fake := bus.NewFake() + payload := &fakeDeferredPayload{Value: 1} + chainBuilder := fake.Chain( + bus.NewJob("chain:first", payload), + bus.NewJob("chain:second", nil).OnQueue("dedicated"), + ).OnQueue("chain-default") + batchBuilder := fake.Batch( + bus.NewJob("batch:first", payload), + bus.NewJob("batch:second", nil).OnQueue("priority"), + ).Name("compatibility batch").OnQueue("batch-default").AllowFailures() + if len(fake.Queue().ChainRecords()) != 0 || len(fake.Queue().BatchRecords()) != 0 { + t.Fatal("builder construction recorded workflow state") + } + payload.Value = 2 + + chainID, err := chainBuilder.Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch chain: %v", err) + } + batchID, err := batchBuilder.Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch batch: %v", err) + } + chains := fake.Queue().ChainRecords() + if len(chains) != 1 || chains[0].ChainID != chainID || chains[0].Queue != "chain-default" { + t.Fatalf("chain record = %+v", chains) + } + if got := string(chains[0].Nodes[0].Job.Payload); got != `{"value":2}` { + t.Fatalf("deferred chain payload = %q", got) + } + if chains[0].Nodes[0].Job.Options.Queue != "chain-default" || chains[0].Nodes[1].Job.Options.Queue != "dedicated" { + t.Fatalf("chain queue precedence = %+v", chains[0].Nodes) + } + batches := fake.Queue().BatchRecords() + if len(batches) != 1 || batches[0].BatchID != batchID || batches[0].Name != "compatibility batch" || !batches[0].AllowFailed { + t.Fatalf("batch record = %+v", batches) + } + if batches[0].Jobs[0].Job.Options.Queue != "batch-default" || batches[0].Jobs[1].Job.Options.Queue != "priority" { + t.Fatalf("batch queue precedence = %+v", batches[0].Jobs) + } + if _, err := fake.FindChain(context.Background(), chainID); err != nil { + t.Fatalf("find accepted chain: %v", err) + } + if state, err := fake.FindBatch(context.Background(), batchID); err != nil || state.Total != 2 { + t.Fatalf("find accepted batch = %+v, %v", state, err) + } +} + +// TestFakeRejectedBuildersRemainInvisible verifies validation and context +// failures do not create false-positive legacy assertions. +func TestFakeRejectedBuildersRemainInvisible(t *testing.T) { + fake := bus.NewFake() + _ = fake.Chain(bus.NewJob("abandoned", nil)) + _ = fake.Batch(bus.NewJob("abandoned", nil)) + if _, err := fake.Dispatch(context.Background(), bus.NewJob("", nil)); err == nil { + t.Fatal("empty direct type error = nil") + } + if _, err := fake.Dispatch(context.Background(), bus.NewJob("bad:direct-payload", failingJSONPayload{})); err == nil { + t.Fatal("invalid direct payload error = nil") + } + if _, err := fake.Chain().Dispatch(context.Background()); err == nil { + t.Fatal("empty chain error = nil") + } + if _, err := fake.Batch().Dispatch(context.Background()); err == nil { + t.Fatal("empty batch error = nil") + } + if _, err := fake.Chain(bus.NewJob("", nil)).Dispatch(context.Background()); err == nil { + t.Fatal("empty chain member type error = nil") + } + if _, err := fake.Batch(bus.NewJob("bad:retry", nil).Retry(-1)).Dispatch(context.Background()); err == nil { + t.Fatal("invalid batch retry error = nil") + } + if _, err := fake.Chain(bus.NewJob("bad:chain-payload", failingJSONPayload{})).Dispatch(context.Background()); err == nil { + t.Fatal("invalid chain payload error = nil") + } + if _, err := fake.Batch(bus.NewJob("bad:batch-payload", failingJSONPayload{})).Dispatch(context.Background()); err == nil { + t.Fatal("invalid batch payload error = nil") + } + canceled, cancel := context.WithCancel(context.Background()) + cancel() + callbackCalled := false + if _, err := fake.Dispatch(canceled, bus.NewJob("cancelled:direct", nil)); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled direct error = %v", err) + } + if _, err := fake.Chain(bus.NewJob("cancelled:chain", nil)). + Catch(func(context.Context, bus.ChainState, error) error { callbackCalled = true; return nil }). + Finally(func(context.Context, bus.ChainState) error { callbackCalled = true; return nil }). + Dispatch(canceled); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled chain error = %v", err) + } + if _, err := fake.Batch(bus.NewJob("cancelled:batch", nil)). + Progress(func(context.Context, bus.BatchState) error { callbackCalled = true; return nil }). + Then(func(context.Context, bus.BatchState) error { callbackCalled = true; return nil }). + Catch(func(context.Context, bus.BatchState, error) error { callbackCalled = true; return nil }). + Finally(func(context.Context, bus.BatchState) error { callbackCalled = true; return nil }). + Dispatch(canceled); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled batch error = %v", err) + } + if callbackCalled { + t.Fatal("recording fake invoked a workflow callback") + } + if len(fake.Queue().Records()) != 0 || len(fake.Queue().ChainRecords()) != 0 || len(fake.Queue().BatchRecords()) != 0 { + t.Fatalf("rejected records = direct:%d chains:%d batches:%d", len(fake.Queue().Records()), len(fake.Queue().ChainRecords()), len(fake.Queue().BatchRecords())) + } +} + +// TestFakeConcurrentCompatibilityViews exercises legacy conversion and shared +// root inspection under the race detector. +func TestFakeConcurrentCompatibilityViews(t *testing.T) { + fake := bus.NewFake() + var wait sync.WaitGroup + for worker := 0; worker < 9; worker++ { + worker := worker + wait.Add(1) + go func() { + defer wait.Done() + for iteration := 0; iteration < 30; iteration++ { + switch worker % 3 { + case 0: + _, _ = fake.Dispatch(context.Background(), bus.NewJob("direct:legacy", map[string]int{"iteration": iteration})) + case 1: + _, _ = fake.Chain(bus.NewJob("chain:legacy", iteration)).Dispatch(context.Background()) + case 2: + _, _ = fake.Batch(bus.NewJob("batch:legacy", iteration)).Dispatch(context.Background()) + } + _ = fake.Queue().Records() + _ = fake.Queue().ChainRecords() + _ = fake.Queue().BatchRecords() + } + }() + } + wait.Wait() + if got := len(fake.Queue().Records()); got != 90 { + t.Fatalf("concurrent direct records = %d, want 90", got) + } + if got := len(fake.Queue().ChainRecords()); got != 90 { + t.Fatalf("concurrent chain records = %d, want 90", got) + } + if got := len(fake.Queue().BatchRecords()); got != 90 { + t.Fatalf("concurrent batch records = %d, want 90", got) + } +} diff --git a/bus/middleware.go b/bus/middleware.go index dbafa98..6736beb 100644 --- a/bus/middleware.go +++ b/bus/middleware.go @@ -1,204 +1,73 @@ package bus -import ( - "context" - "errors" - "fmt" - "time" -) - -type Next func(ctx context.Context, jc Context) error - -// Middleware can intercept bus job execution. -// @group Middleware -type Middleware interface { - Handle(ctx context.Context, jc Context, next Next) error -} - -// MiddlewareFunc adapts a function to Middleware. -// @group Middleware -type MiddlewareFunc func(ctx context.Context, jc Context, next Next) error +import "github.com/goforj/queue" -// Handle calls the wrapped middleware function. -// @group Middleware +// Next invokes the next workflow middleware or handler. // -// Example: middleware func -// -// mw := bus.MiddlewareFunc(func(ctx context.Context, jc bus.Context, next bus.Next) error { -// return next(ctx, jc) -// }) -// _ = mw -func (f MiddlewareFunc) Handle(ctx context.Context, jc Context, next Next) error { - return f(ctx, jc, next) -} - -func chainMiddleware(middlewares []Middleware, final Next) Next { - if len(middlewares) == 0 { - return final - } - next := final - for i := len(middlewares) - 1; i >= 0; i-- { - m := middlewares[i] - if m == nil { - continue - } - currentNext := next - next = func(ctx context.Context, jc Context) error { - return m.Handle(ctx, jc, currentNext) - } - } - return next -} - -var ( - ErrSkipped = errors.New("bus job skipped by middleware") - ErrRateLimited = errors.New("bus job rate limited") - ErrOverlapping = errors.New("bus job overlap prevented") -) +// Deprecated: use queue.Next. +type Next = queue.Next -type RetryPolicy struct{} - -// Handle passes execution through without modification. -// @group Middleware -// -// Example: retry policy passthrough +// Middleware intercepts workflow job execution. // -// policy := bus.RetryPolicy{} -// _ = policy -func (RetryPolicy) Handle(ctx context.Context, jc Context, next Next) error { - return next(ctx, jc) -} +// Deprecated: use queue.Middleware. +type Middleware = queue.Middleware -type SkipWhen struct { - Predicate func(ctx context.Context, jc Context) bool -} - -// Handle skips job execution when Predicate returns true. -// @group Middleware -// -// Example: skip by predicate +// MiddlewareFunc adapts a function to workflow middleware. // -// mw := bus.SkipWhen{ -// Predicate: func(context.Context, bus.Context) bool { return true }, -// } -// _ = mw -func (s SkipWhen) Handle(ctx context.Context, jc Context, next Next) error { - if s.Predicate != nil && s.Predicate(ctx, jc) { - return nil - } - return next(ctx, jc) -} - -type FailOnError struct { - When func(err error) bool -} +// Deprecated: use queue.MiddlewareFunc. +type MiddlewareFunc = queue.MiddlewareFunc -// Handle wraps matched errors as fatal errors to stop retries. -// @group Middleware +// RetryPolicy is the legacy pass-through retry policy helper. // -// Example: fail on any error -// -// mw := bus.FailOnError{ -// When: func(err error) bool { return err != nil }, -// } -// _ = mw -func (f FailOnError) Handle(ctx context.Context, jc Context, next Next) error { - err := next(ctx, jc) - if err == nil { - return nil - } - if f.When == nil || f.When(err) { - return fatalError{cause: err} - } - return err -} - -type fatalError struct { - cause error -} - -func (f fatalError) Error() string { return fmt.Sprintf("fatal bus error: %v", f.cause) } -func (f fatalError) Unwrap() error { return f.cause } - -type RateLimiter interface { - Allow(ctx context.Context, key string) (allowed bool, retryAfter time.Duration, err error) -} +// Deprecated: use queue.RetryPolicy. +type RetryPolicy = queue.RetryPolicy -type RateLimit struct { - Key func(ctx context.Context, jc Context) string - Limiter RateLimiter -} - -// Handle applies limiter checks before executing the next handler. -// @group Middleware +// SkipWhen skips execution when its predicate matches. // -// Example: rate limit middleware +// Deprecated: use queue.SkipWhen. +type SkipWhen = queue.SkipWhen + +// FailOnError converts matched errors into terminal failures. // -// mw := bus.RateLimit{ -// Key: func(context.Context, bus.Context) string { return "emails" }, -// } -// _ = mw -func (r RateLimit) Handle(ctx context.Context, jc Context, next Next) error { - if r.Limiter == nil { - return next(ctx, jc) - } - key := jc.JobType - if r.Key != nil { - if k := r.Key(ctx, jc); k != "" { - key = k - } - } - allowed, _, err := r.Limiter.Allow(ctx, key) - if err != nil { - return err - } - if !allowed { - return ErrRateLimited - } - return next(ctx, jc) -} +// Deprecated: use queue.FailOnError. +type FailOnError = queue.FailOnError -type Lock interface { - Release(ctx context.Context) error -} +// RateLimiter decides whether a keyed workflow job may execute. +// +// Deprecated: use queue.RateLimiter. +type RateLimiter = queue.RateLimiter -type Locker interface { - Acquire(ctx context.Context, key string, ttl time.Duration) (Lock, bool, error) -} +// RateLimit applies rate limiting before workflow job execution. +// +// Deprecated: use queue.RateLimit. +type RateLimit = queue.RateLimit -type WithoutOverlapping struct { - Key func(ctx context.Context, jc Context) string - TTL time.Duration - Locker Locker -} +// Lock is released after overlap-protected execution completes. +// +// Deprecated: use queue.Lock. +type Lock = queue.Lock -// Handle acquires a lock and prevents concurrent overlap for the same key. -// @group Middleware +// Locker acquires locks used to prevent overlapping execution. // -// Example: without overlapping +// Deprecated: use queue.Locker. +type Locker = queue.Locker + +// WithoutOverlapping prevents concurrent execution for one key. // -// mw := bus.WithoutOverlapping{ -// Key: func(context.Context, bus.Context) string { return "job-key" }, -// TTL: 30 * time.Second, -// } -// _ = mw -func (w WithoutOverlapping) Handle(ctx context.Context, jc Context, next Next) error { - if w.Locker == nil { - return next(ctx, jc) - } - key := jc.JobType - if w.Key != nil { - if k := w.Key(ctx, jc); k != "" { - key = k - } - } - lock, ok, err := w.Locker.Acquire(ctx, key, w.TTL) - if err != nil { - return err - } - if !ok { - return ErrOverlapping - } - defer func() { _ = lock.Release(ctx) }() - return next(ctx, jc) -} +// Deprecated: use queue.WithoutOverlapping. +type WithoutOverlapping = queue.WithoutOverlapping + +var ( + // ErrSkipped indicates middleware intentionally skipped workflow job execution. + // + // Deprecated: use queue.ErrSkipped. + ErrSkipped = queue.ErrSkipped + // ErrRateLimited indicates middleware denied workflow job execution under its current rate limit. + // + // Deprecated: use queue.ErrRateLimited. + ErrRateLimited = queue.ErrRateLimited + // ErrOverlapping indicates middleware prevented overlapping workflow job execution. + // + // Deprecated: use queue.ErrOverlapping. + ErrOverlapping = queue.ErrOverlapping +) diff --git a/bus/middleware_internal_test.go b/bus/middleware_internal_test.go deleted file mode 100644 index cf6f6b1..0000000 --- a/bus/middleware_internal_test.go +++ /dev/null @@ -1,15 +0,0 @@ -package bus - -import ( - "errors" - "testing" -) - -func TestFatalErrorUnwrap(t *testing.T) { - base := errors.New("boom") - err := fatalError{cause: base} - if !errors.Is(err, base) { - t.Fatal("expected fatalError to unwrap to base error") - } -} - diff --git a/bus/middleware_test.go b/bus/middleware_test.go index a7518c0..4e1d85b 100644 --- a/bus/middleware_test.go +++ b/bus/middleware_test.go @@ -3,7 +3,6 @@ package bus_test import ( "context" "errors" - "strings" "testing" "time" @@ -81,7 +80,8 @@ func TestSkipWhenMiddlewareSkipsHandler(t *testing.T) { } } -func TestFailOnErrorWrapsFatal(t *testing.T) { +// TestFailOnErrorMarksPermanent verifies middleware uses the shared terminal-error contract and stops physical retries. +func TestFailOnErrorMarksPermanent(t *testing.T) { q, err := newBusTestRuntime(queue.Config{Driver: queue.DriverSync}) if err != nil { t.Fatalf("new sync queue: %v", err) @@ -93,15 +93,24 @@ func TestFailOnErrorWrapsFatal(t *testing.T) { if err := b.StartWorkers(context.Background()); err != nil { t.Fatalf("start workers: %v", err) } + cause := errors.New("boom") + calls := 0 b.Register("monitor:poll", func(context.Context, bus.Context) error { - return errors.New("boom") + calls++ + return cause }) - _, err = b.Dispatch(context.Background(), bus.NewJob("monitor:poll", nil)) + _, err = b.Dispatch(context.Background(), bus.NewJob("monitor:poll", nil).Retry(4)) if err == nil { t.Fatal("expected error") } - if !strings.Contains(err.Error(), "fatal bus error") { - t.Fatalf("expected fatal wrapping, got %v", err) + if !queue.IsPermanent(err) || !errors.Is(err, cause) { + t.Fatalf("expected permanent error preserving cause, got %v", err) + } + if err.Error() != "fatal bus error: boom" { + t.Fatalf("error text = %q, want compatibility prefix", err.Error()) + } + if calls != 1 { + t.Fatalf("handler calls = %d, want 1", calls) } } diff --git a/bus/runtime_cleanup_test.go b/bus/runtime_cleanup_test.go deleted file mode 100644 index 7f76d93..0000000 --- a/bus/runtime_cleanup_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package bus - -import ( - "context" - "testing" -) - -func TestRuntimeCleansChainCallbacksAfterFinally(t *testing.T) { - q := newSyncTestRuntime() - bi, err := New(q) - if err != nil { - t.Fatalf("new bus: %v", err) - } - r := bi.(*runtime) - if err := r.StartWorkers(context.Background()); err != nil { - t.Fatalf("start workers: %v", err) - } - - r.Register("monitor:poll", func(context.Context, Context) error { return nil }) - if _, err := r.Chain(NewJob("monitor:poll", nil)). - Finally(func(context.Context, ChainState) error { return nil }). - Dispatch(context.Background()); err != nil { - t.Fatalf("dispatch chain: %v", err) - } - - r.mu.RLock() - n := len(r.chainCallbacks) - r.mu.RUnlock() - if n != 0 { - t.Fatalf("expected chain callbacks map cleaned, got len=%d", n) - } -} - -func TestRuntimeCleansBatchCallbacksAfterFinally(t *testing.T) { - q := newSyncTestRuntime() - bi, err := New(q) - if err != nil { - t.Fatalf("new bus: %v", err) - } - r := bi.(*runtime) - if err := r.StartWorkers(context.Background()); err != nil { - t.Fatalf("start workers: %v", err) - } - - r.Register("monitor:poll", func(context.Context, Context) error { return nil }) - if _, err := r.Batch(NewJob("monitor:poll", nil)). - Finally(func(context.Context, BatchState) error { return nil }). - Dispatch(context.Background()); err != nil { - t.Fatalf("dispatch batch: %v", err) - } - - r.mu.RLock() - n := len(r.batchCallbacks) - r.mu.RUnlock() - if n != 0 { - t.Fatalf("expected batch callbacks map cleaned, got len=%d", n) - } -} diff --git a/bus/source_compat_test.go b/bus/source_compat_test.go new file mode 100644 index 0000000..43ed328 --- /dev/null +++ b/bus/source_compat_test.go @@ -0,0 +1,449 @@ +package bus_test + +import ( + "context" + "database/sql" + "testing" + "time" + + "github.com/goforj/queue" + "github.com/goforj/queue/bus" + "github.com/goforj/queue/bus/driver/temporal" +) + +var ( + _ bus.Bus = (*sourceCompatBus)(nil) + _ bus.Store = (*sourceCompatStore)(nil) + _ bus.Observer = sourceCompatObserver{} + _ bus.Middleware = sourceCompatMiddleware{} + _ bus.ChainBuilder = (*sourceCompatChainBuilder)(nil) + _ bus.BatchBuilder = (*sourceCompatBatchBuilder)(nil) + _ queue.ChainBuilder = (*sourceCompatRootChainBuilder)(nil) + _ queue.BatchBuilder = (*sourceCompatRootBatchBuilder)(nil) + _ bus.Bus = (*temporal.Adapter)(nil) +) + +var ( + sourceCompatBusContextFromRoot bus.Context = queue.Message{SchemaVersion: 1} + sourceCompatRootMessageFromBus queue.Message = bus.Context{SchemaVersion: 1} + sourceCompatBusResultFromRoot bus.DispatchResult = queue.DispatchResult{DispatchID: "root"} + sourceCompatRootResultFromBus queue.DispatchResult = bus.DispatchResult{DispatchID: "bus"} + sourceCompatBusOptionsFromRoot bus.JobOptions = queue.StoredJobOptions{Queue: "root"} + sourceCompatRootOptionsFromBus queue.StoredJobOptions = bus.JobOptions{Queue: "bus"} + sourceCompatBusChainStateFromRoot bus.ChainState = queue.ChainState{ChainID: "root-chain"} + sourceCompatRootChainStateFromBus queue.ChainState = bus.ChainState{ChainID: "bus-chain"} + sourceCompatBusBatchStateFromRoot bus.BatchState = queue.BatchState{BatchID: "root-batch"} + sourceCompatRootBatchStateFromBus queue.BatchState = bus.BatchState{BatchID: "bus-batch"} + sourceCompatRootMiddleware queue.Middleware = sourceCompatMiddleware{} + sourceCompatBusMiddlewareFromRoot bus.Middleware = sourceCompatRootMiddleware + sourceCompatRootMiddlewareRoundTrip queue.Middleware = sourceCompatBusMiddlewareFromRoot + sourceCompatRootStore queue.WorkflowStore = &sourceCompatStore{} + sourceCompatBusStoreFromRoot bus.Store = sourceCompatRootStore + sourceCompatRootStoreRoundTrip queue.WorkflowStore = sourceCompatBusStoreFromRoot + sourceCompatRootMiddlewareFunc queue.MiddlewareFunc = bus.MiddlewareFunc(func(ctx context.Context, message bus.Context, next bus.Next) error { + return next(ctx, message) + }) + sourceCompatBusMiddlewareFunc bus.MiddlewareFunc = sourceCompatRootMiddlewareFunc +) + +type sourceCompatBus struct{} + +type sourceCompatJobOptions bus.JobOptions + +type sourceCompatDispatchResult bus.DispatchResult + +type sourceCompatChainRecord bus.ChainRecord + +type sourceCompatChainState bus.ChainState + +type sourceCompatBatchRecord bus.BatchRecord + +type sourceCompatBatchState bus.BatchState + +type sourceCompatSQLStoreConfig bus.SQLStoreConfig + +// Register accepts the legacy named bus handler contract. +func (*sourceCompatBus) Register(string, bus.Handler) {} + +// Dispatch returns a legacy dispatch result for a legacy job value. +func (*sourceCompatBus) Dispatch(context.Context, bus.Job) (bus.DispatchResult, error) { + return bus.DispatchResult{}, nil +} + +// Chain returns a custom builder with the legacy self-returning method set. +func (*sourceCompatBus) Chain(...bus.Job) bus.ChainBuilder { + return &sourceCompatChainBuilder{} +} + +// Batch returns a custom builder with the legacy self-returning method set. +func (*sourceCompatBus) Batch(...bus.Job) bus.BatchBuilder { + return &sourceCompatBatchBuilder{} +} + +// StartWorkers preserves the legacy lifecycle signature. +func (*sourceCompatBus) StartWorkers(context.Context) error { return nil } + +// Shutdown preserves the legacy lifecycle signature. +func (*sourceCompatBus) Shutdown(context.Context) error { return nil } + +// FindBatch returns the legacy batch state type. +func (*sourceCompatBus) FindBatch(context.Context, string) (bus.BatchState, error) { + return bus.BatchState{}, nil +} + +// FindChain returns the legacy chain state type. +func (*sourceCompatBus) FindChain(context.Context, string) (bus.ChainState, error) { + return bus.ChainState{}, nil +} + +// Prune preserves the legacy workflow retention signature. +func (*sourceCompatBus) Prune(context.Context, time.Time) error { return nil } + +type sourceCompatStore struct{} + +// CreateChain accepts the legacy chain record type. +func (*sourceCompatStore) CreateChain(context.Context, bus.ChainRecord) error { return nil } + +// AdvanceChain returns the legacy opaque chain node type. +func (*sourceCompatStore) AdvanceChain(context.Context, string, string) (*bus.ChainNode, bool, error) { + return nil, false, nil +} + +// FailChain preserves the legacy chain failure signature. +func (*sourceCompatStore) FailChain(context.Context, string, error) error { return nil } + +// GetChain returns the legacy chain state type. +func (*sourceCompatStore) GetChain(context.Context, string) (bus.ChainState, error) { + return bus.ChainState{}, nil +} + +// CreateBatch accepts the legacy batch record type. +func (*sourceCompatStore) CreateBatch(context.Context, bus.BatchRecord) error { return nil } + +// MarkBatchJobStarted preserves the legacy batch start signature. +func (*sourceCompatStore) MarkBatchJobStarted(context.Context, string, string) error { return nil } + +// MarkBatchJobSucceeded returns the legacy batch state and completion flag. +func (*sourceCompatStore) MarkBatchJobSucceeded(context.Context, string, string) (bus.BatchState, bool, error) { + return bus.BatchState{}, false, nil +} + +// MarkBatchJobFailed preserves the legacy failure-cause argument and result types. +func (*sourceCompatStore) MarkBatchJobFailed(context.Context, string, string, error) (bus.BatchState, bool, error) { + return bus.BatchState{}, false, nil +} + +// CancelBatch preserves the legacy batch cancellation signature. +func (*sourceCompatStore) CancelBatch(context.Context, string) error { return nil } + +// GetBatch returns the legacy batch state type. +func (*sourceCompatStore) GetBatch(context.Context, string) (bus.BatchState, error) { + return bus.BatchState{}, nil +} + +// MarkCallbackInvoked preserves the legacy callback idempotency signature. +func (*sourceCompatStore) MarkCallbackInvoked(context.Context, string) (bool, error) { + return true, nil +} + +// Prune preserves the legacy store retention signature. +func (*sourceCompatStore) Prune(context.Context, time.Time) error { return nil } + +type sourceCompatObserver struct{} + +// Observe accepts the keyed legacy event model without relying on its rejected unkeyed layout. +func (sourceCompatObserver) Observe(context.Context, bus.Event) {} + +type sourceCompatMiddleware struct{} + +// Handle accepts the legacy context and continuation types. +func (sourceCompatMiddleware) Handle(ctx context.Context, message bus.Context, next bus.Next) error { + return next(ctx, message) +} + +type sourceCompatChainBuilder struct{} + +// OnQueue returns the legacy chain builder interface. +func (builder *sourceCompatChainBuilder) OnQueue(string) bus.ChainBuilder { return builder } + +// Catch accepts the legacy chain state callback. +func (builder *sourceCompatChainBuilder) Catch(func(context.Context, bus.ChainState, error) error) bus.ChainBuilder { + return builder +} + +// Finally accepts the legacy terminal chain callback. +func (builder *sourceCompatChainBuilder) Finally(func(context.Context, bus.ChainState) error) bus.ChainBuilder { + return builder +} + +// Dispatch preserves the legacy chain dispatch signature. +func (*sourceCompatChainBuilder) Dispatch(context.Context) (string, error) { return "", nil } + +type sourceCompatBatchBuilder struct{} + +// Name returns the legacy batch builder interface. +func (builder *sourceCompatBatchBuilder) Name(string) bus.BatchBuilder { return builder } + +// OnQueue returns the legacy batch builder interface. +func (builder *sourceCompatBatchBuilder) OnQueue(string) bus.BatchBuilder { return builder } + +// AllowFailures returns the legacy batch builder interface. +func (builder *sourceCompatBatchBuilder) AllowFailures() bus.BatchBuilder { return builder } + +// Progress accepts the legacy batch progress callback. +func (builder *sourceCompatBatchBuilder) Progress(func(context.Context, bus.BatchState) error) bus.BatchBuilder { + return builder +} + +// Then accepts the legacy successful batch callback. +func (builder *sourceCompatBatchBuilder) Then(func(context.Context, bus.BatchState) error) bus.BatchBuilder { + return builder +} + +// Catch accepts the legacy failed batch callback. +func (builder *sourceCompatBatchBuilder) Catch(func(context.Context, bus.BatchState, error) error) bus.BatchBuilder { + return builder +} + +// Finally accepts the legacy terminal batch callback. +func (builder *sourceCompatBatchBuilder) Finally(func(context.Context, bus.BatchState) error) bus.BatchBuilder { + return builder +} + +// Dispatch preserves the legacy batch dispatch signature. +func (*sourceCompatBatchBuilder) Dispatch(context.Context) (string, error) { return "", nil } + +type sourceCompatRootChainBuilder struct{} + +// OnQueue returns the root chain builder interface. +func (builder *sourceCompatRootChainBuilder) OnQueue(string) queue.ChainBuilder { return builder } + +// Catch accepts the root chain state callback. +func (builder *sourceCompatRootChainBuilder) Catch(func(context.Context, queue.ChainState, error) error) queue.ChainBuilder { + return builder +} + +// Finally accepts the root terminal chain callback. +func (builder *sourceCompatRootChainBuilder) Finally(func(context.Context, queue.ChainState) error) queue.ChainBuilder { + return builder +} + +// Dispatch preserves the root chain dispatch signature. +func (*sourceCompatRootChainBuilder) Dispatch(context.Context) (string, error) { return "", nil } + +type sourceCompatRootBatchBuilder struct{} + +// Name returns the root batch builder interface. +func (builder *sourceCompatRootBatchBuilder) Name(string) queue.BatchBuilder { return builder } + +// OnQueue returns the root batch builder interface. +func (builder *sourceCompatRootBatchBuilder) OnQueue(string) queue.BatchBuilder { return builder } + +// AllowFailures returns the root batch builder interface. +func (builder *sourceCompatRootBatchBuilder) AllowFailures() queue.BatchBuilder { return builder } + +// Progress accepts the root batch progress callback. +func (builder *sourceCompatRootBatchBuilder) Progress(func(context.Context, queue.BatchState) error) queue.BatchBuilder { + return builder +} + +// Then accepts the root successful batch callback. +func (builder *sourceCompatRootBatchBuilder) Then(func(context.Context, queue.BatchState) error) queue.BatchBuilder { + return builder +} + +// Catch accepts the root failed batch callback. +func (builder *sourceCompatRootBatchBuilder) Catch(func(context.Context, queue.BatchState, error) error) queue.BatchBuilder { + return builder +} + +// Finally accepts the root terminal batch callback. +func (builder *sourceCompatRootBatchBuilder) Finally(func(context.Context, queue.BatchState) error) queue.BatchBuilder { + return builder +} + +// Dispatch preserves the root batch dispatch signature. +func (*sourceCompatRootBatchBuilder) Dispatch(context.Context) (string, error) { return "", nil } + +// TestBusV1SourceCompatibility freezes the external source forms retained by the deprecated forwarding facade. +func TestBusV1SourceCompatibility(t *testing.T) { + fixedTime := time.Unix(1_704_067_200, 123_000_000) + keyedOptions := bus.JobOptions{ + Queue: "critical", + Delay: time.Second, + Timeout: 2 * time.Second, + Retry: 3, + Backoff: 4 * time.Second, + UniqueFor: 5 * time.Second, + } + unkeyedOptions := bus.JobOptions(sourceCompatJobOptions{"bulk", 6 * time.Second, 7 * time.Second, 8, 9 * time.Second, 10 * time.Second}) + keyedJob := bus.Job{Type: "reports:keyed", Payload: map[string]int{"id": 1}, Options: keyedOptions} + unkeyedJob := bus.Job{"reports:unkeyed", map[string]int{"id": 2}, unkeyedOptions} + mutableJob := bus.NewJob("reports:mutable", nil) + mutableJob.Type = "reports:mutated" + mutableJob.Payload = []byte("payload") + mutableJob.Options = keyedOptions + mutableJob.Options.Queue = "mutated" + mutableJob.Options.Retry = 11 + if keyedJob.Type != "reports:keyed" || unkeyedJob.Type != "reports:unkeyed" || mutableJob.Type != "reports:mutated" || mutableJob.Options.Queue != "mutated" || mutableJob.Options.Retry != 11 { + t.Fatalf("legacy job source forms changed: keyed=%+v unkeyed=%+v mutable=%+v", keyedJob, unkeyedJob, mutableJob) + } + if unkeyedOptions.Queue != "bulk" || unkeyedOptions.Delay != 6*time.Second || unkeyedOptions.Timeout != 7*time.Second || unkeyedOptions.Retry != 8 || unkeyedOptions.Backoff != 9*time.Second || unkeyedOptions.UniqueFor != 10*time.Second { + t.Fatalf("legacy unkeyed job option order changed: %+v", unkeyedOptions) + } + storedJob := bus.StoredJob{Type: "reports:stored", Payload: []byte(`{"id":3}`), Options: keyedOptions} + storedNode := bus.ChainNode{NodeID: "stored-node", Job: storedJob} + var selectedOptions bus.JobOptions = storedNode.Job.Options + storedNode.Job.Options = selectedOptions + var rootStoredJob queue.StoredJob = storedNode.Job + var busStoredJob bus.StoredJob = rootStoredJob + if busStoredJob.Type != "reports:stored" || busStoredJob.Options.Queue != "critical" { + t.Fatalf("legacy stored job selectors changed: %+v", busStoredJob) + } + + keyedResult := bus.DispatchResult{DispatchID: "dispatch-keyed"} + unkeyedResult := bus.DispatchResult(sourceCompatDispatchResult{"dispatch-unkeyed"}) + if keyedResult.DispatchID != "dispatch-keyed" || unkeyedResult.DispatchID != "dispatch-unkeyed" { + t.Fatalf("legacy dispatch result source forms changed: keyed=%+v unkeyed=%+v", keyedResult, unkeyedResult) + } + + keyedChainRecord := bus.ChainRecord{ + ChainID: "chain-keyed", + DispatchID: "dispatch-keyed", + Queue: "critical", + Nodes: nil, + CreatedAt: fixedTime, + } + unkeyedChainRecord := bus.ChainRecord(sourceCompatChainRecord{"chain-unkeyed", "dispatch-unkeyed", "bulk", nil, fixedTime}) + keyedChainState := bus.ChainState{ + ChainID: "chain-state-keyed", + DispatchID: "dispatch-state-keyed", + Queue: "critical", + Nodes: nil, + NextIndex: 2, + Completed: true, + Failed: false, + Failure: "", + CreatedAt: fixedTime, + UpdatedAt: fixedTime.Add(time.Second), + } + unkeyedChainState := bus.ChainState(sourceCompatChainState{"chain-state-unkeyed", "dispatch-state-unkeyed", "bulk", nil, 1, false, true, "failed", fixedTime, fixedTime.Add(2 * time.Second)}) + if keyedChainRecord.ChainID != "chain-keyed" || unkeyedChainRecord.Queue != "bulk" || keyedChainState.NextIndex != 2 || !unkeyedChainState.Failed || unkeyedChainState.Failure != "failed" { + t.Fatalf("legacy chain source forms changed: records=%+v/%+v states=%+v/%+v", keyedChainRecord, unkeyedChainRecord, keyedChainState, unkeyedChainState) + } + + keyedBatchRecord := bus.BatchRecord{ + BatchID: "batch-keyed", + DispatchID: "batch-dispatch-keyed", + Name: "keyed batch", + Queue: "critical", + AllowFailed: true, + Jobs: nil, + CreatedAt: fixedTime, + } + unkeyedBatchRecord := bus.BatchRecord(sourceCompatBatchRecord{"batch-unkeyed", "batch-dispatch-unkeyed", "unkeyed batch", "bulk", false, nil, fixedTime}) + keyedBatchState := bus.BatchState{ + BatchID: "batch-state-keyed", + DispatchID: "batch-state-dispatch-keyed", + Name: "keyed state", + Queue: "critical", + AllowFailed: true, + Total: 4, + Pending: 1, + Processed: 3, + Failed: 1, + Cancelled: false, + Completed: true, + CreatedAt: fixedTime, + UpdatedAt: fixedTime.Add(time.Second), + } + unkeyedBatchState := bus.BatchState(sourceCompatBatchState{"batch-state-unkeyed", "batch-state-dispatch-unkeyed", "unkeyed state", "bulk", false, 5, 2, 3, 1, true, true, fixedTime, fixedTime.Add(2 * time.Second)}) + if keyedBatchRecord.BatchID != "batch-keyed" || unkeyedBatchRecord.Name != "unkeyed batch" || keyedBatchState.Processed != 3 || !unkeyedBatchState.Cancelled || unkeyedBatchState.Total != 5 { + t.Fatalf("legacy batch source forms changed: records=%+v/%+v states=%+v/%+v", keyedBatchRecord, unkeyedBatchRecord, keyedBatchState, unkeyedBatchState) + } + + keyedSQLConfig := bus.SQLStoreConfig{DB: (*sql.DB)(nil), DriverName: "sqlite", DSN: "file:keyed", AutoMigrate: true} + unkeyedSQLConfig := bus.SQLStoreConfig(sourceCompatSQLStoreConfig{nil, "sqlite", "file:unkeyed", true}) + if keyedSQLConfig.DriverName != "sqlite" || keyedSQLConfig.DSN != "file:keyed" || !keyedSQLConfig.AutoMigrate || unkeyedSQLConfig.DSN != "file:unkeyed" || !unkeyedSQLConfig.AutoMigrate { + t.Fatalf("legacy SQL store config source forms changed: keyed=%+v unkeyed=%+v", keyedSQLConfig, unkeyedSQLConfig) + } + + options := []bus.Option{ + nil, + bus.WithObserver(sourceCompatObserver{}), + bus.WithStore(&sourceCompatStore{}), + bus.WithClock(func() time.Time { return fixedTime }), + bus.WithMiddleware(sourceCompatMiddleware{}), + } + if len(options) != 5 || options[0] != nil || options[1] == nil || options[2] == nil || options[3] == nil || options[4] == nil { + t.Fatalf("legacy option slice changed: %+v", options) + } + + fake := bus.NewFake() + var concreteFake *bus.Fake = fake + if _, err := concreteFake.Dispatch(context.Background(), keyedJob); err != nil { + t.Fatalf("dispatch through legacy fake: %v", err) + } + if _, err := concreteFake.Chain(keyedJob, unkeyedJob).Dispatch(context.Background()); err != nil { + t.Fatalf("chain through legacy fake: %v", err) + } + if _, err := concreteFake.Batch(keyedJob, unkeyedJob).Dispatch(context.Background()); err != nil { + t.Fatalf("batch through legacy fake: %v", err) + } + keyedSpec := bus.BatchSpec{JobTypes: []string{"reports:keyed", "reports:unkeyed"}} + unkeyedSpec := bus.BatchSpec{[]string{"reports:keyed", "reports:unkeyed"}} + if len(keyedSpec.JobTypes) != 2 || len(unkeyedSpec.JobTypes) != 2 { + t.Fatalf("legacy batch spec source forms changed: keyed=%+v unkeyed=%+v", keyedSpec, unkeyedSpec) + } + concreteFake.AssertDispatched(t, "reports:keyed") + concreteFake.AssertChained(t, keyedSpec.JobTypes) + concreteFake.AssertBatched(t, func(spec bus.BatchSpec) bool { + return len(spec.JobTypes) == 2 && spec.JobTypes[0] == "reports:keyed" && spec.JobTypes[1] == "reports:unkeyed" + }) + + temporalAdapter, err := temporal.New(temporal.Config{}) + if err != nil { + t.Fatalf("construct temporal compatibility adapter: %v", err) + } + var temporalBus bus.Bus = temporalAdapter + if temporalBus == nil { + t.Fatal("temporal adapter no longer satisfies bus.Bus") + } + + _ = []any{ + sourceCompatBusContextFromRoot, + sourceCompatRootMessageFromBus, + sourceCompatBusResultFromRoot, + sourceCompatRootResultFromBus, + sourceCompatBusOptionsFromRoot, + sourceCompatRootOptionsFromBus, + sourceCompatBusChainStateFromRoot, + sourceCompatRootChainStateFromBus, + sourceCompatBusBatchStateFromRoot, + sourceCompatRootBatchStateFromBus, + sourceCompatRootMiddlewareRoundTrip, + sourceCompatRootStoreRoundTrip, + sourceCompatBusMiddlewareFunc, + } +} + +// TestBuilderInterfacesRemainSourceDistinct pins the legacy self-returning method sets. +func TestBuilderInterfacesRemainSourceDistinct(t *testing.T) { + var legacyChain bus.ChainBuilder = &sourceCompatChainBuilder{} + var rootChain queue.ChainBuilder = &sourceCompatRootChainBuilder{} + if _, ok := any(legacyChain).(queue.ChainBuilder); ok { + t.Fatal("legacy chain builder unexpectedly satisfies the root self-returning contract") + } + if _, ok := any(rootChain).(bus.ChainBuilder); ok { + t.Fatal("root chain builder unexpectedly satisfies the legacy self-returning contract") + } + + var legacyBatch bus.BatchBuilder = &sourceCompatBatchBuilder{} + var rootBatch queue.BatchBuilder = &sourceCompatRootBatchBuilder{} + if _, ok := any(legacyBatch).(queue.BatchBuilder); ok { + t.Fatal("legacy batch builder unexpectedly satisfies the root self-returning contract") + } + if _, ok := any(rootBatch).(bus.BatchBuilder); ok { + t.Fatal("root batch builder unexpectedly satisfies the legacy self-returning contract") + } +} diff --git a/bus/store.go b/bus/store.go index 4af09c8..e21ac7b 100644 --- a/bus/store.go +++ b/bus/store.go @@ -1,83 +1,96 @@ package bus -import ( - "context" - "errors" - "time" +import "github.com/goforj/queue" + +// StoredJob is the stable logical-job shape persisted inside workflow records. +// +// Deprecated: use queue.StoredJob. +type StoredJob = queue.StoredJob + +// ChainNode is one persisted chain step. +// +// Deprecated: use queue.ChainNode. +type ChainNode = queue.ChainNode + +// ChainRecord is the persisted representation used to create a chain. +// +// Deprecated: use queue.ChainRecord. +type ChainRecord = queue.ChainRecord + +// ChainState is the persisted view of a chain workflow. +// +// Deprecated: use queue.ChainState. +type ChainState = queue.ChainState + +// BatchJob is one persisted batch member. +// +// Deprecated: use queue.BatchJob. +type BatchJob = queue.BatchJob + +// BatchJobOutcome identifies the durable result that first settled one batch member. +// +// Deprecated: use queue.BatchJobOutcome. +type BatchJobOutcome = queue.BatchJobOutcome + +const ( + // BatchJobSucceeded records successful member settlement. + // + // Deprecated: use queue.BatchJobSucceeded. + BatchJobSucceeded = queue.BatchJobSucceeded + // BatchJobFailed records failed member settlement. + // + // Deprecated: use queue.BatchJobFailed. + BatchJobFailed = queue.BatchJobFailed ) -var ErrNotFound = errors.New("bus record not found") +// BatchRecord is the persisted representation used to create a batch. +// +// Deprecated: use queue.BatchRecord. +type BatchRecord = queue.BatchRecord -type ChainNode struct { - NodeID string - Job wireJob -} +// BatchState is the persisted view of a batch workflow. +// +// Deprecated: use queue.BatchState. +type BatchState = queue.BatchState -type ChainRecord struct { - ChainID string - DispatchID string - Queue string - Nodes []ChainNode - CreatedAt time.Time -} +// Store persists chain, batch, and callback state. +// +// Deprecated: use queue.WorkflowStore. +type Store = queue.WorkflowStore -type ChainState struct { - ChainID string - DispatchID string - Queue string - Nodes []ChainNode - NextIndex int - Completed bool - Failed bool - Failure string - CreatedAt time.Time - UpdatedAt time.Time -} +// WorkflowOutcomeStore adds first-writer ownership to the compatibility store contract. +// +// Deprecated: use queue.WorkflowOutcomeStore. +type WorkflowOutcomeStore = queue.WorkflowOutcomeStore -type BatchRecord struct { - BatchID string - DispatchID string - Name string - Queue string - AllowFailed bool - Jobs []BatchJob - CreatedAt time.Time -} +// SQLStoreConfig configures the SQL-backed workflow store. +// +// Deprecated: use queue.SQLStoreConfig. +type SQLStoreConfig = queue.SQLStoreConfig + +// ErrNotFound indicates a workflow state record is not present. +// +// Deprecated: use queue.ErrWorkflowNotFound. +var ErrNotFound = queue.ErrWorkflowNotFound -type BatchJob struct { - JobID string - Job wireJob +// NewMemoryStore creates an in-memory workflow state store. +// +// Deprecated: use queue.NewMemoryStore. +func NewMemoryStore() Store { + return queue.NewMemoryStore() } -type BatchState struct { - BatchID string - DispatchID string - Name string - Queue string - AllowFailed bool - Total int - Pending int - Processed int - Failed int - Cancelled bool - Completed bool - CreatedAt time.Time - UpdatedAt time.Time +// NewSQLStore creates a SQL-backed workflow state store. +// +// Deprecated: use queue.NewSQLStore. +func NewSQLStore(cfg SQLStoreConfig) (Store, error) { + return queue.NewSQLStore(cfg) } -type Store interface { - CreateChain(ctx context.Context, rec ChainRecord) error - AdvanceChain(ctx context.Context, chainID string, completedNode string) (next *ChainNode, done bool, err error) - FailChain(ctx context.Context, chainID string, cause error) error - GetChain(ctx context.Context, chainID string) (ChainState, error) - - CreateBatch(ctx context.Context, rec BatchRecord) error - MarkBatchJobStarted(ctx context.Context, batchID, jobID string) error - MarkBatchJobSucceeded(ctx context.Context, batchID, jobID string) (BatchState, bool, error) - MarkBatchJobFailed(ctx context.Context, batchID, jobID string, cause error) (BatchState, bool, error) - CancelBatch(ctx context.Context, batchID string) error - GetBatch(ctx context.Context, batchID string) (BatchState, error) - - MarkCallbackInvoked(ctx context.Context, key string) (bool, error) - Prune(ctx context.Context, before time.Time) error +// NewSQLStoreWithManagedSchema creates a SQL-backed workflow state store +// without executing schema DDL. +// +// Deprecated: use queue.NewSQLStoreWithManagedSchema. +func NewSQLStoreWithManagedSchema(cfg SQLStoreConfig) (Store, error) { + return queue.NewSQLStoreWithManagedSchema(cfg) } diff --git a/bus/store_contract_test.go b/bus/store_contract_test.go deleted file mode 100644 index 03a5d79..0000000 --- a/bus/store_contract_test.go +++ /dev/null @@ -1,299 +0,0 @@ -package bus - -import ( - "context" - "errors" - "path/filepath" - "testing" - "time" - - _ "modernc.org/sqlite" -) - -func testStoreFactories(t *testing.T) map[string]func(t *testing.T) Store { - t.Helper() - return map[string]func(t *testing.T) Store{ - "memory": func(t *testing.T) Store { - t.Helper() - return NewMemoryStore() - }, - "sql_sqlite": func(t *testing.T) Store { - t.Helper() - dsn := filepath.Join(t.TempDir(), "store-contract.db") - store, err := NewSQLStore(SQLStoreConfig{ - DriverName: "sqlite", - DSN: dsn, - }) - if err != nil { - t.Fatalf("new sql store: %v", err) - } - return store - }, - } -} - -func TestStoreContract_NotFound(t *testing.T) { - for name, factory := range testStoreFactories(t) { - t.Run(name, func(t *testing.T) { - s := factory(t) - ctx := context.Background() - - if _, err := s.GetChain(ctx, "missing"); !errors.Is(err, ErrNotFound) { - t.Fatalf("expected chain ErrNotFound, got %v", err) - } - if _, err := s.GetBatch(ctx, "missing"); !errors.Is(err, ErrNotFound) { - t.Fatalf("expected batch ErrNotFound, got %v", err) - } - }) - } -} - -func TestStoreContract_ChainAdvanceIdempotent(t *testing.T) { - for name, factory := range testStoreFactories(t) { - t.Run(name, func(t *testing.T) { - s := factory(t) - ctx := context.Background() - chainID := "chain-contract" - - if err := s.CreateChain(ctx, ChainRecord{ - ChainID: chainID, - DispatchID: "d1", - Queue: "default", - Nodes: []ChainNode{ - {NodeID: "n1", Job: wireJob{Type: "monitor:poll"}}, - {NodeID: "n2", Job: wireJob{Type: "monitor:downsample"}}, - }, - CreatedAt: time.Now(), - }); err != nil { - t.Fatalf("create chain: %v", err) - } - - next, done, err := s.AdvanceChain(ctx, chainID, "n1") - if err != nil { - t.Fatalf("advance first: %v", err) - } - if done || next == nil || next.NodeID != "n2" { - t.Fatalf("expected next n2 on first advance, done=%v next=%+v", done, next) - } - - next, done, err = s.AdvanceChain(ctx, chainID, "n1") - if err != nil { - t.Fatalf("advance duplicate: %v", err) - } - if done || next == nil || next.NodeID != "n2" { - t.Fatalf("expected idempotent duplicate advance, done=%v next=%+v", done, next) - } - - next, done, err = s.AdvanceChain(ctx, chainID, "n2") - if err != nil { - t.Fatalf("advance final: %v", err) - } - if !done || next != nil { - t.Fatalf("expected chain done with nil next, done=%v next=%+v", done, next) - } - }) - } -} - -func TestStoreContract_BatchTerminalBehavior(t *testing.T) { - for name, factory := range testStoreFactories(t) { - t.Run(name, func(t *testing.T) { - s := factory(t) - ctx := context.Background() - batchID := "batch-contract" - - if err := s.CreateBatch(ctx, BatchRecord{ - BatchID: batchID, - DispatchID: "d1", - Name: "contract", - Queue: "default", - AllowFailed: false, - Jobs: []BatchJob{ - {JobID: "j1", Job: wireJob{Type: "monitor:poll"}}, - {JobID: "j2", Job: wireJob{Type: "monitor:downsample"}}, - }, - CreatedAt: time.Now(), - }); err != nil { - t.Fatalf("create batch: %v", err) - } - - st, done, err := s.MarkBatchJobSucceeded(ctx, batchID, "j1") - if err != nil { - t.Fatalf("mark success: %v", err) - } - if done { - t.Fatal("expected batch not done after first success") - } - if st.Pending != 1 || st.Processed != 1 || st.Failed != 0 { - t.Fatalf("unexpected mid state: %+v", st) - } - - st, done, err = s.MarkBatchJobFailed(ctx, batchID, "j2", errors.New("boom")) - if err != nil { - t.Fatalf("mark failed: %v", err) - } - if !done { - t.Fatal("expected batch done on failure when allow_failed=false") - } - if !st.Completed || !st.Cancelled || st.Failed != 1 { - t.Fatalf("unexpected terminal state: %+v", st) - } - }) - } -} - -func TestStoreContract_CallbackMarkerIdempotent(t *testing.T) { - for name, factory := range testStoreFactories(t) { - t.Run(name, func(t *testing.T) { - s := factory(t) - ctx := context.Background() - key := "batch_finally:contract" - - first, err := s.MarkCallbackInvoked(ctx, key) - if err != nil { - t.Fatalf("first callback marker: %v", err) - } - if !first { - t.Fatal("expected first callback marker=true") - } - - second, err := s.MarkCallbackInvoked(ctx, key) - if err != nil { - t.Fatalf("second callback marker: %v", err) - } - if second { - t.Fatal("expected second callback marker=false") - } - }) - } -} - -func TestStoreContract_PruneClearsOldCallbackMarkers(t *testing.T) { - for name, factory := range testStoreFactories(t) { - t.Run(name, func(t *testing.T) { - s := factory(t) - ctx := context.Background() - key := "batch_finally:contract-prune" - - first, err := s.MarkCallbackInvoked(ctx, key) - if err != nil { - t.Fatalf("first callback marker: %v", err) - } - if !first { - t.Fatal("expected first callback marker=true") - } - - // Future cutoff ensures just-inserted marker is considered old. - if err := s.Prune(ctx, time.Now().Add(1*time.Minute)); err != nil { - t.Fatalf("prune markers: %v", err) - } - - again, err := s.MarkCallbackInvoked(ctx, key) - if err != nil { - t.Fatalf("callback marker after prune: %v", err) - } - if !again { - t.Fatal("expected callback marker to be insertable again after prune") - } - }) - } -} - -func TestStoreContract_BatchAllowFailuresContinues(t *testing.T) { - for name, factory := range testStoreFactories(t) { - t.Run(name, func(t *testing.T) { - s := factory(t) - ctx := context.Background() - batchID := "batch-allow-fail-contract" - - if err := s.CreateBatch(ctx, BatchRecord{ - BatchID: batchID, - DispatchID: "d1", - Name: "allow-fail", - Queue: "default", - AllowFailed: true, - Jobs: []BatchJob{ - {JobID: "j1", Job: wireJob{Type: "monitor:poll"}}, - {JobID: "j2", Job: wireJob{Type: "monitor:downsample"}}, - {JobID: "j3", Job: wireJob{Type: "monitor:alert"}}, - }, - CreatedAt: time.Now(), - }); err != nil { - t.Fatalf("create batch: %v", err) - } - - st, done, err := s.MarkBatchJobFailed(ctx, batchID, "j1", errors.New("boom")) - if err != nil { - t.Fatalf("mark first failed: %v", err) - } - if done { - t.Fatal("expected batch to continue when allow_failed=true") - } - if st.Cancelled { - t.Fatal("expected batch not cancelled when allow_failed=true") - } - - st, done, err = s.MarkBatchJobSucceeded(ctx, batchID, "j2") - if err != nil { - t.Fatalf("mark second success: %v", err) - } - if done { - t.Fatal("expected batch still not done after second job") - } - - st, done, err = s.MarkBatchJobSucceeded(ctx, batchID, "j3") - if err != nil { - t.Fatalf("mark third success: %v", err) - } - if !done || !st.Completed { - t.Fatalf("expected batch completed, done=%v state=%+v", done, st) - } - if st.Failed != 1 || st.Processed != 3 || st.Pending != 0 { - t.Fatalf("unexpected final counters: %+v", st) - } - }) - } -} - -func TestStoreContract_BatchDuplicateTerminalUpdateDoesNotDoubleCount(t *testing.T) { - for name, factory := range testStoreFactories(t) { - t.Run(name, func(t *testing.T) { - s := factory(t) - ctx := context.Background() - batchID := "batch-dup-contract" - - if err := s.CreateBatch(ctx, BatchRecord{ - BatchID: batchID, - DispatchID: "d1", - Name: "dup", - Queue: "default", - AllowFailed: true, - Jobs: []BatchJob{ - {JobID: "j1", Job: wireJob{Type: "monitor:poll"}}, - }, - CreatedAt: time.Now(), - }); err != nil { - t.Fatalf("create batch: %v", err) - } - - st, done, err := s.MarkBatchJobSucceeded(ctx, batchID, "j1") - if err != nil { - t.Fatalf("mark success first: %v", err) - } - if !done || !st.Completed { - t.Fatalf("expected completed after first success, done=%v state=%+v", done, st) - } - - st, done, err = s.MarkBatchJobSucceeded(ctx, batchID, "j1") - if err != nil { - t.Fatalf("mark success duplicate: %v", err) - } - if !done || !st.Completed { - t.Fatalf("expected completed after duplicate success, done=%v state=%+v", done, st) - } - if st.Processed != 1 || st.Pending != 0 || st.Failed != 0 { - t.Fatalf("expected counters unchanged after duplicate terminal update, got %+v", st) - } - }) - } -} diff --git a/bus/store_managed_schema_test.go b/bus/store_managed_schema_test.go new file mode 100644 index 0000000..f9b0b07 --- /dev/null +++ b/bus/store_managed_schema_test.go @@ -0,0 +1,37 @@ +package bus + +import ( + "context" + "database/sql" + "errors" + "path/filepath" + "testing" + + _ "modernc.org/sqlite" +) + +// TestNewSQLStoreWithManagedSchemaForwardsWithoutDDL proves the deprecated bus +// constructor retains the root caller-managed schema behavior. +func TestNewSQLStoreWithManagedSchemaForwardsWithoutDDL(t *testing.T) { + ctx := context.Background() + db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "bus-managed-empty.db")) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + + store, err := NewSQLStoreWithManagedSchema(SQLStoreConfig{DB: db, DriverName: "sqlite"}) + if err != nil { + t.Fatalf("new managed-schema store: %v", err) + } + if _, err := store.GetChain(ctx, "missing"); err == nil || errors.Is(err, ErrNotFound) { + t.Fatalf("unprovisioned managed schema error = %v", err) + } + var tableCount int + if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name LIKE 'bus_%'`).Scan(&tableCount); err != nil { + t.Fatalf("count workflow tables: %v", err) + } + if tableCount != 0 { + t.Fatalf("deprecated managed-schema constructor created %d workflow tables", tableCount) + } +} diff --git a/bus/store_memory.go b/bus/store_memory.go deleted file mode 100644 index 56e11e8..0000000 --- a/bus/store_memory.go +++ /dev/null @@ -1,270 +0,0 @@ -package bus - -import ( - "context" - "sync" - "time" -) - -// NewMemoryStore creates an in-memory orchestration store implementation. -// @group Constructors -// -// Example: new memory store -// -// store := bus.NewMemoryStore() -// _ = store -func NewMemoryStore() Store { - return &memoryStore{ - chains: make(map[string]*memoryChain), - batch: make(map[string]*memoryBatch), - callbacks: make(map[string]time.Time), - } -} - -type memoryStore struct { - mu sync.Mutex - chains map[string]*memoryChain - batch map[string]*memoryBatch - callbacks map[string]time.Time -} - -var _ Store = (*memoryStore)(nil) - -type memoryChain struct { - state ChainState - completedNode map[string]bool -} - -type batchJobStatus struct { - started bool - done bool - failed bool -} - -type memoryBatch struct { - state BatchState - jobs map[string]batchJobStatus -} - -func (m *memoryStore) CreateChain(_ context.Context, rec ChainRecord) error { - m.mu.Lock() - defer m.mu.Unlock() - now := time.Now() - m.chains[rec.ChainID] = &memoryChain{ - state: ChainState{ - ChainID: rec.ChainID, - DispatchID: rec.DispatchID, - Queue: rec.Queue, - Nodes: rec.Nodes, - NextIndex: 0, - CreatedAt: rec.CreatedAt, - UpdatedAt: now, - }, - completedNode: make(map[string]bool), - } - return nil -} - -func (m *memoryStore) AdvanceChain(_ context.Context, chainID string, completedNode string) (next *ChainNode, done bool, err error) { - m.mu.Lock() - defer m.mu.Unlock() - ch, ok := m.chains[chainID] - if !ok { - return nil, false, ErrNotFound - } - if ch.state.Completed || ch.state.Failed { - return nil, true, nil - } - if ch.completedNode[completedNode] { - if ch.state.NextIndex >= len(ch.state.Nodes) { - return nil, true, nil - } - n := ch.state.Nodes[ch.state.NextIndex] - return &n, false, nil - } - ch.completedNode[completedNode] = true - ch.state.NextIndex++ - ch.state.UpdatedAt = time.Now() - if ch.state.NextIndex >= len(ch.state.Nodes) { - ch.state.Completed = true - return nil, true, nil - } - n := ch.state.Nodes[ch.state.NextIndex] - return &n, false, nil -} - -func (m *memoryStore) FailChain(_ context.Context, chainID string, cause error) error { - m.mu.Lock() - defer m.mu.Unlock() - ch, ok := m.chains[chainID] - if !ok { - return ErrNotFound - } - if !ch.state.Completed { - ch.state.Failed = true - if cause != nil { - ch.state.Failure = cause.Error() - } - ch.state.UpdatedAt = time.Now() - } - return nil -} - -func (m *memoryStore) GetChain(_ context.Context, chainID string) (ChainState, error) { - m.mu.Lock() - defer m.mu.Unlock() - ch, ok := m.chains[chainID] - if !ok { - return ChainState{}, ErrNotFound - } - return ch.state, nil -} - -func (m *memoryStore) CreateBatch(_ context.Context, rec BatchRecord) error { - m.mu.Lock() - defer m.mu.Unlock() - now := time.Now() - st := BatchState{ - BatchID: rec.BatchID, - DispatchID: rec.DispatchID, - Name: rec.Name, - Queue: rec.Queue, - AllowFailed: rec.AllowFailed, - Total: len(rec.Jobs), - Pending: len(rec.Jobs), - CreatedAt: rec.CreatedAt, - UpdatedAt: now, - } - jm := make(map[string]batchJobStatus, len(rec.Jobs)) - for _, job := range rec.Jobs { - jm[job.JobID] = batchJobStatus{} - } - m.batch[rec.BatchID] = &memoryBatch{ - state: st, - jobs: jm, - } - return nil -} - -func (m *memoryStore) MarkBatchJobStarted(_ context.Context, batchID, jobID string) error { - m.mu.Lock() - defer m.mu.Unlock() - b, ok := m.batch[batchID] - if !ok { - return ErrNotFound - } - js := b.jobs[jobID] - js.started = true - b.jobs[jobID] = js - b.state.UpdatedAt = time.Now() - return nil -} - -func (m *memoryStore) MarkBatchJobSucceeded(_ context.Context, batchID, jobID string) (BatchState, bool, error) { - m.mu.Lock() - defer m.mu.Unlock() - b, ok := m.batch[batchID] - if !ok { - return BatchState{}, false, ErrNotFound - } - js := b.jobs[jobID] - if !js.done { - js.done = true - b.jobs[jobID] = js - b.state.Pending-- - b.state.Processed++ - } - if b.state.Pending <= 0 { - b.state.Completed = true - b.state.UpdatedAt = time.Now() - return b.state, true, nil - } - b.state.UpdatedAt = time.Now() - return b.state, false, nil -} - -func (m *memoryStore) MarkBatchJobFailed(_ context.Context, batchID, jobID string, _ error) (BatchState, bool, error) { - m.mu.Lock() - defer m.mu.Unlock() - b, ok := m.batch[batchID] - if !ok { - return BatchState{}, false, ErrNotFound - } - js := b.jobs[jobID] - if !js.done { - js.done = true - js.failed = true - b.jobs[jobID] = js - b.state.Pending-- - b.state.Processed++ - b.state.Failed++ - } - if !b.state.AllowFailed { - b.state.Cancelled = true - b.state.Completed = true - b.state.UpdatedAt = time.Now() - return b.state, true, nil - } - if b.state.Pending <= 0 { - b.state.Completed = true - b.state.UpdatedAt = time.Now() - return b.state, true, nil - } - b.state.UpdatedAt = time.Now() - return b.state, false, nil -} - -func (m *memoryStore) CancelBatch(_ context.Context, batchID string) error { - m.mu.Lock() - defer m.mu.Unlock() - b, ok := m.batch[batchID] - if !ok { - return ErrNotFound - } - b.state.Cancelled = true - b.state.Completed = true - b.state.UpdatedAt = time.Now() - return nil -} - -func (m *memoryStore) GetBatch(_ context.Context, batchID string) (BatchState, error) { - m.mu.Lock() - defer m.mu.Unlock() - b, ok := m.batch[batchID] - if !ok { - return BatchState{}, ErrNotFound - } - return b.state, nil -} - -func (m *memoryStore) MarkCallbackInvoked(_ context.Context, key string) (bool, error) { - m.mu.Lock() - defer m.mu.Unlock() - if _, exists := m.callbacks[key]; exists { - return false, nil - } - m.callbacks[key] = time.Now() - return true, nil -} - -func (m *memoryStore) Prune(_ context.Context, before time.Time) error { - m.mu.Lock() - defer m.mu.Unlock() - - for chainID, ch := range m.chains { - if (ch.state.Completed || ch.state.Failed) && ch.state.UpdatedAt.Before(before) { - delete(m.chains, chainID) - } - } - for batchID, b := range m.batch { - if b.state.Completed && b.state.UpdatedAt.Before(before) { - delete(m.batch, batchID) - } - } - for key, createdAt := range m.callbacks { - if createdAt.Before(before) { - delete(m.callbacks, key) - } - } - return nil -} diff --git a/bus/store_memory_test.go b/bus/store_memory_test.go deleted file mode 100644 index df7ca66..0000000 --- a/bus/store_memory_test.go +++ /dev/null @@ -1,86 +0,0 @@ -package bus - -import ( - "context" - "errors" - "testing" - "time" -) - -func TestMemoryStorePruneRemovesTerminalRecordsOnly(t *testing.T) { - s := NewMemoryStore() - ctx := context.Background() - old := time.Now().Add(-2 * time.Hour) - cutoff := time.Now().Add(1 * time.Minute) - - if err := s.CreateChain(ctx, ChainRecord{ - ChainID: "chain-old-done", - DispatchID: "d1", - Queue: "default", - Nodes: []ChainNode{{NodeID: "n1", Job: wireJob{Type: "monitor:poll"}}}, - CreatedAt: old, - }); err != nil { - t.Fatalf("create chain old done: %v", err) - } - if _, _, err := s.AdvanceChain(ctx, "chain-old-done", "n1"); err != nil { - t.Fatalf("advance old done chain: %v", err) - } - - if err := s.CreateChain(ctx, ChainRecord{ - ChainID: "chain-old-failed", - DispatchID: "d2", - Queue: "default", - Nodes: []ChainNode{{NodeID: "n1", Job: wireJob{Type: "monitor:downsample"}}}, - CreatedAt: old, - }); err != nil { - t.Fatalf("create chain old failed: %v", err) - } - if err := s.FailChain(ctx, "chain-old-failed", errors.New("boom")); err != nil { - t.Fatalf("fail old chain: %v", err) - } - - if err := s.CreateBatch(ctx, BatchRecord{ - BatchID: "batch-old-done", - DispatchID: "d3", - Name: "old", - Queue: "default", - AllowFailed: true, - Jobs: []BatchJob{{JobID: "j1", Job: wireJob{Type: "monitor:poll"}}}, - CreatedAt: old, - }); err != nil { - t.Fatalf("create batch old done: %v", err) - } - if _, _, err := s.MarkBatchJobSucceeded(ctx, "batch-old-done", "j1"); err != nil { - t.Fatalf("mark batch old done: %v", err) - } - - if err := s.CreateChain(ctx, ChainRecord{ - ChainID: "chain-active", - DispatchID: "d4", - Queue: "default", - Nodes: []ChainNode{ - {NodeID: "n1", Job: wireJob{Type: "monitor:poll"}}, - {NodeID: "n2", Job: wireJob{Type: "monitor:alert"}}, - }, - CreatedAt: time.Now(), - }); err != nil { - t.Fatalf("create active chain: %v", err) - } - - if err := s.Prune(ctx, cutoff); err != nil { - t.Fatalf("prune: %v", err) - } - - if _, err := s.GetChain(ctx, "chain-old-done"); !errors.Is(err, ErrNotFound) { - t.Fatalf("expected old completed chain pruned, got err=%v", err) - } - if _, err := s.GetChain(ctx, "chain-old-failed"); !errors.Is(err, ErrNotFound) { - t.Fatalf("expected old failed chain pruned, got err=%v", err) - } - if _, err := s.GetBatch(ctx, "batch-old-done"); !errors.Is(err, ErrNotFound) { - t.Fatalf("expected old completed batch pruned, got err=%v", err) - } - if _, err := s.GetChain(ctx, "chain-active"); err != nil { - t.Fatalf("expected active chain retained, got err=%v", err) - } -} diff --git a/bus/store_sql.go b/bus/store_sql.go deleted file mode 100644 index e4a6f58..0000000 --- a/bus/store_sql.go +++ /dev/null @@ -1,594 +0,0 @@ -package bus - -import ( - "context" - "database/sql" - "encoding/json" - "fmt" - "strings" - "sync" - "time" -) - -type SQLStoreConfig struct { - DB *sql.DB - DriverName string - DSN string - AutoMigrate bool -} - -// NewSQLStore creates a SQL-backed orchestration store. -// @group Constructors -// -// Example: new sql store -// -// store, _ := bus.NewSQLStore(bus.SQLStoreConfig{ -// DriverName: "sqlite", -// DSN: "file:bus.db?_busy_timeout=5000", -// }) -// _ = store -func NewSQLStore(cfg SQLStoreConfig) (Store, error) { - if cfg.DB == nil { - if cfg.DriverName == "" { - return nil, fmt.Errorf("sql store driver name is required") - } - if cfg.DSN == "" { - return nil, fmt.Errorf("sql store dsn is required") - } - db, err := sql.Open(cfg.DriverName, cfg.DSN) - if err != nil { - return nil, err - } - cfg.DB = db - } - if cfg.DriverName == "" { - cfg.DriverName = "sqlite" - } - if !cfg.AutoMigrate { - cfg.AutoMigrate = true - } - return &sqlStore{ - db: cfg.DB, - driverName: cfg.DriverName, - autoMigrate: cfg.AutoMigrate, - }, nil -} - -type sqlStore struct { - db *sql.DB - driverName string - autoMigrate bool - - ensureOnce sync.Once - ensureErr error -} - -var _ Store = (*sqlStore)(nil) - -func (s *sqlStore) ensureSchema(ctx context.Context) error { - s.ensureOnce.Do(func() { - if !s.autoMigrate { - return - } - stmts := []string{ - `CREATE TABLE IF NOT EXISTS bus_chains ( - chain_id TEXT PRIMARY KEY, - dispatch_id TEXT NOT NULL, - queue_name TEXT NOT NULL, - nodes_json BLOB NOT NULL, - next_index INTEGER NOT NULL, - completed INTEGER NOT NULL, - failed INTEGER NOT NULL, - failure TEXT NOT NULL, - created_at_ms BIGINT NOT NULL, - updated_at_ms BIGINT NOT NULL - )`, - `CREATE TABLE IF NOT EXISTS bus_chain_completed_nodes ( - chain_id TEXT NOT NULL, - node_id TEXT NOT NULL, - created_at_ms BIGINT NOT NULL, - PRIMARY KEY (chain_id, node_id) - )`, - `CREATE TABLE IF NOT EXISTS bus_batches ( - batch_id TEXT PRIMARY KEY, - dispatch_id TEXT NOT NULL, - name TEXT NOT NULL, - queue_name TEXT NOT NULL, - allow_failed INTEGER NOT NULL, - total_jobs INTEGER NOT NULL, - pending_jobs INTEGER NOT NULL, - processed_jobs INTEGER NOT NULL, - failed_jobs INTEGER NOT NULL, - cancelled INTEGER NOT NULL, - completed INTEGER NOT NULL, - created_at_ms BIGINT NOT NULL, - updated_at_ms BIGINT NOT NULL - )`, - `CREATE TABLE IF NOT EXISTS bus_batch_jobs ( - batch_id TEXT NOT NULL, - job_id TEXT NOT NULL, - started INTEGER NOT NULL, - done INTEGER NOT NULL, - failed INTEGER NOT NULL, - PRIMARY KEY (batch_id, job_id) - )`, - `CREATE TABLE IF NOT EXISTS bus_callback_invocations ( - callback_key TEXT PRIMARY KEY, - created_at_ms BIGINT NOT NULL - )`, - } - for _, stmt := range stmts { - if _, err := s.db.ExecContext(ctx, s.rebind(stmt)); err != nil { - s.ensureErr = err - return - } - } - }) - return s.ensureErr -} - -func (s *sqlStore) CreateChain(ctx context.Context, rec ChainRecord) error { - if err := s.ensureSchema(ctx); err != nil { - return err - } - nodesJSON, err := json.Marshal(rec.Nodes) - if err != nil { - return err - } - if rec.CreatedAt.IsZero() { - rec.CreatedAt = time.Now() - } - now := rec.CreatedAt.UnixMilli() - _, err = s.db.ExecContext(ctx, s.rebind(`INSERT INTO bus_chains - (chain_id, dispatch_id, queue_name, nodes_json, next_index, completed, failed, failure, created_at_ms, updated_at_ms) - VALUES (?, ?, ?, ?, 0, 0, 0, '', ?, ?)`), - rec.ChainID, rec.DispatchID, rec.Queue, nodesJSON, now, now, - ) - return err -} - -func (s *sqlStore) AdvanceChain(ctx context.Context, chainID string, completedNode string) (next *ChainNode, done bool, err error) { - if err := s.ensureSchema(ctx); err != nil { - return nil, false, err - } - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return nil, false, err - } - defer func() { _ = tx.Rollback() }() - - st, err := s.getChainTx(ctx, tx, chainID) - if err != nil { - return nil, false, err - } - if st.Completed || st.Failed { - if err := tx.Commit(); err != nil { - return nil, false, err - } - return nil, true, nil - } - inserted, err := s.insertChainCompletedNode(ctx, tx, chainID, completedNode) - if err != nil { - return nil, false, err - } - if inserted { - st.NextIndex++ - if st.NextIndex >= len(st.Nodes) { - st.Completed = true - } - if err := s.updateChainStateTx(ctx, tx, st); err != nil { - return nil, false, err - } - } - if err := tx.Commit(); err != nil { - return nil, false, err - } - if st.Completed || st.NextIndex >= len(st.Nodes) { - return nil, true, nil - } - n := st.Nodes[st.NextIndex] - return &n, false, nil -} - -func (s *sqlStore) FailChain(ctx context.Context, chainID string, cause error) error { - if err := s.ensureSchema(ctx); err != nil { - return err - } - msg := "" - if cause != nil { - msg = cause.Error() - } - _, err := s.db.ExecContext(ctx, s.rebind(`UPDATE bus_chains SET failed=1, failure=?, updated_at_ms=? WHERE chain_id=?`), msg, time.Now().UnixMilli(), chainID) - return err -} - -func (s *sqlStore) GetChain(ctx context.Context, chainID string) (ChainState, error) { - if err := s.ensureSchema(ctx); err != nil { - return ChainState{}, err - } - row := s.db.QueryRowContext(ctx, s.rebind(`SELECT dispatch_id, queue_name, nodes_json, next_index, completed, failed, failure, created_at_ms, updated_at_ms - FROM bus_chains WHERE chain_id=?`), chainID) - var ( - dispatchID, queueName, failure string - nodesJSON []byte - nextIndex, completed, failed int - createdMS, updatedMS int64 - ) - if err := row.Scan(&dispatchID, &queueName, &nodesJSON, &nextIndex, &completed, &failed, &failure, &createdMS, &updatedMS); err != nil { - if err == sql.ErrNoRows { - return ChainState{}, ErrNotFound - } - return ChainState{}, err - } - var nodes []ChainNode - if err := json.Unmarshal(nodesJSON, &nodes); err != nil { - return ChainState{}, err - } - return ChainState{ - ChainID: chainID, - DispatchID: dispatchID, - Queue: queueName, - Nodes: nodes, - NextIndex: nextIndex, - Completed: completed == 1, - Failed: failed == 1, - Failure: failure, - CreatedAt: time.UnixMilli(createdMS), - UpdatedAt: time.UnixMilli(updatedMS), - }, nil -} - -func (s *sqlStore) CreateBatch(ctx context.Context, rec BatchRecord) error { - if err := s.ensureSchema(ctx); err != nil { - return err - } - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return err - } - defer func() { _ = tx.Rollback() }() - - if rec.CreatedAt.IsZero() { - rec.CreatedAt = time.Now() - } - now := rec.CreatedAt.UnixMilli() - allow := 0 - if rec.AllowFailed { - allow = 1 - } - _, err = tx.ExecContext(ctx, s.rebind(`INSERT INTO bus_batches - (batch_id, dispatch_id, name, queue_name, allow_failed, total_jobs, pending_jobs, processed_jobs, failed_jobs, cancelled, completed, created_at_ms, updated_at_ms) - VALUES (?, ?, ?, ?, ?, ?, ?, 0, 0, 0, 0, ?, ?)`), - rec.BatchID, rec.DispatchID, rec.Name, rec.Queue, allow, len(rec.Jobs), len(rec.Jobs), now, now, - ) - if err != nil { - return err - } - for _, job := range rec.Jobs { - if _, err := tx.ExecContext(ctx, s.rebind(`INSERT INTO bus_batch_jobs (batch_id, job_id, started, done, failed) VALUES (?, ?, 0, 0, 0)`), rec.BatchID, job.JobID); err != nil { - return err - } - } - return tx.Commit() -} - -func (s *sqlStore) MarkBatchJobStarted(ctx context.Context, batchID, jobID string) error { - if err := s.ensureSchema(ctx); err != nil { - return err - } - _, err := s.db.ExecContext(ctx, s.rebind(`UPDATE bus_batch_jobs SET started=1 WHERE batch_id=? AND job_id=?`), batchID, jobID) - return err -} - -func (s *sqlStore) MarkBatchJobSucceeded(ctx context.Context, batchID, jobID string) (BatchState, bool, error) { - return s.markBatchTerminal(ctx, batchID, jobID, false) -} - -func (s *sqlStore) MarkBatchJobFailed(ctx context.Context, batchID, jobID string, _ error) (BatchState, bool, error) { - return s.markBatchTerminal(ctx, batchID, jobID, true) -} - -func (s *sqlStore) CancelBatch(ctx context.Context, batchID string) error { - if err := s.ensureSchema(ctx); err != nil { - return err - } - _, err := s.db.ExecContext(ctx, s.rebind(`UPDATE bus_batches SET cancelled=1, completed=1, updated_at_ms=? WHERE batch_id=?`), time.Now().UnixMilli(), batchID) - return err -} - -func (s *sqlStore) GetBatch(ctx context.Context, batchID string) (BatchState, error) { - if err := s.ensureSchema(ctx); err != nil { - return BatchState{}, err - } - row := s.db.QueryRowContext(ctx, s.rebind(`SELECT dispatch_id, name, queue_name, allow_failed, total_jobs, pending_jobs, processed_jobs, failed_jobs, cancelled, completed, created_at_ms, updated_at_ms - FROM bus_batches WHERE batch_id=?`), batchID) - var ( - dispatchID, name, queueName string - allow, total, pending, processed, failed int - cancelled, completed int - createdMS, updatedMS int64 - ) - if err := row.Scan(&dispatchID, &name, &queueName, &allow, &total, &pending, &processed, &failed, &cancelled, &completed, &createdMS, &updatedMS); err != nil { - if err == sql.ErrNoRows { - return BatchState{}, ErrNotFound - } - return BatchState{}, err - } - return BatchState{ - BatchID: batchID, - DispatchID: dispatchID, - Name: name, - Queue: queueName, - AllowFailed: allow == 1, - Total: total, - Pending: pending, - Processed: processed, - Failed: failed, - Cancelled: cancelled == 1, - Completed: completed == 1, - CreatedAt: time.UnixMilli(createdMS), - UpdatedAt: time.UnixMilli(updatedMS), - }, nil -} - -func (s *sqlStore) MarkCallbackInvoked(ctx context.Context, key string) (bool, error) { - if err := s.ensureSchema(ctx); err != nil { - return false, err - } - now := time.Now().UnixMilli() - switch s.driverName { - case "pgx", "postgres": - res, err := s.db.ExecContext(ctx, `INSERT INTO bus_callback_invocations (callback_key, created_at_ms) VALUES ($1, $2) ON CONFLICT (callback_key) DO NOTHING`, key, now) - if err != nil { - return false, err - } - n, _ := res.RowsAffected() - return n > 0, nil - case "mysql": - res, err := s.db.ExecContext(ctx, `INSERT IGNORE INTO bus_callback_invocations (callback_key, created_at_ms) VALUES (?, ?)`, key, now) - if err != nil { - return false, err - } - n, _ := res.RowsAffected() - return n > 0, nil - default: - res, err := s.db.ExecContext(ctx, `INSERT INTO bus_callback_invocations (callback_key, created_at_ms) VALUES (?, ?) ON CONFLICT(callback_key) DO NOTHING`, key, now) - if err != nil { - return false, err - } - n, _ := res.RowsAffected() - return n > 0, nil - } -} - -func (s *sqlStore) Prune(ctx context.Context, before time.Time) error { - if err := s.ensureSchema(ctx); err != nil { - return err - } - cutoff := before.UnixMilli() - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return err - } - defer func() { _ = tx.Rollback() }() - - // Remove chain node-idempotency rows for terminal chains before pruning chains. - if _, err := tx.ExecContext(ctx, s.rebind(`DELETE FROM bus_chain_completed_nodes WHERE chain_id IN ( - SELECT chain_id FROM bus_chains WHERE updated_at_ms < ? AND (completed=1 OR failed=1) - )`), cutoff); err != nil { - return err - } - if _, err := tx.ExecContext(ctx, s.rebind(`DELETE FROM bus_chains WHERE updated_at_ms < ? AND (completed=1 OR failed=1)`), cutoff); err != nil { - return err - } - - // Remove per-job state for terminal batches before pruning batches. - if _, err := tx.ExecContext(ctx, s.rebind(`DELETE FROM bus_batch_jobs WHERE batch_id IN ( - SELECT batch_id FROM bus_batches WHERE updated_at_ms < ? AND completed=1 - )`), cutoff); err != nil { - return err - } - if _, err := tx.ExecContext(ctx, s.rebind(`DELETE FROM bus_batches WHERE updated_at_ms < ? AND completed=1`), cutoff); err != nil { - return err - } - - // Callback markers are safe to prune independently. - if _, err := tx.ExecContext(ctx, s.rebind(`DELETE FROM bus_callback_invocations WHERE created_at_ms < ?`), cutoff); err != nil { - return err - } - - return tx.Commit() -} - -func (s *sqlStore) markBatchTerminal(ctx context.Context, batchID, jobID string, isFailure bool) (BatchState, bool, error) { - if err := s.ensureSchema(ctx); err != nil { - return BatchState{}, false, err - } - tx, err := s.db.BeginTx(ctx, nil) - if err != nil { - return BatchState{}, false, err - } - defer func() { _ = tx.Rollback() }() - - var done int - row := tx.QueryRowContext(ctx, s.rebind(`SELECT done FROM bus_batch_jobs WHERE batch_id=? AND job_id=?`), batchID, jobID) - if err := row.Scan(&done); err != nil { - if err == sql.ErrNoRows { - return BatchState{}, false, ErrNotFound - } - return BatchState{}, false, err - } - if done == 0 { - failed := 0 - if isFailure { - failed = 1 - } - if _, err := tx.ExecContext(ctx, s.rebind(`UPDATE bus_batch_jobs SET done=1, failed=?, started=1 WHERE batch_id=? AND job_id=?`), failed, batchID, jobID); err != nil { - return BatchState{}, false, err - } - } - - st, err := s.getBatchTx(ctx, tx, batchID) - if err != nil { - return BatchState{}, false, err - } - if done == 0 { - st.Pending-- - st.Processed++ - if isFailure { - st.Failed++ - } - } - if isFailure && !st.AllowFailed { - st.Cancelled = true - st.Completed = true - } - if st.Pending <= 0 { - st.Completed = true - } - if err := s.updateBatchStateTx(ctx, tx, st); err != nil { - return BatchState{}, false, err - } - if err := tx.Commit(); err != nil { - return BatchState{}, false, err - } - return st, st.Completed, nil -} - -func (s *sqlStore) getChainTx(ctx context.Context, tx *sql.Tx, chainID string) (ChainState, error) { - row := tx.QueryRowContext(ctx, s.rebind(`SELECT dispatch_id, queue_name, nodes_json, next_index, completed, failed, failure, created_at_ms, updated_at_ms FROM bus_chains WHERE chain_id=?`), chainID) - var ( - dispatchID, queueName, failure string - nodesJSON []byte - nextIndex, completed, failed int - createdMS, updatedMS int64 - ) - if err := row.Scan(&dispatchID, &queueName, &nodesJSON, &nextIndex, &completed, &failed, &failure, &createdMS, &updatedMS); err != nil { - if err == sql.ErrNoRows { - return ChainState{}, ErrNotFound - } - return ChainState{}, err - } - var nodes []ChainNode - if err := json.Unmarshal(nodesJSON, &nodes); err != nil { - return ChainState{}, err - } - return ChainState{ - ChainID: chainID, - DispatchID: dispatchID, - Queue: queueName, - Nodes: nodes, - NextIndex: nextIndex, - Completed: completed == 1, - Failed: failed == 1, - Failure: failure, - CreatedAt: time.UnixMilli(createdMS), - UpdatedAt: time.UnixMilli(updatedMS), - }, nil -} - -func (s *sqlStore) updateChainStateTx(ctx context.Context, tx *sql.Tx, st ChainState) error { - completed := 0 - if st.Completed { - completed = 1 - } - failed := 0 - if st.Failed { - failed = 1 - } - _, err := tx.ExecContext(ctx, s.rebind(`UPDATE bus_chains SET next_index=?, completed=?, failed=?, failure=?, updated_at_ms=? WHERE chain_id=?`), - st.NextIndex, completed, failed, st.Failure, time.Now().UnixMilli(), st.ChainID, - ) - return err -} - -func (s *sqlStore) getBatchTx(ctx context.Context, tx *sql.Tx, batchID string) (BatchState, error) { - row := tx.QueryRowContext(ctx, s.rebind(`SELECT dispatch_id, name, queue_name, allow_failed, total_jobs, pending_jobs, processed_jobs, failed_jobs, cancelled, completed, created_at_ms, updated_at_ms FROM bus_batches WHERE batch_id=?`), batchID) - var ( - dispatchID, name, queueName string - allow, total, pending, processed, failed int - cancelled, completed int - createdMS, updatedMS int64 - ) - if err := row.Scan(&dispatchID, &name, &queueName, &allow, &total, &pending, &processed, &failed, &cancelled, &completed, &createdMS, &updatedMS); err != nil { - if err == sql.ErrNoRows { - return BatchState{}, ErrNotFound - } - return BatchState{}, err - } - return BatchState{ - BatchID: batchID, - DispatchID: dispatchID, - Name: name, - Queue: queueName, - AllowFailed: allow == 1, - Total: total, - Pending: pending, - Processed: processed, - Failed: failed, - Cancelled: cancelled == 1, - Completed: completed == 1, - CreatedAt: time.UnixMilli(createdMS), - UpdatedAt: time.UnixMilli(updatedMS), - }, nil -} - -func (s *sqlStore) updateBatchStateTx(ctx context.Context, tx *sql.Tx, st BatchState) error { - cancelled := 0 - if st.Cancelled { - cancelled = 1 - } - completed := 0 - if st.Completed { - completed = 1 - } - _, err := tx.ExecContext(ctx, s.rebind(`UPDATE bus_batches SET pending_jobs=?, processed_jobs=?, failed_jobs=?, cancelled=?, completed=?, updated_at_ms=? WHERE batch_id=?`), - st.Pending, st.Processed, st.Failed, cancelled, completed, time.Now().UnixMilli(), st.BatchID, - ) - return err -} - -func (s *sqlStore) insertChainCompletedNode(ctx context.Context, tx *sql.Tx, chainID, nodeID string) (bool, error) { - now := time.Now().UnixMilli() - switch s.driverName { - case "pgx", "postgres": - res, err := tx.ExecContext(ctx, `INSERT INTO bus_chain_completed_nodes (chain_id, node_id, created_at_ms) VALUES ($1, $2, $3) ON CONFLICT (chain_id, node_id) DO NOTHING`, chainID, nodeID, now) - if err != nil { - return false, err - } - n, _ := res.RowsAffected() - return n > 0, nil - case "mysql": - res, err := tx.ExecContext(ctx, `INSERT IGNORE INTO bus_chain_completed_nodes (chain_id, node_id, created_at_ms) VALUES (?, ?, ?)`, chainID, nodeID, now) - if err != nil { - return false, err - } - n, _ := res.RowsAffected() - return n > 0, nil - default: - res, err := tx.ExecContext(ctx, `INSERT INTO bus_chain_completed_nodes (chain_id, node_id, created_at_ms) VALUES (?, ?, ?) ON CONFLICT(chain_id, node_id) DO NOTHING`, chainID, nodeID, now) - if err != nil { - return false, err - } - n, _ := res.RowsAffected() - return n > 0, nil - } -} - -func (s *sqlStore) rebind(query string) string { - if s.driverName != "pgx" && s.driverName != "postgres" { - return query - } - var b strings.Builder - b.Grow(len(query) + 8) - i := 1 - for _, r := range query { - if r == '?' { - b.WriteString(fmt.Sprintf("$%d", i)) - i++ - continue - } - b.WriteRune(r) - } - return b.String() -} diff --git a/bus/store_sql_compat_test.go b/bus/store_sql_compat_test.go new file mode 100644 index 0000000..f9165a0 --- /dev/null +++ b/bus/store_sql_compat_test.go @@ -0,0 +1,401 @@ +package bus + +import ( + "context" + "database/sql" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" + + _ "modernc.org/sqlite" +) + +const legacyV1NodesJSON = `[{"NodeID":"legacy-node-1","Job":{"type":"reports:build","payload":"eyJpZCI6MX0=","options":{"Queue":"critical","Delay":2000000000,"Timeout":15000000000,"Retry":4,"Backoff":500000000,"UniqueFor":30000000000}}},{"NodeID":"legacy-node-2","Job":{"type":"reports:notify","payload":"bnVsbA==","options":{"Queue":"critical","Delay":0,"Timeout":0,"Retry":0,"Backoff":0,"UniqueFor":0}}}]` + +// TestSQLStoreV1PersistedDataCompatibility proves the current store can read and safely mutate the frozen v1 SQLite layout. +func TestSQLStoreV1PersistedDataCompatibility(t *testing.T) { + const ( + legacyCreatedMS = int64(1704067200123) + legacyUpdatedMS = int64(1704067201123) + pruneCutoffMS = int64(1705000000000) + ) + + ctx := context.Background() + db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "workflow-v1.db")) + if err != nil { + t.Fatalf("open compatibility database: %v", err) + } + t.Cleanup(func() { + if closeErr := db.Close(); closeErr != nil { + t.Errorf("close compatibility database: %v", closeErr) + } + }) + loadLegacySQLStoreFixture(t, ctx, db) + seedLegacyDualTerminalChain(t, ctx, db) + + store, err := NewSQLStore(SQLStoreConfig{DB: db, DriverName: "sqlite"}) + if err != nil { + t.Fatalf("construct store over v1 database: %v", err) + } + + activeChain, err := store.GetChain(ctx, "compat-chain-mutate") + if err != nil { + t.Fatalf("read v1 active chain: %v", err) + } + if activeChain.DispatchID != "compat-dispatch-mutate" || activeChain.Queue != "critical" || activeChain.NextIndex != 0 || activeChain.Completed || activeChain.Failed { + t.Fatalf("active chain state changed: %+v", activeChain) + } + if activeChain.CreatedAt.UnixMilli() != legacyCreatedMS || activeChain.UpdatedAt.UnixMilli() != legacyUpdatedMS { + t.Fatalf("active chain timestamps changed: created=%d updated=%d", activeChain.CreatedAt.UnixMilli(), activeChain.UpdatedAt.UnixMilli()) + } + if len(activeChain.Nodes) != 2 { + t.Fatalf("active chain node count=%d, want 2", len(activeChain.Nodes)) + } + firstJob := activeChain.Nodes[0].Job + if activeChain.Nodes[0].NodeID != "legacy-node-1" || firstJob.Type != "reports:build" || string(firstJob.Payload) != `{"id":1}` { + t.Fatalf("first legacy node did not decode: %+v", activeChain.Nodes[0]) + } + if firstJob.Options.Queue != "critical" || firstJob.Options.Delay != 2*time.Second || firstJob.Options.Timeout != 15*time.Second || firstJob.Options.Retry != 4 || firstJob.Options.Backoff != 500*time.Millisecond || firstJob.Options.UniqueFor != 30*time.Second { + t.Fatalf("legacy nested job options changed: %+v", firstJob.Options) + } + secondJob := activeChain.Nodes[1].Job + if activeChain.Nodes[1].NodeID != "legacy-node-2" || secondJob.Type != "reports:notify" || string(secondJob.Payload) != "null" { + t.Fatalf("second legacy node did not decode: %+v", activeChain.Nodes[1]) + } + + completedChain, err := store.GetChain(ctx, "compat-chain-completed-old") + if err != nil { + t.Fatalf("read v1 completed chain: %v", err) + } + if !completedChain.Completed || completedChain.Failed || completedChain.Failure != "" || completedChain.NextIndex != 2 || completedChain.UpdatedAt.UnixMilli() != 1704067205000 { + t.Fatalf("completed chain state changed: %+v", completedChain) + } + dualTerminalChain, err := store.GetChain(ctx, "compat-chain-dual-terminal-old") + if err != nil { + t.Fatalf("read injected dual-terminal chain: %v", err) + } + if !dualTerminalChain.Completed || !dualTerminalChain.Failed || dualTerminalChain.Failure != "late legacy failure" || dualTerminalChain.NextIndex != 2 || dualTerminalChain.UpdatedAt.UnixMilli() != 1704067205000 { + t.Fatalf("dual-terminal chain state changed: %+v", dualTerminalChain) + } + failedChain, err := store.GetChain(ctx, "compat-chain-failed-old") + if err != nil { + t.Fatalf("read v1 failed chain: %v", err) + } + if failedChain.Completed || !failedChain.Failed || failedChain.Failure != "legacy failure" || failedChain.NextIndex != 1 { + t.Fatalf("failed chain state changed: %+v", failedChain) + } + + activeBatch, err := store.GetBatch(ctx, "compat-batch-mutate") + if err != nil { + t.Fatalf("read v1 active batch: %v", err) + } + if activeBatch.Name != "legacy mutable batch" || activeBatch.Queue != "bulk" || !activeBatch.AllowFailed || activeBatch.Total != 2 || activeBatch.Pending != 2 || activeBatch.Processed != 0 || activeBatch.Failed != 0 || activeBatch.Cancelled || activeBatch.Completed { + t.Fatalf("active batch state changed: %+v", activeBatch) + } + if activeBatch.CreatedAt.UnixMilli() != legacyCreatedMS || activeBatch.UpdatedAt.UnixMilli() != legacyUpdatedMS { + t.Fatalf("active batch timestamps changed: created=%d updated=%d", activeBatch.CreatedAt.UnixMilli(), activeBatch.UpdatedAt.UnixMilli()) + } + terminalBatch, err := store.GetBatch(ctx, "compat-batch-terminal-old") + if err != nil { + t.Fatalf("read v1 terminal batch: %v", err) + } + if !terminalBatch.Completed || terminalBatch.Cancelled || terminalBatch.Total != 2 || terminalBatch.Pending != 0 || terminalBatch.Processed != 2 || terminalBatch.Failed != 1 { + t.Fatalf("terminal batch state changed: %+v", terminalBatch) + } + + assertLegacySQLStoreSchema(t, ctx, db) + assertLegacyNodesJSON(t, ctx, db, "compat-chain-mutate") + + existingCallback, err := store.MarkCallbackInvoked(ctx, "chain_finally:compat-chain-completed-old") + if err != nil { + t.Fatalf("claim existing v1 callback marker: %v", err) + } + if existingCallback { + t.Fatal("existing v1 callback marker was claimed twice") + } + newCallback, err := store.MarkCallbackInvoked(ctx, "batch_finally:compat-batch-mutate") + if err != nil { + t.Fatalf("claim new callback marker: %v", err) + } + if !newCallback { + t.Fatal("new callback marker was not claimed") + } + duplicateCallback, err := store.MarkCallbackInvoked(ctx, "batch_finally:compat-batch-mutate") + if err != nil { + t.Fatalf("claim duplicate callback marker: %v", err) + } + if duplicateCallback { + t.Fatal("new callback marker was claimed twice") + } + + next, done, err := store.AdvanceChain(ctx, "compat-chain-mutate", "legacy-node-1") + if err != nil { + t.Fatalf("advance v1 chain: %v", err) + } + if done || next == nil || next.NodeID != "legacy-node-2" || next.Job.Type != "reports:notify" || string(next.Job.Payload) != "null" { + t.Fatalf("first v1 chain advance returned done=%v next=%+v", done, next) + } + duplicateNext, duplicateDone, err := store.AdvanceChain(ctx, "compat-chain-mutate", "legacy-node-1") + if err != nil { + t.Fatalf("repeat v1 chain advance: %v", err) + } + if duplicateDone || duplicateNext == nil || duplicateNext.NodeID != "legacy-node-2" { + t.Fatalf("duplicate v1 chain advance returned done=%v next=%+v", duplicateDone, duplicateNext) + } + next, done, err = store.AdvanceChain(ctx, "compat-chain-mutate", "legacy-node-2") + if err != nil { + t.Fatalf("complete v1 chain: %v", err) + } + if !done || next != nil { + t.Fatalf("completed v1 chain returned done=%v next=%+v", done, next) + } + mutatedChain, err := store.GetChain(ctx, "compat-chain-mutate") + if err != nil { + t.Fatalf("read mutated v1 chain: %v", err) + } + if !mutatedChain.Completed || mutatedChain.Failed || mutatedChain.NextIndex != 2 || mutatedChain.CreatedAt.UnixMilli() != legacyCreatedMS || mutatedChain.UpdatedAt.UnixMilli() <= legacyUpdatedMS { + t.Fatalf("mutated v1 chain state is inconsistent: %+v", mutatedChain) + } + assertLegacyNodesJSON(t, ctx, db, "compat-chain-mutate") + assertLegacySQLCount(t, ctx, db, `SELECT COUNT(*) FROM bus_chain_completed_nodes WHERE chain_id=?`, 2, "compat-chain-mutate") + + if err := store.MarkBatchJobStarted(ctx, "compat-batch-mutate", "legacy-batch-job-1"); err != nil { + t.Fatalf("start v1 batch job: %v", err) + } + batchAfterSuccess, batchDone, err := store.MarkBatchJobSucceeded(ctx, "compat-batch-mutate", "legacy-batch-job-1") + if err != nil { + t.Fatalf("complete v1 batch job: %v", err) + } + if batchDone || batchAfterSuccess.Pending != 1 || batchAfterSuccess.Processed != 1 || batchAfterSuccess.Failed != 0 { + t.Fatalf("v1 batch state after success is inconsistent: done=%v state=%+v", batchDone, batchAfterSuccess) + } + duplicateBatch, duplicateBatchDone, err := store.MarkBatchJobSucceeded(ctx, "compat-batch-mutate", "legacy-batch-job-1") + if err != nil { + t.Fatalf("repeat v1 batch completion: %v", err) + } + if duplicateBatchDone || duplicateBatch.Pending != 1 || duplicateBatch.Processed != 1 || duplicateBatch.Failed != 0 { + t.Fatalf("duplicate v1 batch completion changed counters: done=%v state=%+v", duplicateBatchDone, duplicateBatch) + } + mutatedBatch, batchDone, err := store.MarkBatchJobFailed(ctx, "compat-batch-mutate", "legacy-batch-job-2", errors.New("legacy compatible failure")) + if err != nil { + t.Fatalf("fail v1 batch job: %v", err) + } + if !batchDone || !mutatedBatch.Completed || mutatedBatch.Cancelled || mutatedBatch.Pending != 0 || mutatedBatch.Processed != 2 || mutatedBatch.Failed != 1 || mutatedBatch.CreatedAt.UnixMilli() != legacyCreatedMS || mutatedBatch.UpdatedAt.UnixMilli() <= legacyUpdatedMS { + t.Fatalf("mutated v1 batch state is inconsistent: done=%v state=%+v", batchDone, mutatedBatch) + } + assertLegacySQLCount(t, ctx, db, `SELECT COUNT(*) FROM bus_batch_jobs WHERE batch_id=? AND started=1 AND done=1`, 2, "compat-batch-mutate") + assertLegacySQLCount(t, ctx, db, `SELECT COUNT(*) FROM bus_batch_jobs WHERE batch_id=? AND failed=1`, 1, "compat-batch-mutate") + + if err := store.Prune(ctx, time.UnixMilli(pruneCutoffMS)); err != nil { + t.Fatalf("prune v1 persisted state: %v", err) + } + for _, chainID := range []string{"compat-chain-completed-old", "compat-chain-dual-terminal-old", "compat-chain-failed-old"} { + if _, findErr := store.GetChain(ctx, chainID); !errors.Is(findErr, ErrNotFound) { + t.Fatalf("old terminal chain %q survived prune: %v", chainID, findErr) + } + } + for _, chainID := range []string{"compat-chain-mutate", "compat-chain-active-old", "compat-chain-completed-recent"} { + if _, findErr := store.GetChain(ctx, chainID); findErr != nil { + t.Fatalf("retained chain %q was lost during prune: %v", chainID, findErr) + } + } + if _, findErr := store.GetBatch(ctx, "compat-batch-terminal-old"); !errors.Is(findErr, ErrNotFound) { + t.Fatalf("old terminal batch survived prune: %v", findErr) + } + for _, batchID := range []string{"compat-batch-mutate", "compat-batch-active-old", "compat-batch-terminal-recent"} { + if _, findErr := store.GetBatch(ctx, batchID); findErr != nil { + t.Fatalf("retained batch %q was lost during prune: %v", batchID, findErr) + } + } + + assertLegacySQLCount(t, ctx, db, `SELECT COUNT(*) FROM bus_chains`, 3) + assertLegacySQLCount(t, ctx, db, `SELECT COUNT(*) FROM bus_chain_completed_nodes`, 5) + assertLegacySQLCount(t, ctx, db, `SELECT COUNT(*) FROM bus_batches`, 3) + assertLegacySQLCount(t, ctx, db, `SELECT COUNT(*) FROM bus_batch_jobs`, 4) + assertLegacySQLCount(t, ctx, db, `SELECT COUNT(*) FROM bus_callback_invocations`, 2) + assertLegacySQLCount(t, ctx, db, `SELECT COUNT(*) FROM bus_callback_invocations WHERE callback_key=?`, 0, "chain_finally:compat-chain-completed-old") + assertLegacySQLCount(t, ctx, db, `SELECT COUNT(*) FROM bus_callback_invocations WHERE callback_key=?`, 1, "chain_finally:compat-chain-completed-recent") + assertLegacySQLCount(t, ctx, db, `SELECT COUNT(*) FROM bus_callback_invocations WHERE callback_key=?`, 1, "batch_finally:compat-batch-mutate") + assertLegacyNodesJSON(t, ctx, db, "compat-chain-active-old") + assertLegacySQLStoreSchema(t, ctx, db) +} + +// loadLegacySQLStoreFixture loads literal v1 SQL in one transaction so setup cannot leave a partially seeded compatibility database. +func loadLegacySQLStoreFixture(t *testing.T, ctx context.Context, db *sql.DB) { + t.Helper() + fixturePath := filepath.Join("testdata", "compat", "workflow", "v1", "sqlite.sql") + fixture, err := os.ReadFile(fixturePath) + if err != nil { + t.Fatalf("read compatibility fixture: %v", err) + } + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin compatibility fixture transaction: %v", err) + } + defer func() { _ = tx.Rollback() }() + for _, rawStatement := range strings.Split(string(fixture), ";") { + statement := strings.TrimSpace(rawStatement) + if statement == "" { + continue + } + if _, err := tx.ExecContext(ctx, statement); err != nil { + t.Fatalf("execute compatibility fixture statement: %v", err) + } + } + if err := tx.Commit(); err != nil { + t.Fatalf("commit compatibility fixture: %v", err) + } +} + +// seedLegacyDualTerminalChain adds a post-fixture anomaly without rewriting +// the frozen v1 compatibility baseline used to detect historical drift. +func seedLegacyDualTerminalChain(t *testing.T, ctx context.Context, db *sql.DB) { + t.Helper() + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin dual-terminal chain setup: %v", err) + } + defer func() { _ = tx.Rollback() }() + result, err := tx.ExecContext(ctx, `INSERT INTO bus_chains + (chain_id, dispatch_id, queue_name, nodes_json, next_index, completed, failed, failure, created_at_ms, updated_at_ms) + SELECT 'compat-chain-dual-terminal-old', 'compat-dispatch-dual-terminal-old', queue_name, nodes_json, next_index, completed, 1, 'late legacy failure', created_at_ms, updated_at_ms + FROM bus_chains WHERE chain_id='compat-chain-completed-old'`) + if err != nil { + t.Fatalf("insert dual-terminal chain: %v", err) + } + inserted, err := result.RowsAffected() + if err != nil { + t.Fatalf("count inserted dual-terminal chains: %v", err) + } + if inserted != 1 { + t.Fatalf("inserted dual-terminal chains=%d, want 1", inserted) + } + result, err = tx.ExecContext(ctx, `INSERT INTO bus_chain_completed_nodes (chain_id, node_id, created_at_ms) + SELECT 'compat-chain-dual-terminal-old', node_id, created_at_ms + FROM bus_chain_completed_nodes WHERE chain_id='compat-chain-completed-old'`) + if err != nil { + t.Fatalf("insert dual-terminal chain nodes: %v", err) + } + inserted, err = result.RowsAffected() + if err != nil { + t.Fatalf("count inserted dual-terminal chain nodes: %v", err) + } + if inserted != 2 { + t.Fatalf("inserted dual-terminal chain nodes=%d, want 2", inserted) + } + if err := tx.Commit(); err != nil { + t.Fatalf("commit dual-terminal chain setup: %v", err) + } +} + +// assertLegacySQLStoreSchema verifies v1 columns remain unchanged while the +// additive transition-receipt table is installed alongside them. +func assertLegacySQLStoreSchema(t *testing.T, ctx context.Context, db *sql.DB) { + t.Helper() + wantTables := []string{ + "bus_batch_jobs", + "bus_batches", + "bus_callback_invocations", + "bus_chain_completed_nodes", + "bus_chains", + "bus_workflow_transition_receipts", + } + wantColumns := map[string][]string{ + "bus_batch_jobs": {"batch_id", "job_id", "started", "done", "failed"}, + "bus_batches": {"batch_id", "dispatch_id", "name", "queue_name", "allow_failed", "total_jobs", "pending_jobs", "processed_jobs", "failed_jobs", "cancelled", "completed", "created_at_ms", "updated_at_ms"}, + "bus_callback_invocations": {"callback_key", "created_at_ms"}, + "bus_chain_completed_nodes": {"chain_id", "node_id", "created_at_ms"}, + "bus_chains": {"chain_id", "dispatch_id", "queue_name", "nodes_json", "next_index", "completed", "failed", "failure", "created_at_ms", "updated_at_ms"}, + "bus_workflow_transition_receipts": {"workflow_kind", "receipt_version", "event_schema_version", "workflow_id", "member_id", "workflow_dispatch_id", "workflow_created_at_ms", "outcome", "owner_delivery_id", "owner_attempt", "job_dispatch_id", "job_id", "job_fingerprint", "aggregate_completed", "aggregate_cancelled", "created_at_ms"}, + } + + rows, err := db.QueryContext(ctx, `SELECT name FROM sqlite_master WHERE type='table' AND name LIKE 'bus_%' ORDER BY name`) + if err != nil { + t.Fatalf("query compatibility tables: %v", err) + } + var gotTables []string + for rows.Next() { + var table string + if err := rows.Scan(&table); err != nil { + _ = rows.Close() + t.Fatalf("scan compatibility table: %v", err) + } + gotTables = append(gotTables, table) + } + if err := rows.Close(); err != nil { + t.Fatalf("close compatibility table rows: %v", err) + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate compatibility tables: %v", err) + } + if !slices.Equal(gotTables, wantTables) { + t.Fatalf("compatibility tables=%v, want %v", gotTables, wantTables) + } + + for _, table := range wantTables { + columns, err := legacySQLColumnNames(ctx, db, table) + if err != nil { + t.Fatalf("query columns for %s: %v", table, err) + } + if !slices.Equal(columns, wantColumns[table]) { + t.Fatalf("compatibility columns for %s=%v, want %v", table, columns, wantColumns[table]) + } + } +} + +// legacySQLColumnNames returns SQLite column names in declaration order for one frozen fixture table. +func legacySQLColumnNames(ctx context.Context, db *sql.DB, table string) ([]string, error) { + rows, err := db.QueryContext(ctx, fmt.Sprintf("PRAGMA table_info(%q)", table)) + if err != nil { + return nil, err + } + defer rows.Close() + var columns []string + for rows.Next() { + var ( + columnID int + name string + columnType string + notNull int + defaultValue any + primaryKey int + ) + if err := rows.Scan(&columnID, &name, &columnType, ¬Null, &defaultValue, &primaryKey); err != nil { + return nil, err + } + columns = append(columns, name) + } + if err := rows.Err(); err != nil { + return nil, err + } + return columns, nil +} + +// assertLegacyNodesJSON verifies state mutations never rewrite the persisted v1 node envelope. +func assertLegacyNodesJSON(t *testing.T, ctx context.Context, db *sql.DB, chainID string) { + t.Helper() + var nodesJSON string + if err := db.QueryRowContext(ctx, `SELECT nodes_json FROM bus_chains WHERE chain_id=?`, chainID).Scan(&nodesJSON); err != nil { + t.Fatalf("read nodes_json for %s: %v", chainID, err) + } + if nodesJSON != legacyV1NodesJSON { + t.Fatalf("nodes_json for %s changed:\n got: %s\nwant: %s", chainID, nodesJSON, legacyV1NodesJSON) + } +} + +// assertLegacySQLCount verifies a compatibility row set has the exact expected cardinality. +func assertLegacySQLCount(t *testing.T, ctx context.Context, db *sql.DB, query string, want int, args ...any) { + t.Helper() + var got int + if err := db.QueryRowContext(ctx, query, args...).Scan(&got); err != nil { + t.Fatalf("query compatibility row count: %v", err) + } + if got != want { + t.Fatalf("compatibility row count=%d, want %d for %q", got, want, query) + } +} diff --git a/bus/store_sql_test.go b/bus/store_sql_test.go deleted file mode 100644 index 0cc3921..0000000 --- a/bus/store_sql_test.go +++ /dev/null @@ -1,331 +0,0 @@ -package bus - -import ( - "context" - "database/sql" - "errors" - "path/filepath" - "strings" - "testing" - "time" - - _ "modernc.org/sqlite" -) - -func newSQLiteStore(t *testing.T) Store { - t.Helper() - dsn := filepath.Join(t.TempDir(), "bus-store.db") - store, err := NewSQLStore(SQLStoreConfig{ - DriverName: "sqlite", - DSN: dsn, - }) - if err != nil { - t.Fatalf("new sql store: %v", err) - } - return store -} - -func TestSQLStoreChainAdvanceIdempotent(t *testing.T) { - s := newSQLiteStore(t) - ctx := context.Background() - chainID := "chain-1" - if err := s.CreateChain(ctx, ChainRecord{ - ChainID: chainID, - DispatchID: "d1", - Queue: "default", - Nodes: []ChainNode{ - {NodeID: "n1", Job: wireJob{Type: "a"}}, - {NodeID: "n2", Job: wireJob{Type: "b"}}, - }, - CreatedAt: time.Now(), - }); err != nil { - t.Fatalf("create chain: %v", err) - } - - next, done, err := s.AdvanceChain(ctx, chainID, "n1") - if err != nil { - t.Fatalf("advance chain first: %v", err) - } - if done { - t.Fatal("expected not done after first advance") - } - if next == nil || next.NodeID != "n2" { - t.Fatalf("expected next n2, got %+v", next) - } - - // duplicate completion should not double-advance - next, done, err = s.AdvanceChain(ctx, chainID, "n1") - if err != nil { - t.Fatalf("advance chain duplicate: %v", err) - } - if done { - t.Fatal("expected not done after duplicate completion") - } - if next == nil || next.NodeID != "n2" { - t.Fatalf("expected next n2 on duplicate, got %+v", next) - } - - next, done, err = s.AdvanceChain(ctx, chainID, "n2") - if err != nil { - t.Fatalf("advance chain final: %v", err) - } - if !done || next != nil { - t.Fatalf("expected done with nil next, got done=%v next=%+v", done, next) - } -} - -func TestSQLStoreBatchLifecycle(t *testing.T) { - s := newSQLiteStore(t) - ctx := context.Background() - batchID := "batch-1" - - if err := s.CreateBatch(ctx, BatchRecord{ - BatchID: batchID, - DispatchID: "d1", - Name: "monitor sweep", - Queue: "default", - AllowFailed: false, - Jobs: []BatchJob{ - {JobID: "j1", Job: wireJob{Type: "monitor:poll"}}, - {JobID: "j2", Job: wireJob{Type: "monitor:downsample"}}, - }, - CreatedAt: time.Now(), - }); err != nil { - t.Fatalf("create batch: %v", err) - } - - if err := s.MarkBatchJobStarted(ctx, batchID, "j1"); err != nil { - t.Fatalf("mark started: %v", err) - } - st, done, err := s.MarkBatchJobSucceeded(ctx, batchID, "j1") - if err != nil { - t.Fatalf("mark succeeded: %v", err) - } - if done { - t.Fatal("expected not done after first success") - } - if st.Processed != 1 || st.Pending != 1 || st.Failed != 0 { - t.Fatalf("unexpected state after success: %+v", st) - } - - st, done, err = s.MarkBatchJobFailed(ctx, batchID, "j2", nil) - if err != nil { - t.Fatalf("mark failed: %v", err) - } - if !done { - t.Fatal("expected done after failure when allow_failed=false") - } - if !st.Completed || !st.Cancelled || st.Failed != 1 { - t.Fatalf("unexpected terminal state: %+v", st) - } -} - -func TestSQLStoreCallbackMarkerIdempotent(t *testing.T) { - s := newSQLiteStore(t) - ctx := context.Background() - key := "chain_finally:chain-1" - - first, err := s.MarkCallbackInvoked(ctx, key) - if err != nil { - t.Fatalf("mark callback first: %v", err) - } - if !first { - t.Fatal("expected first callback marker insert to be true") - } - - second, err := s.MarkCallbackInvoked(ctx, key) - if err != nil { - t.Fatalf("mark callback second: %v", err) - } - if second { - t.Fatal("expected duplicate callback marker insert to be false") - } -} - -func TestSQLStorePruneRemovesOldTerminalState(t *testing.T) { - s := newSQLiteStore(t) - ctx := context.Background() - old := time.Now().Add(-2 * time.Hour) - cutoff := time.Now().Add(1 * time.Minute) - - if err := s.CreateChain(ctx, ChainRecord{ - ChainID: "chain-old-done", - DispatchID: "d1", - Queue: "default", - Nodes: []ChainNode{{NodeID: "n1", Job: wireJob{Type: "monitor:poll"}}}, - CreatedAt: old, - }); err != nil { - t.Fatalf("create chain old done: %v", err) - } - if _, _, err := s.AdvanceChain(ctx, "chain-old-done", "n1"); err != nil { - t.Fatalf("advance old chain: %v", err) - } - - if err := s.CreateBatch(ctx, BatchRecord{ - BatchID: "batch-old-done", - DispatchID: "d2", - Name: "old-batch", - Queue: "default", - AllowFailed: true, - Jobs: []BatchJob{{JobID: "j1", Job: wireJob{Type: "monitor:poll"}}}, - CreatedAt: old, - }); err != nil { - t.Fatalf("create batch old done: %v", err) - } - if _, _, err := s.MarkBatchJobSucceeded(ctx, "batch-old-done", "j1"); err != nil { - t.Fatalf("mark old batch done: %v", err) - } - - if _, err := s.MarkCallbackInvoked(ctx, "batch_then:batch-old-done"); err != nil { - t.Fatalf("mark callback marker: %v", err) - } - - if err := s.CreateChain(ctx, ChainRecord{ - ChainID: "chain-active", - DispatchID: "d3", - Queue: "default", - Nodes: []ChainNode{ - {NodeID: "n1", Job: wireJob{Type: "monitor:poll"}}, - {NodeID: "n2", Job: wireJob{Type: "monitor:alert"}}, - }, - CreatedAt: time.Now(), - }); err != nil { - t.Fatalf("create chain active: %v", err) - } - - if err := s.Prune(ctx, cutoff); err != nil { - t.Fatalf("prune: %v", err) - } - - if _, err := s.GetChain(ctx, "chain-old-done"); !errors.Is(err, ErrNotFound) { - t.Fatalf("expected old chain pruned, got err=%v", err) - } - if _, err := s.GetBatch(ctx, "batch-old-done"); !errors.Is(err, ErrNotFound) { - t.Fatalf("expected old batch pruned, got err=%v", err) - } - if _, err := s.GetChain(ctx, "chain-active"); err != nil { - t.Fatalf("expected active chain retained, got err=%v", err) - } -} - -func TestNewSQLStoreValidationAndDefaults(t *testing.T) { - if _, err := NewSQLStore(SQLStoreConfig{}); err == nil || !strings.Contains(err.Error(), "driver name is required") { - t.Fatalf("expected driver validation error, got %v", err) - } - if _, err := NewSQLStore(SQLStoreConfig{DriverName: "sqlite"}); err == nil || !strings.Contains(err.Error(), "dsn is required") { - t.Fatalf("expected dsn validation error, got %v", err) - } - - db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "defaults.db")) - if err != nil { - t.Fatalf("open sqlite: %v", err) - } - defer db.Close() - - store, err := NewSQLStore(SQLStoreConfig{DB: db}) - if err != nil { - t.Fatalf("new sql store with db: %v", err) - } - ss, ok := store.(*sqlStore) - if !ok { - t.Fatalf("expected *sqlStore, got %T", store) - } - if ss.driverName != "sqlite" { - t.Fatalf("expected default driver sqlite, got %q", ss.driverName) - } - if !ss.autoMigrate { - t.Fatal("expected autoMigrate default true") - } -} - -func TestSQLStoreFailChainAndCancelBatch(t *testing.T) { - s := newSQLiteStore(t) - ctx := context.Background() - - if err := s.CreateChain(ctx, ChainRecord{ - ChainID: "chain-fail", - DispatchID: "d-fail", - Queue: "default", - Nodes: []ChainNode{{NodeID: "n1", Job: wireJob{Type: "monitor:poll"}}}, - CreatedAt: time.Now(), - }); err != nil { - t.Fatalf("create chain: %v", err) - } - if err := s.FailChain(ctx, "chain-fail", errors.New("boom")); err != nil { - t.Fatalf("fail chain: %v", err) - } - st, err := s.GetChain(ctx, "chain-fail") - if err != nil { - t.Fatalf("get chain: %v", err) - } - if !st.Failed || st.Failure != "boom" { - t.Fatalf("expected failed chain with boom, got %+v", st) - } - - if err := s.CreateBatch(ctx, BatchRecord{ - BatchID: "batch-cancel", - DispatchID: "d-cancel", - Name: "cancel-me", - Queue: "default", - AllowFailed: true, - Jobs: []BatchJob{{JobID: "j1", Job: wireJob{Type: "monitor:poll"}}}, - CreatedAt: time.Now(), - }); err != nil { - t.Fatalf("create batch: %v", err) - } - if err := s.CancelBatch(ctx, "batch-cancel"); err != nil { - t.Fatalf("cancel batch: %v", err) - } - bs, err := s.GetBatch(ctx, "batch-cancel") - if err != nil { - t.Fatalf("get batch: %v", err) - } - if !bs.Cancelled || !bs.Completed { - t.Fatalf("expected cancelled completed batch, got %+v", bs) - } -} - -func TestSQLStoreBatchTerminalIdempotentAndNotFound(t *testing.T) { - s := newSQLiteStore(t) - ctx := context.Background() - - if _, _, err := s.MarkBatchJobSucceeded(ctx, "missing-batch", "missing-job"); !errors.Is(err, ErrNotFound) { - t.Fatalf("expected ErrNotFound for missing batch job, got %v", err) - } - - if err := s.CreateBatch(ctx, BatchRecord{ - BatchID: "batch-idem", - DispatchID: "d-idem", - Name: "idem", - Queue: "default", - AllowFailed: true, - Jobs: []BatchJob{{JobID: "j1", Job: wireJob{Type: "monitor:poll"}}}, - CreatedAt: time.Now(), - }); err != nil { - t.Fatalf("create batch: %v", err) - } - - st1, done1, err := s.MarkBatchJobSucceeded(ctx, "batch-idem", "j1") - if err != nil { - t.Fatalf("first mark succeeded: %v", err) - } - if !done1 || st1.Processed != 1 || st1.Pending != 0 { - t.Fatalf("unexpected first terminal state: done=%v state=%+v", done1, st1) - } - - st2, done2, err := s.MarkBatchJobSucceeded(ctx, "batch-idem", "j1") - if err != nil { - t.Fatalf("second mark succeeded: %v", err) - } - if !done2 || st2.Processed != 1 || st2.Pending != 0 { - t.Fatalf("expected idempotent terminal state, got done=%v state=%+v", done2, st2) - } -} - -func TestSQLStoreRebindForPostgres(t *testing.T) { - s := &sqlStore{driverName: "postgres"} - got := s.rebind("SELECT * FROM t WHERE a=? AND b=?") - if got != "SELECT * FROM t WHERE a=$1 AND b=$2" { - t.Fatalf("unexpected rebind result: %q", got) - } -} diff --git a/bus/test_runtime_adapter_test.go b/bus/test_runtime_adapter_test.go deleted file mode 100644 index 91c4a55..0000000 --- a/bus/test_runtime_adapter_test.go +++ /dev/null @@ -1,55 +0,0 @@ -package bus - -import ( - "context" - "encoding/json" - "fmt" - - "github.com/goforj/queue/busruntime" -) - -type testInboundJob struct { - payload []byte -} - -func (j testInboundJob) Bind(dst any) error { - return json.Unmarshal(j.payload, dst) -} - -func (j testInboundJob) PayloadBytes() []byte { - return append([]byte(nil), j.payload...) -} - -type syncTestRuntime struct { - handlers map[string]busruntime.Handler -} - -func newSyncTestRuntime() *syncTestRuntime { - return &syncTestRuntime{handlers: make(map[string]busruntime.Handler)} -} - -func (r *syncTestRuntime) BusRegister(jobType string, handler busruntime.Handler) { - if r.handlers == nil { - r.handlers = make(map[string]busruntime.Handler) - } - r.handlers[jobType] = handler -} - -func (r *syncTestRuntime) BusDispatch(ctx context.Context, jobType string, payload []byte, _ busruntime.JobOptions) error { - h, ok := r.handlers[jobType] - if !ok || h == nil { - return fmt.Errorf("handler not registered for %q", jobType) - } - return h(ctx, testInboundJob{payload: append([]byte(nil), payload...)}) -} - -func (r *syncTestRuntime) StartWorkers(context.Context) error { return nil } -func (r *syncTestRuntime) Shutdown(context.Context) error { return nil } - -func (r *syncTestRuntime) DispatchJSON(ctx context.Context, jobType string, payload any) error { - b, err := json.Marshal(payload) - if err != nil { - return err - } - return r.BusDispatch(ctx, jobType, b, busruntime.JobOptions{}) -} diff --git a/bus/testdata/compat/workflow/v1/sqlite.sql b/bus/testdata/compat/workflow/v1/sqlite.sql new file mode 100644 index 0000000..421f614 --- /dev/null +++ b/bus/testdata/compat/workflow/v1/sqlite.sql @@ -0,0 +1,89 @@ +CREATE TABLE bus_chains ( + chain_id TEXT PRIMARY KEY, + dispatch_id TEXT NOT NULL, + queue_name TEXT NOT NULL, + nodes_json BLOB NOT NULL, + next_index INTEGER NOT NULL, + completed INTEGER NOT NULL, + failed INTEGER NOT NULL, + failure TEXT NOT NULL, + created_at_ms BIGINT NOT NULL, + updated_at_ms BIGINT NOT NULL +); + +CREATE TABLE bus_chain_completed_nodes ( + chain_id TEXT NOT NULL, + node_id TEXT NOT NULL, + created_at_ms BIGINT NOT NULL, + PRIMARY KEY (chain_id, node_id) +); + +CREATE TABLE bus_batches ( + batch_id TEXT PRIMARY KEY, + dispatch_id TEXT NOT NULL, + name TEXT NOT NULL, + queue_name TEXT NOT NULL, + allow_failed INTEGER NOT NULL, + total_jobs INTEGER NOT NULL, + pending_jobs INTEGER NOT NULL, + processed_jobs INTEGER NOT NULL, + failed_jobs INTEGER NOT NULL, + cancelled INTEGER NOT NULL, + completed INTEGER NOT NULL, + created_at_ms BIGINT NOT NULL, + updated_at_ms BIGINT NOT NULL +); + +CREATE TABLE bus_batch_jobs ( + batch_id TEXT NOT NULL, + job_id TEXT NOT NULL, + started INTEGER NOT NULL, + done INTEGER NOT NULL, + failed INTEGER NOT NULL, + PRIMARY KEY (batch_id, job_id) +); + +CREATE TABLE bus_callback_invocations ( + callback_key TEXT PRIMARY KEY, + created_at_ms BIGINT NOT NULL +); + +INSERT INTO bus_chains + (chain_id, dispatch_id, queue_name, nodes_json, next_index, completed, failed, failure, created_at_ms, updated_at_ms) +VALUES + ('compat-chain-mutate', 'compat-dispatch-mutate', 'critical', '[{"NodeID":"legacy-node-1","Job":{"type":"reports:build","payload":"eyJpZCI6MX0=","options":{"Queue":"critical","Delay":2000000000,"Timeout":15000000000,"Retry":4,"Backoff":500000000,"UniqueFor":30000000000}}},{"NodeID":"legacy-node-2","Job":{"type":"reports:notify","payload":"bnVsbA==","options":{"Queue":"critical","Delay":0,"Timeout":0,"Retry":0,"Backoff":0,"UniqueFor":0}}}]', 0, 0, 0, '', 1704067200123, 1704067201123), + ('compat-chain-active-old', 'compat-dispatch-active-old', 'default', '[{"NodeID":"legacy-node-1","Job":{"type":"reports:build","payload":"eyJpZCI6MX0=","options":{"Queue":"critical","Delay":2000000000,"Timeout":15000000000,"Retry":4,"Backoff":500000000,"UniqueFor":30000000000}}},{"NodeID":"legacy-node-2","Job":{"type":"reports:notify","payload":"bnVsbA==","options":{"Queue":"critical","Delay":0,"Timeout":0,"Retry":0,"Backoff":0,"UniqueFor":0}}}]', 1, 0, 0, '', 1704067200123, 1704067205000), + ('compat-chain-completed-old', 'compat-dispatch-completed-old', 'default', '[{"NodeID":"legacy-node-1","Job":{"type":"reports:build","payload":"eyJpZCI6MX0=","options":{"Queue":"critical","Delay":2000000000,"Timeout":15000000000,"Retry":4,"Backoff":500000000,"UniqueFor":30000000000}}},{"NodeID":"legacy-node-2","Job":{"type":"reports:notify","payload":"bnVsbA==","options":{"Queue":"critical","Delay":0,"Timeout":0,"Retry":0,"Backoff":0,"UniqueFor":0}}}]', 2, 1, 0, '', 1704067200123, 1704067205000), + ('compat-chain-failed-old', 'compat-dispatch-failed-old', 'default', '[{"NodeID":"legacy-node-1","Job":{"type":"reports:build","payload":"eyJpZCI6MX0=","options":{"Queue":"critical","Delay":2000000000,"Timeout":15000000000,"Retry":4,"Backoff":500000000,"UniqueFor":30000000000}}},{"NodeID":"legacy-node-2","Job":{"type":"reports:notify","payload":"bnVsbA==","options":{"Queue":"critical","Delay":0,"Timeout":0,"Retry":0,"Backoff":0,"UniqueFor":0}}}]', 1, 0, 1, 'legacy failure', 1704067200123, 1704067205000), + ('compat-chain-completed-recent', 'compat-dispatch-completed-recent', 'critical', '[{"NodeID":"legacy-node-1","Job":{"type":"reports:build","payload":"eyJpZCI6MX0=","options":{"Queue":"critical","Delay":2000000000,"Timeout":15000000000,"Retry":4,"Backoff":500000000,"UniqueFor":30000000000}}},{"NodeID":"legacy-node-2","Job":{"type":"reports:notify","payload":"bnVsbA==","options":{"Queue":"critical","Delay":0,"Timeout":0,"Retry":0,"Backoff":0,"UniqueFor":0}}}]', 2, 1, 0, '', 1706000000123, 1706000001123); + +INSERT INTO bus_chain_completed_nodes (chain_id, node_id, created_at_ms) +VALUES + ('compat-chain-active-old', 'legacy-node-1', 1704067204000), + ('compat-chain-completed-old', 'legacy-node-1', 1704067203000), + ('compat-chain-completed-old', 'legacy-node-2', 1704067204000), + ('compat-chain-failed-old', 'legacy-node-1', 1704067203000), + ('compat-chain-completed-recent', 'legacy-node-1', 1706000000123), + ('compat-chain-completed-recent', 'legacy-node-2', 1706000001123); + +INSERT INTO bus_batches + (batch_id, dispatch_id, name, queue_name, allow_failed, total_jobs, pending_jobs, processed_jobs, failed_jobs, cancelled, completed, created_at_ms, updated_at_ms) +VALUES + ('compat-batch-mutate', 'compat-batch-dispatch-mutate', 'legacy mutable batch', 'bulk', 1, 2, 2, 0, 0, 0, 0, 1704067200123, 1704067201123), + ('compat-batch-active-old', 'compat-batch-dispatch-active-old', 'legacy active batch', 'default', 0, 1, 1, 0, 0, 0, 0, 1704067200123, 1704067205000), + ('compat-batch-terminal-old', 'compat-batch-dispatch-terminal-old', 'legacy terminal batch', 'bulk', 1, 2, 0, 2, 1, 0, 1, 1704067200123, 1704067205000), + ('compat-batch-terminal-recent', 'compat-batch-dispatch-terminal-recent', 'recent terminal batch', 'critical', 0, 1, 0, 1, 0, 0, 1, 1706000000123, 1706000001123); + +INSERT INTO bus_batch_jobs (batch_id, job_id, started, done, failed) +VALUES + ('compat-batch-mutate', 'legacy-batch-job-1', 0, 0, 0), + ('compat-batch-mutate', 'legacy-batch-job-2', 0, 0, 0), + ('compat-batch-active-old', 'legacy-active-job-1', 0, 0, 0), + ('compat-batch-terminal-old', 'legacy-terminal-job-1', 1, 1, 0), + ('compat-batch-terminal-old', 'legacy-terminal-job-2', 1, 1, 1), + ('compat-batch-terminal-recent', 'legacy-recent-job-1', 1, 1, 0); + +INSERT INTO bus_callback_invocations (callback_key, created_at_ms) +VALUES + ('chain_finally:compat-chain-completed-old', 1704067205000), + ('chain_finally:compat-chain-completed-recent', 1706000001123); diff --git a/bus/testhooks_integration.go b/bus/testhooks_integration.go index 024ab5e..e06485c 100644 --- a/bus/testhooks_integration.go +++ b/bus/testhooks_integration.go @@ -8,30 +8,35 @@ import ( "fmt" "github.com/goforj/queue/busruntime" + "github.com/goforj/queue/internal/workflow" ) type integrationTestInboundJob struct { payload []byte } +// Bind decodes one integration delivery into the engine envelope. func (j integrationTestInboundJob) Bind(dst any) error { return json.Unmarshal(j.payload, dst) } +// PayloadBytes returns a copy of the integration delivery payload. func (j integrationTestInboundJob) PayloadBytes() []byte { return append([]byte(nil), j.payload...) } // IntegrationTestRuntime is a minimal in-memory runtime used by integration tests -// that need to dispatch internal bus jobs directly. +// that need to dispatch physical workflow deliveries directly. type IntegrationTestRuntime struct { handlers map[string]busruntime.Handler } +// NewIntegrationTestRuntime creates an in-memory raw runtime for integration fixtures. func NewIntegrationTestRuntime() *IntegrationTestRuntime { return &IntegrationTestRuntime{handlers: make(map[string]busruntime.Handler)} } +// BusRegister records a physical workflow handler by delivery type. func (r *IntegrationTestRuntime) BusRegister(jobType string, handler busruntime.Handler) { if r.handlers == nil { r.handlers = make(map[string]busruntime.Handler) @@ -39,6 +44,7 @@ func (r *IntegrationTestRuntime) BusRegister(jobType string, handler busruntime. r.handlers[jobType] = handler } +// BusDispatch invokes the physical workflow handler synchronously. func (r *IntegrationTestRuntime) BusDispatch(ctx context.Context, jobType string, payload []byte, _ busruntime.JobOptions) error { h, ok := r.handlers[jobType] if !ok || h == nil { @@ -47,9 +53,13 @@ func (r *IntegrationTestRuntime) BusDispatch(ctx context.Context, jobType string return h(ctx, integrationTestInboundJob{payload: append([]byte(nil), payload...)}) } +// StartWorkers is inert because the integration runtime dispatches synchronously. func (r *IntegrationTestRuntime) StartWorkers(context.Context) error { return nil } -func (r *IntegrationTestRuntime) Shutdown(context.Context) error { return nil } +// Shutdown is inert because the integration runtime owns no asynchronous resources. +func (r *IntegrationTestRuntime) Shutdown(context.Context) error { return nil } + +// DispatchJSON encodes a literal integration envelope before physical dispatch. func (r *IntegrationTestRuntime) DispatchJSON(ctx context.Context, jobType string, payload any) error { b, err := json.Marshal(payload) if err != nil { @@ -58,6 +68,7 @@ func (r *IntegrationTestRuntime) DispatchJSON(ctx context.Context, jobType strin return r.BusDispatch(ctx, jobType, b, busruntime.JobOptions{}) } +// InternalCallbackJobTypeForIntegration returns the version-one callback delivery name. func InternalCallbackJobTypeForIntegration() string { - return internalJobCallback + return workflow.CallbackDeliveryType } diff --git a/bus/testhooks_integration_test.go b/bus/testhooks_integration_test.go new file mode 100644 index 0000000..ce8145f --- /dev/null +++ b/bus/testhooks_integration_test.go @@ -0,0 +1,90 @@ +//go:build integration + +package bus + +import ( + "context" + "errors" + "testing" + + "github.com/goforj/queue/busruntime" + "github.com/goforj/queue/internal/workflow" +) + +// integrationHookPayload gives both direct bytes and JSON dispatch a concrete bind target. +type integrationHookPayload struct { + Value int `json:"value"` +} + +// integrationHookMarshalFailure makes the integration hook's encoding failure deterministic. +type integrationHookMarshalFailure struct { + err error +} + +// MarshalJSON returns the configured error so DispatchJSON's encoding boundary remains observable. +func (p integrationHookMarshalFailure) MarshalJSON() ([]byte, error) { + return nil, p.err +} + +// TestIntegrationTestRuntimeExercisesPhysicalDispatchHooks verifies the +// integration-only runtime preserves registration, payload, and error behavior. +func TestIntegrationTestRuntimeExercisesPhysicalDispatchHooks(t *testing.T) { + runtime := NewIntegrationTestRuntime() + if err := runtime.StartWorkers(context.Background()); err != nil { + t.Fatalf("start integration runtime: %v", err) + } + if err := runtime.Shutdown(context.Background()); err != nil { + t.Fatalf("shutdown integration runtime: %v", err) + } + if err := runtime.BusDispatch(context.Background(), "missing", nil, busruntime.JobOptions{}); err == nil { + t.Fatal("missing integration handler was accepted") + } + runtime.BusRegister("nil-handler", nil) + if err := runtime.BusDispatch(context.Background(), "nil-handler", nil, busruntime.JobOptions{}); err == nil { + t.Fatal("nil integration handler was accepted") + } + + var zeroValueRuntime IntegrationTestRuntime + var ( + delivered busruntime.InboundJob + got integrationHookPayload + ) + zeroValueRuntime.BusRegister("integration:payload", func(_ context.Context, job busruntime.InboundJob) error { + delivered = job + first := job.PayloadBytes() + first[0] = '!' + if string(job.PayloadBytes()) != `{"value":7}` { + return errors.New("integration payload bytes were not isolated") + } + if err := job.Bind(&got); err != nil { + return err + } + return nil + }) + rawPayload := []byte(`{"value":7}`) + if err := zeroValueRuntime.BusDispatch(context.Background(), "integration:payload", rawPayload, busruntime.JobOptions{}); err != nil { + t.Fatalf("dispatch integration bytes: %v", err) + } + if string(rawPayload) != `{"value":7}` || got.Value != 7 { + t.Fatalf("raw integration payload/bound value = %q/%+v", rawPayload, got) + } + rawPayload[0] = '!' + if string(delivered.PayloadBytes()) != `{"value":7}` { + t.Fatal("integration delivery retained the caller's mutable payload") + } + got = integrationHookPayload{} + if err := zeroValueRuntime.DispatchJSON(context.Background(), "integration:payload", integrationHookPayload{Value: 7}); err != nil { + t.Fatalf("dispatch integration JSON: %v", err) + } + if got.Value != 7 { + t.Fatalf("bound integration payload = %+v", got) + } + + marshalErr := errors.New("integration payload encoding failed") + if err := zeroValueRuntime.DispatchJSON(context.Background(), "integration:payload", integrationHookMarshalFailure{err: marshalErr}); !errors.Is(err, marshalErr) { + t.Fatalf("DispatchJSON encoding error = %v, want %v", err, marshalErr) + } + if got := InternalCallbackJobTypeForIntegration(); got != workflow.CallbackDeliveryType { + t.Fatalf("callback delivery type = %q, want %q", got, workflow.CallbackDeliveryType) + } +} diff --git a/bus/types.go b/bus/types.go index 42af3d1..318057b 100644 --- a/bus/types.go +++ b/bus/types.go @@ -2,157 +2,99 @@ package bus import ( "context" - "encoding/json" "time" + + "github.com/goforj/queue" ) -type Handler func(ctx context.Context, j Context) error +// Handler processes one legacy workflow message. +// +// Deprecated: register handlers on queue.Queue. +type Handler func(ctx context.Context, message Context) error +// Job is the legacy workflow dispatch DTO. +// +// Deprecated: use queue.Job. This type remains distinct because its public +// fields and deferred JSON encoding are part of the compatibility contract. type Job struct { Type string Payload any Options JobOptions } -// NewJob creates a typed bus job payload with optional fluent options. -// @group Constructors -// -// Example: new bus job +// NewJob creates a typed legacy workflow job with optional fluent options. // -// type PollPayload struct { -// URL string `json:"url"` -// } -// job := bus.NewJob("monitor:poll", PollPayload{ -// URL: "https://goforj.dev/health", -// }). -// OnQueue("monitor-critical"). -// Delay(2 * time.Second). -// Timeout(15 * time.Second). -// Retry(3). -// Backoff(500 * time.Millisecond). -// UniqueFor(30 * time.Second) -// _ = job +// Deprecated: use queue.NewJob and its Payload method. +// @group Constructors func NewJob(jobType string, payload any) Job { return Job{Type: jobType, Payload: payload} } // OnQueue sets the target queue for this job. -// @group Job // -// Example: set queue -// -// job := bus.NewJob("emails:send", nil).OnQueue("critical") -// _ = job +// Deprecated: use queue.Job.OnQueue. +// @group Job func (j Job) OnQueue(name string) Job { j.Options.Queue = name return j } // Delay defers job execution. -// @group Job -// -// Example: set delay // -// job := bus.NewJob("emails:send", nil).Delay(2 * time.Second) -// _ = job +// Deprecated: use queue.Job.Delay. +// @group Job func (j Job) Delay(delay time.Duration) Job { j.Options.Delay = delay return j } -// Timeout sets execution timeout for this job. -// @group Job -// -// Example: set timeout +// Timeout sets the execution timeout for this job. // -// job := bus.NewJob("emails:send", nil).Timeout(15 * time.Second) -// _ = job +// Deprecated: use queue.Job.Timeout. +// @group Job func (j Job) Timeout(timeout time.Duration) Job { j.Options.Timeout = timeout return j } -// Retry sets max retry attempts for this job. -// @group Job +// Retry sets the maximum retry count for this job. // -// Example: set retry count -// -// job := bus.NewJob("emails:send", nil).Retry(5) -// _ = job +// Deprecated: use queue.Job.Retry. +// @group Job func (j Job) Retry(max int) Job { j.Options.Retry = max return j } // Backoff sets retry backoff for this job. -// @group Job -// -// Example: set retry backoff // -// job := bus.NewJob("emails:send", nil).Backoff(500 * time.Millisecond) -// _ = job +// Deprecated: use queue.Job.Backoff. +// @group Job func (j Job) Backoff(backoff time.Duration) Job { j.Options.Backoff = backoff return j } -// UniqueFor sets dedupe TTL for this job. -// @group Job -// -// Example: set unique TTL +// UniqueFor sets the deduplication TTL for this job. // -// job := bus.NewJob("emails:send", nil).UniqueFor(30 * time.Second) -// _ = job +// Deprecated: use queue.Job.UniqueFor. +// @group Job func (j Job) UniqueFor(ttl time.Duration) Job { j.Options.UniqueFor = ttl return j } -type JobOptions struct { - Queue string - Delay time.Duration - Timeout time.Duration - Retry int - Backoff time.Duration - UniqueFor time.Duration -} - -type DispatchResult struct { - DispatchID string -} - -type Context struct { - SchemaVersion int - DispatchID string - JobID string - ChainID string - BatchID string - Attempt int - JobType string - payload []byte -} - -// PayloadBytes returns a copy of raw job payload bytes. -// @group Job -// -// Example: read raw payload bytes +// JobOptions contains the legacy workflow delivery options. // -// raw := jc.PayloadBytes() -// _ = raw -func (c Context) PayloadBytes() []byte { - return append([]byte(nil), c.payload...) -} +// Deprecated: configure a queue.Job through its fluent methods. +type JobOptions = queue.StoredJobOptions -// Bind unmarshals the job payload into dst. -// @group Job +// DispatchResult describes an accepted workflow dispatch. // -// Example: bind payload +// Deprecated: use queue.DispatchResult. +type DispatchResult = queue.DispatchResult + +// Context contains a delivered workflow message and its correlation metadata. // -// type PollPayload struct { -// URL string `json:"url"` -// } -// var payload PollPayload -// _ = jc.Bind(&payload) -func (c Context) Bind(dst any) error { - return json.Unmarshal(c.payload, dst) -} +// Deprecated: use queue.Message. +type Context = queue.Message diff --git a/bus/wire_compat_test.go b/bus/wire_compat_test.go new file mode 100644 index 0000000..fb18741 --- /dev/null +++ b/bus/wire_compat_test.go @@ -0,0 +1,304 @@ +package bus_test + +import ( + "context" + "encoding/json" + "errors" + "reflect" + "regexp" + "testing" + "time" + + "github.com/goforj/queue/bus" + "github.com/goforj/queue/busruntime" +) + +var legacyWorkflowIDPattern = regexp.MustCompile(`\b(?:dsp|job|chn|bat|n)_[0-9a-f]{16}\b`) + +type legacyWireCall struct { + jobType string + payload []byte + options busruntime.JobOptions +} + +type legacyWireRuntime struct { + handlers map[string]busruntime.Handler + calls []legacyWireCall + execute bool +} + +// newLegacyWireRuntime creates a transport recorder that can optionally execute registered workflow deliveries inline. +func newLegacyWireRuntime(execute bool) *legacyWireRuntime { + return &legacyWireRuntime{ + handlers: make(map[string]busruntime.Handler), + execute: execute, + } +} + +// BusRegister retains the exact physical handler names selected by the workflow engine. +func (r *legacyWireRuntime) BusRegister(jobType string, handler busruntime.Handler) { + r.handlers[jobType] = handler +} + +// BusDispatch records bytes before optional execution so the fixture observes the transport boundary. +func (r *legacyWireRuntime) BusDispatch(ctx context.Context, jobType string, payload []byte, options busruntime.JobOptions) error { + r.calls = append(r.calls, legacyWireCall{ + jobType: jobType, + payload: append([]byte(nil), payload...), + options: options, + }) + if !r.execute { + return nil + } + handler := r.handlers[jobType] + if handler == nil { + return errors.New("legacy wire runtime handler is not registered") + } + return handler(ctx, legacyInboundJob{payload: payload}) +} + +// StartWorkers is inert because compatibility fixtures execute only at the SPI boundary. +func (r *legacyWireRuntime) StartWorkers(context.Context) error { return nil } + +// Shutdown is inert because compatibility fixtures own no asynchronous resources. +func (r *legacyWireRuntime) Shutdown(context.Context) error { return nil } + +type legacyInboundJob struct { + payload []byte +} + +// Bind decodes the recorded workflow delivery exactly as a physical worker would. +func (j legacyInboundJob) Bind(dst any) error { + return json.Unmarshal(j.payload, dst) +} + +// PayloadBytes returns an isolated copy so handlers cannot mutate recorded compatibility evidence. +func (j legacyInboundJob) PayloadBytes() []byte { + return append([]byte(nil), j.payload...) +} + +type frozenV1Envelope struct { + SchemaVersion int `json:"schema_version"` + DispatchID string `json:"dispatch_id"` + Kind string `json:"kind"` + JobID string `json:"job_id"` + ChainID string `json:"chain_id"` + BatchID string `json:"batch_id"` + NodeID string `json:"node_id"` + Attempt int `json:"attempt"` + Job frozenV1Job `json:"job"` + CallbackKind string `json:"callback_kind"` + Error string `json:"error"` +} + +type frozenV1Job struct { + Type string `json:"type"` + Payload []byte `json:"payload"` + Options frozenV1JobOption `json:"options"` +} + +type frozenV1JobOption struct { + Queue string + Delay time.Duration + Timeout time.Duration + Retry int + Backoff time.Duration + UniqueFor time.Duration +} + +type fixedJSONPayload struct{} + +// MarshalJSON pins deferred legacy payload encoding independently of the payload's Go representation. +func (fixedJSONPayload) MarshalJSON() ([]byte, error) { + return []byte(`{"custom":true}`), nil +} + +type failingJSONPayload struct{} + +// MarshalJSON proves legacy payload errors still occur at Dispatch rather than NewJob. +func (failingJSONPayload) MarshalJSON() ([]byte, error) { + return nil, errors.New("compat marshal failure") +} + +// TestLegacyDirectWorkflowWireV1 freezes the physical type, JSON field order, option casing, and transport options. +func TestLegacyDirectWorkflowWireV1(t *testing.T) { + runtime := newLegacyWireRuntime(false) + workflow, err := bus.New(runtime) + if err != nil { + t.Fatalf("new bus: %v", err) + } + job := bus.NewJob("compat:job", map[string]int{"id": 7}). + OnQueue("critical"). + Delay(2 * time.Second). + Timeout(3 * time.Second). + Retry(4). + Backoff(500 * time.Millisecond). + UniqueFor(30 * time.Second) + if _, err := workflow.Dispatch(context.Background(), job); err != nil { + t.Fatalf("dispatch: %v", err) + } + if len(runtime.calls) != 1 { + t.Fatalf("physical dispatch count = %d, want 1", len(runtime.calls)) + } + call := runtime.calls[0] + if call.jobType != "bus:job" { + t.Fatalf("physical type = %q, want bus:job", call.jobType) + } + gotJSON := legacyWorkflowIDPattern.ReplaceAllString(string(call.payload), "ID") + wantJSON := `{"schema_version":1,"dispatch_id":"ID","kind":"job","job_id":"ID","attempt":0,"job":{"type":"compat:job","payload":"eyJpZCI6N30=","options":{"Queue":"critical","Delay":2000000000,"Timeout":3000000000,"Retry":4,"Backoff":500000000,"UniqueFor":30000000000}}}` + if gotJSON != wantJSON { + t.Fatalf("workflow envelope changed:\n got: %s\nwant: %s", gotJSON, wantJSON) + } + wantOptions := busruntime.JobOptions{ + Queue: "critical", + Delay: 2 * time.Second, + Timeout: 3 * time.Second, + Retry: 4, + Backoff: 500 * time.Millisecond, + UniqueFor: 30 * time.Second, + } + if !reflect.DeepEqual(call.options, wantOptions) { + t.Fatalf("transport options = %+v, want %+v", call.options, wantOptions) + } +} + +// TestLegacyPayloadEncodingV1 preserves the compatibility DTO's deferred json.Marshal semantics. +func TestLegacyPayloadEncodingV1(t *testing.T) { + tests := []struct { + name string + payload any + want string + }{ + {name: "nil", payload: nil, want: "null"}, + {name: "map", payload: map[string]bool{"ready": true}, want: `{"ready":true}`}, + {name: "string", payload: "raw", want: `"raw"`}, + {name: "bytes", payload: []byte{0, 1, 2}, want: `"AAEC"`}, + {name: "raw message", payload: json.RawMessage(`{"raw":true}`), want: `{"raw":true}`}, + {name: "custom marshaler", payload: fixedJSONPayload{}, want: `{"custom":true}`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + runtime := newLegacyWireRuntime(false) + workflow, err := bus.New(runtime) + if err != nil { + t.Fatalf("new bus: %v", err) + } + if _, err := workflow.Dispatch(context.Background(), bus.NewJob("compat:payload", tt.payload)); err != nil { + t.Fatalf("dispatch: %v", err) + } + var envelope frozenV1Envelope + if err := json.Unmarshal(runtime.calls[0].payload, &envelope); err != nil { + t.Fatalf("decode envelope: %v", err) + } + if string(envelope.Job.Payload) != tt.want { + t.Fatalf("encoded payload = %q, want %q", envelope.Job.Payload, tt.want) + } + }) + } + + runtime := newLegacyWireRuntime(false) + workflow, err := bus.New(runtime) + if err != nil { + t.Fatalf("new bus: %v", err) + } + job := bus.NewJob("compat:payload", failingJSONPayload{}) + if len(runtime.calls) != 0 { + t.Fatal("NewJob unexpectedly encoded or dispatched the payload") + } + if _, err := workflow.Dispatch(context.Background(), job); err == nil || err.Error() != "json: error calling MarshalJSON for type bus_test.failingJSONPayload: compat marshal failure" { + t.Fatalf("dispatch error = %v, want deferred marshal failure", err) + } + if len(runtime.calls) != 0 { + t.Fatal("marshal failure reached the physical queue") + } +} + +// TestLegacyWorkflowDeliveryNamesV1 pins every physical orchestration route and callback kind. +func TestLegacyWorkflowDeliveryNamesV1(t *testing.T) { + tests := []struct { + name string + dispatch func(bus.Bus) error + wantTypes []string + wantCallbacks []string + }{ + { + name: "chain success", + dispatch: func(workflow bus.Bus) error { + workflow.Register("compat:ok", func(context.Context, bus.Context) error { return nil }) + _, err := workflow.Chain(bus.NewJob("compat:ok", nil)). + Finally(func(context.Context, bus.ChainState) error { return nil }). + Dispatch(context.Background()) + return err + }, + wantTypes: []string{"bus:chain:node", "bus:callback"}, + wantCallbacks: []string{"", "chain_finally"}, + }, + { + name: "chain failure", + dispatch: func(workflow bus.Bus) error { + workflow.Register("compat:fail", func(context.Context, bus.Context) error { return errors.New("chain failed") }) + _, err := workflow.Chain(bus.NewJob("compat:fail", nil)). + Catch(func(context.Context, bus.ChainState, error) error { return nil }). + Finally(func(context.Context, bus.ChainState) error { return nil }). + Dispatch(context.Background()) + return err + }, + wantTypes: []string{"bus:chain:node", "bus:callback", "bus:callback"}, + wantCallbacks: []string{"", "chain_catch", "chain_finally"}, + }, + { + name: "batch success", + dispatch: func(workflow bus.Bus) error { + workflow.Register("compat:ok", func(context.Context, bus.Context) error { return nil }) + _, err := workflow.Batch(bus.NewJob("compat:ok", nil)). + Then(func(context.Context, bus.BatchState) error { return nil }). + Finally(func(context.Context, bus.BatchState) error { return nil }). + Dispatch(context.Background()) + return err + }, + wantTypes: []string{"bus:batch:job", "bus:callback", "bus:callback"}, + wantCallbacks: []string{"", "batch_then", "batch_finally"}, + }, + { + name: "batch failure", + dispatch: func(workflow bus.Bus) error { + workflow.Register("compat:fail", func(context.Context, bus.Context) error { return errors.New("batch failed") }) + _, err := workflow.Batch(bus.NewJob("compat:fail", nil)). + Catch(func(context.Context, bus.BatchState, error) error { return nil }). + Finally(func(context.Context, bus.BatchState) error { return nil }). + Dispatch(context.Background()) + return err + }, + wantTypes: []string{"bus:batch:job", "bus:callback", "bus:callback"}, + wantCallbacks: []string{"", "batch_catch", "batch_finally"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + runtime := newLegacyWireRuntime(true) + workflow, err := bus.New(runtime) + if err != nil { + t.Fatalf("new bus: %v", err) + } + _ = tt.dispatch(workflow) + if len(runtime.calls) != len(tt.wantTypes) { + t.Fatalf("physical dispatch count = %d, want %d", len(runtime.calls), len(tt.wantTypes)) + } + for i, call := range runtime.calls { + if call.jobType != tt.wantTypes[i] { + t.Fatalf("physical type[%d] = %q, want %q", i, call.jobType, tt.wantTypes[i]) + } + var envelope frozenV1Envelope + if err := json.Unmarshal(call.payload, &envelope); err != nil { + t.Fatalf("decode envelope[%d]: %v", i, err) + } + if envelope.SchemaVersion != 1 { + t.Fatalf("schema version[%d] = %d, want 1", i, envelope.SchemaVersion) + } + if envelope.CallbackKind != tt.wantCallbacks[i] { + t.Fatalf("callback kind[%d] = %q, want %q", i, envelope.CallbackKind, tt.wantCallbacks[i]) + } + } + }) + } +} diff --git a/bus/workflow_adapter_behavior_test.go b/bus/workflow_adapter_behavior_test.go new file mode 100644 index 0000000..6396f33 --- /dev/null +++ b/bus/workflow_adapter_behavior_test.go @@ -0,0 +1,190 @@ +package bus + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "github.com/goforj/queue" + "github.com/goforj/queue/busruntime" +) + +// adapterBranchInboundJob keeps the raw-runtime test at the same serialized +// boundary used by real queue adapters. +type adapterBranchInboundJob struct { + payload []byte +} + +// Bind decodes one physical delivery for the workflow engine. +func (j adapterBranchInboundJob) Bind(dst any) error { + return json.Unmarshal(j.payload, dst) +} + +// PayloadBytes returns an isolated view of the physical delivery. +func (j adapterBranchInboundJob) PayloadBytes() []byte { + return append([]byte(nil), j.payload...) +} + +// adapterBranchRuntime executes registered deliveries synchronously so callback +// conversion is observed rather than merely retained in process-local state. +type adapterBranchRuntime struct { + handlers map[string]busruntime.Handler +} + +// BusRegister records a workflow delivery handler. +func (r *adapterBranchRuntime) BusRegister(jobType string, handler busruntime.Handler) { + if r.handlers == nil { + r.handlers = make(map[string]busruntime.Handler) + } + r.handlers[jobType] = handler +} + +// BusDispatch invokes the registered delivery at the serialized runtime seam. +func (r *adapterBranchRuntime) BusDispatch(ctx context.Context, jobType string, payload []byte, _ busruntime.JobOptions) error { + handler := r.handlers[jobType] + if handler == nil { + return errors.New("adapter branch handler is not registered") + } + return handler(ctx, adapterBranchInboundJob{payload: append([]byte(nil), payload...)}) +} + +// StartWorkers is inert because this test runtime executes synchronously. +func (r *adapterBranchRuntime) StartWorkers(context.Context) error { + return nil +} + +// Shutdown is inert because this test runtime owns no asynchronous work. +func (r *adapterBranchRuntime) Shutdown(context.Context) error { + return nil +} + +// adapterBranchStore records compatibility-store calls whose fallback methods +// are bypassed when the additive atomic store capability is present. +type adapterBranchStore struct { + Store + failChainID string + failCause error + failErr error + + successBatchID string + successJobID string + successState queue.BatchState + successDone bool + successErr error + + failureBatchID string + failureJobID string + failureCause error + failureState queue.BatchState + failureDone bool + failureErr error +} + +// FailChain records the cause without changing its identity. +func (s *adapterBranchStore) FailChain(_ context.Context, chainID string, cause error) error { + s.failChainID = chainID + s.failCause = cause + return s.failErr +} + +// MarkBatchJobSucceeded returns configured legacy aggregate state. +func (s *adapterBranchStore) MarkBatchJobSucceeded(_ context.Context, batchID, jobID string) (queue.BatchState, bool, error) { + s.successBatchID = batchID + s.successJobID = jobID + return s.successState, s.successDone, s.successErr +} + +// MarkBatchJobFailed records the delivery-local cause and returns configured state. +func (s *adapterBranchStore) MarkBatchJobFailed(_ context.Context, batchID, jobID string, cause error) (queue.BatchState, bool, error) { + s.failureBatchID = batchID + s.failureJobID = jobID + s.failureCause = cause + return s.failureState, s.failureDone, s.failureErr +} + +// TestRawRuntimeAdapterRejectsUnsupportedInputs preserves actionable errors at +// both compatibility construction boundaries. +func TestRawRuntimeAdapterRejectsUnsupportedInputs(t *testing.T) { + if compatibility, err := NewWithStore((*queue.Queue)(nil), NewMemoryStore()); compatibility != nil || err == nil || err.Error() != "queue is required" { + t.Fatalf("typed nil queue construction = bus:%v err:%v, want nil/queue is required", compatibility, err) + } + if compatibility, err := New(struct{}{}); compatibility != nil || err == nil || err.Error() != "queue does not support bus runtime adapter" { + t.Fatalf("unsupported runtime construction = bus:%v err:%v", compatibility, err) + } +} + +// TestRawRuntimeBatchProgressConvertsCommittedState proves a non-nil legacy +// progress callback observes the canonical engine state after member settlement. +func TestRawRuntimeBatchProgressConvertsCommittedState(t *testing.T) { + runtime := &adapterBranchRuntime{} + compatibility, err := New(runtime) + if err != nil { + t.Fatalf("new raw runtime adapter: %v", err) + } + compatibility.Register("adapter:batch-progress", func(context.Context, Context) error { return nil }) + + var ( + progressCalls int + progressState BatchState + ) + batchID, err := compatibility.Batch(NewJob("adapter:batch-progress", map[string]int{"id": 7})). + Progress(func(_ context.Context, state BatchState) error { + progressCalls++ + progressState = state + return nil + }). + Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch batch: %v", err) + } + if progressCalls != 1 || progressState.BatchID != batchID || !progressState.Completed || progressState.Pending != 0 || progressState.Processed != 1 { + t.Fatalf("progress calls/state = %d/%+v, want one completed member", progressCalls, progressState) + } +} + +// TestWorkflowAdapterFallbackPreservesNilShapesAndOutcomes verifies legacy +// stores retain nil collection identity, state conversion, and error identity. +func TestWorkflowAdapterFallbackPreservesNilShapesAndOutcomes(t *testing.T) { + if cloneStoredPayload(nil) != nil { + t.Fatal("nil stored payload became a non-nil slice") + } + if toQueueBatchJobs(nil) != nil { + t.Fatal("nil batch jobs became a non-nil slice") + } + if toWorkflowMiddlewares(nil) != nil { + t.Fatal("nil middleware list became a non-nil slice") + } + + failCause := errors.New("chain failed") + failErr := errors.New("chain store unavailable") + successErr := errors.New("success readback unavailable") + failureCause := errors.New("member failed") + failureErr := errors.New("failure readback unavailable") + store := &adapterBranchStore{ + failErr: failErr, + successState: queue.BatchState{BatchID: "batch-success", Processed: 1, Completed: true}, + successDone: true, + successErr: successErr, + failureState: queue.BatchState{BatchID: "batch-failure", Processed: 1, Failed: 1, Completed: true}, + failureDone: true, + failureErr: failureErr, + } + adapter := workflowStoreAdapter{store: store} + + if err := adapter.FailChain(context.Background(), "chain-1", failCause); !errors.Is(err, failErr) || store.failChainID != "chain-1" || store.failCause != failCause { + t.Fatalf("fail chain = id:%q cause:%v err:%v", store.failChainID, store.failCause, err) + } + success, done, err := adapter.MarkBatchJobSucceeded(context.Background(), "batch-success", "job-success") + if !errors.Is(err, successErr) || !done || success.BatchID != "batch-success" || !success.Completed || store.successBatchID != "batch-success" || store.successJobID != "job-success" { + t.Fatalf("successful member conversion = state:%+v done:%t err:%v store:%q/%q", success, done, err, store.successBatchID, store.successJobID) + } + failure, done, err := adapter.MarkBatchJobFailed(context.Background(), "batch-failure", "job-failure", failureCause) + if !errors.Is(err, failureErr) || !done || failure.BatchID != "batch-failure" || failure.Failed != 1 || !failure.Completed || store.failureBatchID != "batch-failure" || store.failureJobID != "job-failure" || store.failureCause != failureCause { + t.Fatalf("failed member conversion = state:%+v done:%t err:%v store:%q/%q/%v", failure, done, err, store.failureBatchID, store.failureJobID, store.failureCause) + } + + if _, ok := toWorkflowStore(store).(workflowOutcomeStoreAdapter); ok { + t.Fatal("legacy store unexpectedly advertised atomic outcome ownership") + } +} diff --git a/bus/workflow_adapters.go b/bus/workflow_adapters.go new file mode 100644 index 0000000..d0f60c7 --- /dev/null +++ b/bus/workflow_adapters.go @@ -0,0 +1,368 @@ +package bus + +import ( + "context" + "time" + + "github.com/goforj/queue" + "github.com/goforj/queue/internal/workflow" +) + +// toQueueMessage converts the private engine context into the root-owned message model. +func toQueueMessage(message workflow.Context) queue.Message { + converted := queue.NewMessage(message.JobType, message.PayloadBytes()) + converted.SchemaVersion = message.SchemaVersion + converted.DispatchID = message.DispatchID + converted.JobID = message.JobID + converted.ChainID = message.ChainID + converted.BatchID = message.BatchID + converted.Attempt = message.Attempt + return converted +} + +// toWorkflowContext converts a root-owned message back into the private engine context. +func toWorkflowContext(message queue.Message) workflow.Context { + return workflow.NewContext( + message.SchemaVersion, + message.DispatchID, + message.JobID, + message.ChainID, + message.BatchID, + message.Attempt, + message.JobType, + message.PayloadBytes(), + ) +} + +// toQueueDispatchResult converts an engine receipt into the root-owned result model. +func toQueueDispatchResult(result workflow.DispatchResult) queue.DispatchResult { + return queue.DispatchResult{DispatchID: result.DispatchID} +} + +// toWorkflowStoredJobOptions converts root-owned delivery policy into the engine model. +func toWorkflowStoredJobOptions(options queue.StoredJobOptions) workflow.JobOptions { + return workflow.JobOptions{ + Queue: options.Queue, + Delay: options.Delay, + Timeout: options.Timeout, + Retry: options.Retry, + Backoff: options.Backoff, + UniqueFor: options.UniqueFor, + } +} + +// toQueueStoredJobOptions converts engine delivery policy into the root-owned model. +func toQueueStoredJobOptions(options workflow.JobOptions) queue.StoredJobOptions { + return queue.StoredJobOptions{ + Queue: options.Queue, + Delay: options.Delay, + Timeout: options.Timeout, + Retry: options.Retry, + Backoff: options.Backoff, + UniqueFor: options.UniqueFor, + } +} + +// cloneStoredPayload isolates mutable persisted payload bytes without changing nil slices. +func cloneStoredPayload(payload []byte) []byte { + if payload == nil { + return nil + } + cloned := make([]byte, len(payload)) + copy(cloned, payload) + return cloned +} + +// toWorkflowStoredJob converts one root-owned persisted job into the engine model. +func toWorkflowStoredJob(job queue.StoredJob) workflow.StoredJob { + return workflow.StoredJob{ + Type: job.Type, + Payload: cloneStoredPayload(job.Payload), + Options: toWorkflowStoredJobOptions(job.Options), + } +} + +// toQueueStoredJob converts one engine persisted job into the root-owned model. +func toQueueStoredJob(job workflow.StoredJob) queue.StoredJob { + return queue.StoredJob{ + Type: job.Type, + Payload: cloneStoredPayload(job.Payload), + Options: toQueueStoredJobOptions(job.Options), + } +} + +// toWorkflowChainNode converts one root-owned chain node into the engine model. +func toWorkflowChainNode(node queue.ChainNode) workflow.ChainNode { + return workflow.ChainNode{NodeID: node.NodeID, Job: toWorkflowStoredJob(node.Job)} +} + +// toQueueChainNode converts one engine chain node into the root-owned model. +func toQueueChainNode(node workflow.ChainNode) queue.ChainNode { + return queue.ChainNode{NodeID: node.NodeID, Job: toQueueStoredJob(node.Job)} +} + +// toWorkflowChainNodes converts a root-owned node slice while retaining nil slices. +func toWorkflowChainNodes(nodes []queue.ChainNode) []workflow.ChainNode { + if nodes == nil { + return nil + } + converted := make([]workflow.ChainNode, len(nodes)) + for i, node := range nodes { + converted[i] = toWorkflowChainNode(node) + } + return converted +} + +// toQueueChainNodes converts an engine node slice while retaining nil slices. +func toQueueChainNodes(nodes []workflow.ChainNode) []queue.ChainNode { + if nodes == nil { + return nil + } + converted := make([]queue.ChainNode, len(nodes)) + for i, node := range nodes { + converted[i] = toQueueChainNode(node) + } + return converted +} + +// toQueueChainRecord converts engine chain creation state for a root-owned store. +func toQueueChainRecord(record workflow.ChainRecord) queue.ChainRecord { + return queue.ChainRecord{ + ChainID: record.ChainID, + DispatchID: record.DispatchID, + Queue: record.Queue, + Nodes: toQueueChainNodes(record.Nodes), + CreatedAt: record.CreatedAt, + } +} + +// toWorkflowChainState converts root-owned chain state into the engine model. +func toWorkflowChainState(state queue.ChainState) workflow.ChainState { + return workflow.ChainState{ + ChainID: state.ChainID, + DispatchID: state.DispatchID, + Queue: state.Queue, + Nodes: toWorkflowChainNodes(state.Nodes), + NextIndex: state.NextIndex, + Completed: state.Completed, + Failed: state.Failed, + Failure: state.Failure, + CreatedAt: state.CreatedAt, + UpdatedAt: state.UpdatedAt, + } +} + +// toQueueChainState converts engine chain state into the root-owned model. +func toQueueChainState(state workflow.ChainState) queue.ChainState { + return queue.ChainState{ + ChainID: state.ChainID, + DispatchID: state.DispatchID, + Queue: state.Queue, + Nodes: toQueueChainNodes(state.Nodes), + NextIndex: state.NextIndex, + Completed: state.Completed, + Failed: state.Failed, + Failure: state.Failure, + CreatedAt: state.CreatedAt, + UpdatedAt: state.UpdatedAt, + } +} + +// toQueueBatchJob converts one engine batch member into the root-owned model. +func toQueueBatchJob(job workflow.BatchJob) queue.BatchJob { + return queue.BatchJob{JobID: job.JobID, Job: toQueueStoredJob(job.Job)} +} + +// toQueueBatchJobs converts an engine member slice while retaining nil slices. +func toQueueBatchJobs(jobs []workflow.BatchJob) []queue.BatchJob { + if jobs == nil { + return nil + } + converted := make([]queue.BatchJob, len(jobs)) + for i, job := range jobs { + converted[i] = toQueueBatchJob(job) + } + return converted +} + +// toQueueBatchRecord converts engine batch creation state for a root-owned store. +func toQueueBatchRecord(record workflow.BatchRecord) queue.BatchRecord { + return queue.BatchRecord{ + BatchID: record.BatchID, + DispatchID: record.DispatchID, + Name: record.Name, + Queue: record.Queue, + AllowFailed: record.AllowFailed, + Jobs: toQueueBatchJobs(record.Jobs), + CreatedAt: record.CreatedAt, + } +} + +// toWorkflowBatchState converts root-owned aggregate state into the engine model. +func toWorkflowBatchState(state queue.BatchState) workflow.BatchState { + return workflow.BatchState{ + BatchID: state.BatchID, + DispatchID: state.DispatchID, + Name: state.Name, + Queue: state.Queue, + AllowFailed: state.AllowFailed, + Total: state.Total, + Pending: state.Pending, + Processed: state.Processed, + Failed: state.Failed, + Cancelled: state.Cancelled, + Completed: state.Completed, + CreatedAt: state.CreatedAt, + UpdatedAt: state.UpdatedAt, + } +} + +// toQueueBatchState converts engine aggregate state into the root-owned model. +func toQueueBatchState(state workflow.BatchState) queue.BatchState { + return queue.BatchState{ + BatchID: state.BatchID, + DispatchID: state.DispatchID, + Name: state.Name, + Queue: state.Queue, + AllowFailed: state.AllowFailed, + Total: state.Total, + Pending: state.Pending, + Processed: state.Processed, + Failed: state.Failed, + Cancelled: state.Cancelled, + Completed: state.Completed, + CreatedAt: state.CreatedAt, + UpdatedAt: state.UpdatedAt, + } +} + +type workflowMiddlewareAdapter struct { + middleware Middleware +} + +var _ workflow.Middleware = workflowMiddlewareAdapter{} + +// Handle preserves middleware message replacement while crossing the private engine boundary. +func (a workflowMiddlewareAdapter) Handle(ctx context.Context, message workflow.Context, next workflow.Next) error { + return a.middleware.Handle(ctx, toQueueMessage(message), func(nextContext context.Context, nextMessage queue.Message) error { + return next(nextContext, toWorkflowContext(nextMessage)) + }) +} + +// toWorkflowMiddlewares converts root-owned middleware into private engine adapters. +func toWorkflowMiddlewares(middlewares []Middleware) []workflow.Middleware { + if middlewares == nil { + return nil + } + converted := make([]workflow.Middleware, 0, len(middlewares)) + for _, middleware := range middlewares { + if middleware != nil { + converted = append(converted, workflowMiddlewareAdapter{middleware: middleware}) + } + } + return converted +} + +type workflowStoreAdapter struct { + store Store +} + +var _ workflow.Store = workflowStoreAdapter{} + +type workflowOutcomeStoreAdapter struct { + workflowStoreAdapter + atomic queue.WorkflowOutcomeStore +} + +// FailChainNode converts an atomic root-store result back into the engine model. +func (a workflowOutcomeStoreAdapter) FailChainNode(ctx context.Context, chainID, nodeID string, cause error) (workflow.ChainState, bool, error) { + state, owned, err := a.atomic.FailChainNode(ctx, chainID, nodeID, cause) + return toWorkflowChainState(state), owned, err +} + +// SettleBatchJob converts an atomic root-store result back into the engine model. +func (a workflowOutcomeStoreAdapter) SettleBatchJob(ctx context.Context, batchID, jobID string, outcome workflow.BatchJobOutcome, cause error) (workflow.BatchState, bool, error) { + state, owned, err := a.atomic.SettleBatchJob(ctx, batchID, jobID, queue.BatchJobOutcome(outcome), cause) + return toWorkflowBatchState(state), owned, err +} + +// toWorkflowStore wraps a root-owned store for the retained raw-runtime route. +func toWorkflowStore(store Store) workflow.Store { + if store == nil { + return nil + } + adapter := workflowStoreAdapter{store: store} + if atomic, ok := store.(queue.WorkflowOutcomeStore); ok { + return workflowOutcomeStoreAdapter{workflowStoreAdapter: adapter, atomic: atomic} + } + return adapter +} + +// CreateChain converts the engine record before invoking the root-owned store. +func (a workflowStoreAdapter) CreateChain(ctx context.Context, record workflow.ChainRecord) error { + return a.store.CreateChain(ctx, toQueueChainRecord(record)) +} + +// AdvanceChain converts the optional root-owned next node back into the engine model. +func (a workflowStoreAdapter) AdvanceChain(ctx context.Context, chainID string, completedNode string) (*workflow.ChainNode, bool, error) { + node, done, err := a.store.AdvanceChain(ctx, chainID, completedNode) + if node == nil { + return nil, done, err + } + converted := toWorkflowChainNode(*node) + return &converted, done, err +} + +// FailChain forwards the terminal cause without changing its error identity. +func (a workflowStoreAdapter) FailChain(ctx context.Context, chainID string, cause error) error { + return a.store.FailChain(ctx, chainID, cause) +} + +// GetChain converts root-owned state back into the engine model. +func (a workflowStoreAdapter) GetChain(ctx context.Context, chainID string) (workflow.ChainState, error) { + state, err := a.store.GetChain(ctx, chainID) + return toWorkflowChainState(state), err +} + +// CreateBatch converts the engine record before invoking the root-owned store. +func (a workflowStoreAdapter) CreateBatch(ctx context.Context, record workflow.BatchRecord) error { + return a.store.CreateBatch(ctx, toQueueBatchRecord(record)) +} + +// MarkBatchJobStarted forwards the retry-safe member-start mutation. +func (a workflowStoreAdapter) MarkBatchJobStarted(ctx context.Context, batchID, jobID string) error { + return a.store.MarkBatchJobStarted(ctx, batchID, jobID) +} + +// MarkBatchJobSucceeded converts the resulting root-owned state back into the engine model. +func (a workflowStoreAdapter) MarkBatchJobSucceeded(ctx context.Context, batchID, jobID string) (workflow.BatchState, bool, error) { + state, done, err := a.store.MarkBatchJobSucceeded(ctx, batchID, jobID) + return toWorkflowBatchState(state), done, err +} + +// MarkBatchJobFailed preserves the failure cause and converts the resulting state. +func (a workflowStoreAdapter) MarkBatchJobFailed(ctx context.Context, batchID, jobID string, cause error) (workflow.BatchState, bool, error) { + state, done, err := a.store.MarkBatchJobFailed(ctx, batchID, jobID, cause) + return toWorkflowBatchState(state), done, err +} + +// CancelBatch forwards aggregate cancellation to the root-owned store. +func (a workflowStoreAdapter) CancelBatch(ctx context.Context, batchID string) error { + return a.store.CancelBatch(ctx, batchID) +} + +// GetBatch converts root-owned aggregate state back into the engine model. +func (a workflowStoreAdapter) GetBatch(ctx context.Context, batchID string) (workflow.BatchState, error) { + state, err := a.store.GetBatch(ctx, batchID) + return toWorkflowBatchState(state), err +} + +// MarkCallbackInvoked forwards the atomic callback claim unchanged. +func (a workflowStoreAdapter) MarkCallbackInvoked(ctx context.Context, key string) (bool, error) { + return a.store.MarkCallbackInvoked(ctx, key) +} + +// Prune forwards workflow retention to the root-owned store. +func (a workflowStoreAdapter) Prune(ctx context.Context, before time.Time) error { + return a.store.Prune(ctx, before) +} diff --git a/bus/workflow_adapters_test.go b/bus/workflow_adapters_test.go new file mode 100644 index 0000000..55acaf4 --- /dev/null +++ b/bus/workflow_adapters_test.go @@ -0,0 +1,290 @@ +package bus + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/goforj/queue" + "github.com/goforj/queue/internal/workflow" +) + +type workflowAdapterStoreStub struct { + Store + advanceNode *queue.ChainNode + advanceDone bool + advanceErr error + cancelBatchID string + cancelBatchErr error +} + +type workflowOutcomeAdapterStoreStub struct { + *workflowAdapterStoreStub + chainState queue.ChainState + chainOwned bool + chainErr error + batchState queue.BatchState + batchOwned bool + batchErr error + batchOutcome queue.BatchJobOutcome +} + +// FailChainNode returns the configured root outcome for adapter conversion. +func (s *workflowOutcomeAdapterStoreStub) FailChainNode(context.Context, string, string, error) (queue.ChainState, bool, error) { + return s.chainState, s.chainOwned, s.chainErr +} + +// SettleBatchJob records the converted outcome and returns configured state. +func (s *workflowOutcomeAdapterStoreStub) SettleBatchJob(_ context.Context, _, _ string, outcome queue.BatchJobOutcome, _ error) (queue.BatchState, bool, error) { + s.batchOutcome = outcome + return s.batchState, s.batchOwned, s.batchErr +} + +// AdvanceChain returns the configured successor and outcome for adapter boundary tests. +func (s *workflowAdapterStoreStub) AdvanceChain(context.Context, string, string) (*queue.ChainNode, bool, error) { + return s.advanceNode, s.advanceDone, s.advanceErr +} + +// CancelBatch records the aggregate identifier and returns the configured error. +func (s *workflowAdapterStoreStub) CancelBatch(_ context.Context, batchID string) error { + s.cancelBatchID = batchID + return s.cancelBatchErr +} + +// TestWorkflowMessageAdaptersPreserveMetadataAndPayload pins both raw-route directions. +func TestWorkflowMessageAdaptersPreserveMetadataAndPayload(t *testing.T) { + payload := []byte(`{"id":7}`) + engineMessage := workflow.NewContext( + 1, + "dispatch-1", + "job-1", + "chain-1", + "batch-1", + 3, + "reports:build", + payload, + ) + payload[0] = '!' + + rootMessage := toQueueMessage(engineMessage) + if rootMessage.SchemaVersion != 1 || rootMessage.DispatchID != "dispatch-1" || rootMessage.JobID != "job-1" || rootMessage.ChainID != "chain-1" || rootMessage.BatchID != "batch-1" || rootMessage.Attempt != 3 || rootMessage.JobType != "reports:build" { + t.Fatalf("root message metadata changed: %+v", rootMessage) + } + if got := string(rootMessage.PayloadBytes()); got != `{"id":7}` { + t.Fatalf("root message payload = %q, want preserved JSON", got) + } + + rootMessage.JobType = "reports:replace" + roundTrip := toWorkflowContext(rootMessage) + if roundTrip.JobType != "reports:replace" || roundTrip.DispatchID != "dispatch-1" || roundTrip.Attempt != 3 { + t.Fatalf("engine message round trip changed: %+v", roundTrip) + } + returnedPayload := roundTrip.PayloadBytes() + returnedPayload[0] = '?' + if got := string(roundTrip.PayloadBytes()); got != `{"id":7}` { + t.Fatalf("engine message payload was not isolated: %q", got) + } +} + +// TestWorkflowMiddlewareAdapterPreservesMessageReplacement verifies the continuation crosses both physical models. +func TestWorkflowMiddlewareAdapterPreservesMessageReplacement(t *testing.T) { + adapter := workflowMiddlewareAdapter{middleware: MiddlewareFunc(func(ctx context.Context, _ Context, next Next) error { + replacement := queue.NewMessage("reports:replacement", []byte(`{"replacement":true}`)) + replacement.SchemaVersion = 1 + replacement.DispatchID = "dispatch-replacement" + return next(ctx, replacement) + })} + + var received workflow.Context + err := adapter.Handle(context.Background(), workflow.NewContext(1, "dispatch-original", "job-1", "", "", 0, "reports:original", []byte(`null`)), func(_ context.Context, message workflow.Context) error { + received = message + return nil + }) + if err != nil { + t.Fatalf("handle middleware: %v", err) + } + if received.JobType != "reports:replacement" || received.DispatchID != "dispatch-replacement" || string(received.PayloadBytes()) != `{"replacement":true}` { + t.Fatalf("replacement message changed across adapter: %+v payload=%q", received, received.PayloadBytes()) + } +} + +// TestWorkflowRecordAdaptersPreservePhysicalShapes verifies nested jobs, policy, times, and byte ownership. +func TestWorkflowRecordAdaptersPreservePhysicalShapes(t *testing.T) { + createdAt := time.Unix(1_704_067_200, 123_000_000) + engineChain := workflow.ChainRecord{ + ChainID: "chain-1", + DispatchID: "dispatch-1", + Queue: "critical", + Nodes: []workflow.ChainNode{{ + NodeID: "node-1", + Job: workflow.StoredJob{ + Type: "reports:build", + Payload: []byte(`{"id":7}`), + Options: workflow.JobOptions{Queue: "critical", Delay: time.Second, Timeout: 2 * time.Second, Retry: 3, Backoff: 4 * time.Second, UniqueFor: 5 * time.Second}, + }, + }}, + CreatedAt: createdAt, + } + rootChain := toQueueChainRecord(engineChain) + if rootChain.ChainID != "chain-1" || rootChain.DispatchID != "dispatch-1" || rootChain.Queue != "critical" || !rootChain.CreatedAt.Equal(createdAt) || len(rootChain.Nodes) != 1 { + t.Fatalf("root chain record changed: %+v", rootChain) + } + job := rootChain.Nodes[0].Job + if job.Type != "reports:build" || string(job.Payload) != `{"id":7}` || job.Options.Delay != time.Second || job.Options.Timeout != 2*time.Second || job.Options.Retry != 3 || job.Options.Backoff != 4*time.Second || job.Options.UniqueFor != 5*time.Second { + t.Fatalf("root stored job changed: %+v", job) + } + rootChain.Nodes[0].Job.Payload[0] = '!' + if got := string(engineChain.Nodes[0].Job.Payload); got != `{"id":7}` { + t.Fatalf("engine payload aliased root payload: %q", got) + } + + rootBatch := queue.BatchState{ + BatchID: "batch-1", + DispatchID: "dispatch-2", + Name: "nightly", + Queue: "bulk", + AllowFailed: true, + Total: 8, + Pending: 3, + Processed: 5, + Failed: 2, + Cancelled: false, + Completed: false, + CreatedAt: createdAt, + UpdatedAt: createdAt.Add(time.Minute), + } + engineBatch := toWorkflowBatchState(rootBatch) + if engineBatch.BatchID != rootBatch.BatchID || engineBatch.DispatchID != rootBatch.DispatchID || engineBatch.Name != rootBatch.Name || engineBatch.Queue != rootBatch.Queue || engineBatch.AllowFailed != rootBatch.AllowFailed || engineBatch.Total != rootBatch.Total || engineBatch.Pending != rootBatch.Pending || engineBatch.Processed != rootBatch.Processed || engineBatch.Failed != rootBatch.Failed || engineBatch.Cancelled != rootBatch.Cancelled || engineBatch.Completed != rootBatch.Completed || !engineBatch.CreatedAt.Equal(rootBatch.CreatedAt) || !engineBatch.UpdatedAt.Equal(rootBatch.UpdatedAt) { + t.Fatalf("engine batch state changed: %+v", engineBatch) + } +} + +// TestWorkflowStoreAdapterAdvanceChainCoversSuccessorBranches pins optional-node conversion and error identity. +func TestWorkflowStoreAdapterAdvanceChainCoversSuccessorBranches(t *testing.T) { + sentinel := errors.New("advance chain failed") + tests := []struct { + name string + node *queue.ChainNode + done bool + err error + wantNode bool + }{ + {name: "nil successor", done: true}, + { + name: "converted successor", + node: &queue.ChainNode{ + NodeID: "node-2", + Job: queue.StoredJob{ + Type: "reports:publish", + Payload: []byte(`{"id":8}`), + Options: queue.StoredJobOptions{ + Queue: "critical", + Delay: time.Second, + Timeout: 2 * time.Second, + Retry: 3, + Backoff: 4 * time.Second, + UniqueFor: 5 * time.Second, + }, + }, + }, + wantNode: true, + }, + { + name: "successor with store error", + node: &queue.ChainNode{ + NodeID: "node-error", + Job: queue.StoredJob{Type: "reports:error", Payload: []byte(`null`)}, + }, + err: sentinel, + wantNode: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + store := &workflowAdapterStoreStub{ + advanceNode: test.node, + advanceDone: test.done, + advanceErr: test.err, + } + adapted := toWorkflowStore(store) + node, done, err := adapted.AdvanceChain(context.Background(), "chain-1", "node-1") + if done != test.done { + t.Fatalf("done = %t, want %t", done, test.done) + } + if err != test.err { + t.Fatalf("error = %v, want exact identity %v", err, test.err) + } + if (node != nil) != test.wantNode { + t.Fatalf("successor = %+v, want present=%t", node, test.wantNode) + } + if node == nil { + return + } + if node.NodeID != test.node.NodeID || node.Job.Type != test.node.Job.Type || string(node.Job.Payload) != string(test.node.Job.Payload) { + t.Fatalf("converted successor = %+v, want %+v", node, test.node) + } + if node.Job.Options.Queue != test.node.Job.Options.Queue || node.Job.Options.Delay != test.node.Job.Options.Delay || node.Job.Options.Timeout != test.node.Job.Options.Timeout || node.Job.Options.Retry != test.node.Job.Options.Retry || node.Job.Options.Backoff != test.node.Job.Options.Backoff || node.Job.Options.UniqueFor != test.node.Job.Options.UniqueFor { + t.Fatalf("converted successor options = %+v, want %+v", node.Job.Options, test.node.Job.Options) + } + if len(node.Job.Payload) > 0 { + node.Job.Payload[0] = '!' + if test.node.Job.Payload[0] == '!' { + t.Fatal("converted successor payload aliases the physical store value") + } + } + }) + } + + if got := toWorkflowStore(nil); got != nil { + t.Fatalf("nil store adapted to %T, want nil", got) + } +} + +// TestWorkflowStoreAdapterCancelBatchPreservesIDAndError verifies cancellation forwarding without error wrapping. +func TestWorkflowStoreAdapterCancelBatchPreservesIDAndError(t *testing.T) { + sentinel := errors.New("cancel batch failed") + store := &workflowAdapterStoreStub{cancelBatchErr: sentinel} + adapted := toWorkflowStore(store) + + err := adapted.CancelBatch(context.Background(), "batch-7") + if err != sentinel { + t.Fatalf("cancel error = %v, want exact identity %v", err, sentinel) + } + if store.cancelBatchID != "batch-7" { + t.Fatalf("cancelled batch = %q, want batch-7", store.cancelBatchID) + } +} + +// TestWorkflowOutcomeStoreAdapterPreservesOwnership proves the deprecated raw +// route forwards the one canonical root capability instead of redefining it. +func TestWorkflowOutcomeStoreAdapterPreservesOwnership(t *testing.T) { + chainErr := errors.New("chain outcome failed") + batchErr := errors.New("batch outcome failed") + store := &workflowOutcomeAdapterStoreStub{ + workflowAdapterStoreStub: &workflowAdapterStoreStub{}, + chainState: queue.ChainState{ChainID: "chain-outcome", Failed: true}, + chainOwned: true, + chainErr: chainErr, + batchState: queue.BatchState{BatchID: "batch-outcome", Processed: 1}, + batchOwned: false, + batchErr: batchErr, + } + adapted := toWorkflowStore(store) + outcomes, ok := adapted.(interface { + FailChainNode(context.Context, string, string, error) (workflow.ChainState, bool, error) + SettleBatchJob(context.Context, string, string, workflow.BatchJobOutcome, error) (workflow.BatchState, bool, error) + }) + if !ok { + t.Fatalf("capable store adapted as %T without outcome capability", adapted) + } + chainState, owned, err := outcomes.FailChainNode(context.Background(), "chain-outcome", "node-outcome", chainErr) + if err != chainErr || !owned || chainState.ChainID != "chain-outcome" || !chainState.Failed { + t.Fatalf("chain outcome = state:%+v owned:%t err:%v", chainState, owned, err) + } + batchState, owned, err := outcomes.SettleBatchJob(context.Background(), "batch-outcome", "job-outcome", workflow.BatchJobFailed, batchErr) + if err != batchErr || owned || batchState.BatchID != "batch-outcome" || store.batchOutcome != queue.BatchJobFailed { + t.Fatalf("batch outcome = state:%+v owned:%t err:%v stored:%q", batchState, owned, err, store.batchOutcome) + } +} diff --git a/busruntime/delivery_settlement.go b/busruntime/delivery_settlement.go new file mode 100644 index 0000000..da7915c --- /dev/null +++ b/busruntime/delivery_settlement.go @@ -0,0 +1,170 @@ +package busruntime + +import ( + "context" + "sync" +) + +// DeliverySettlement defers facts that become true only after a worker commits broker settlement. +// Its zero value is ready for use, and Commit is idempotent. +type DeliverySettlement struct { + mu sync.Mutex + committed bool + applicationStateCommitted bool + callbacks []func() +} + +// DeliverySettlementIdentity is an opaque, comparable identity for one +// physical handler invocation. It permits correlation without exposing the +// driver's authority to commit the underlying settlement. +type DeliverySettlementIdentity struct { + settlement *DeliverySettlement +} + +type deliverySettlementContextKey struct{} +type deliveryProvenanceContextKey struct{} + +// DeliveryProvenance identifies the current settlement generation and any +// earlier generation whose durable delivery definitely remained unsettled. +// Workflow state must independently prove which generation owns a transition. +type DeliveryProvenance struct { + // GenerationID is an opaque identifier generated by the settlement owner + // for the physical delivery currently invoking the handler. + GenerationID string + // RecoveredGenerationID identifies the earlier unsettled generation when + // that identity survived reclamation. + RecoveredGenerationID string + // Recovered reports stale-generation evidence even when a legacy delivery + // did not retain an owner identifier. + Recovered bool +} + +// WithDeliverySettlement attaches one settlement boundary to a handler context. +// Drivers that acknowledge or delete deliveries after handler execution use the returned value to commit deferred facts. +func WithDeliverySettlement(ctx context.Context) (context.Context, *DeliverySettlement) { + if ctx == nil { + ctx = context.Background() + } + settlement := &DeliverySettlement{} + return context.WithValue(ctx, deliverySettlementContextKey{}, settlement), settlement +} + +// DeliverySettlementIdentityFromContext returns the opaque identity for the +// physical settlement boundary attached to ctx. +func DeliverySettlementIdentityFromContext(ctx context.Context) (DeliverySettlementIdentity, bool) { + settlement, ok := deliverySettlementFromContext(ctx) + if !ok { + return DeliverySettlementIdentity{}, false + } + return DeliverySettlementIdentity{settlement: settlement}, true +} + +// deliverySettlementFromContext returns the driver-owned mutable settlement +// boundary without exposing settlement authority outside this package. +func deliverySettlementFromContext(ctx context.Context) (*DeliverySettlement, bool) { + if ctx == nil { + return nil, false + } + settlement, ok := ctx.Value(deliverySettlementContextKey{}).(*DeliverySettlement) + return settlement, ok && settlement != nil +} + +// WithDeliveryProvenance attaches settlement-owner generation identity to one +// physical handler invocation. Callers must not manufacture recovery evidence +// for an ordinary duplicate. +func WithDeliveryProvenance(ctx context.Context, provenance DeliveryProvenance) context.Context { + if ctx == nil { + ctx = context.Background() + } + return context.WithValue(ctx, deliveryProvenanceContextKey{}, provenance) +} + +// DeliveryProvenanceFromContext returns settlement-owner generation identity +// without treating it as proof of a workflow transition. +func DeliveryProvenanceFromContext(ctx context.Context) (DeliveryProvenance, bool) { + if ctx == nil { + return DeliveryProvenance{}, false + } + provenance, ok := ctx.Value(deliveryProvenanceContextKey{}).(DeliveryProvenance) + return provenance, ok +} + +// MarkDeliveryApplicationStateCommitted records that the current generation +// durably mutated application state before physical delivery settlement. SQL +// drivers use this only to preserve truthful provenance across a later +// same-attempt infrastructure redelivery. +func MarkDeliveryApplicationStateCommitted(ctx context.Context) bool { + settlement, ok := deliverySettlementFromContext(ctx) + if !ok { + return false + } + settlement.mu.Lock() + settlement.applicationStateCommitted = true + settlement.mu.Unlock() + return true +} + +// ApplicationStateCommitted reports whether the current generation durably +// mutated application state before the driver attempted physical settlement. +func (s *DeliverySettlement) ApplicationStateCommitted() bool { + if s == nil { + return false + } + s.mu.Lock() + defer s.mu.Unlock() + return s.applicationStateCommitted +} + +// DeferUntilDeliveryCommitted registers fn when ctx carries a driver-owned settlement boundary. +// It returns false when the driver cannot report settlement, allowing callers to preserve legacy immediate behavior. +func DeferUntilDeliveryCommitted(ctx context.Context, fn func()) bool { + if fn == nil { + return false + } + settlement, ok := deliverySettlementFromContext(ctx) + if !ok { + return false + } + settlement.deferCommit(fn) + return true +} + +// Commit publishes every deferred fact after the driver has positively settled the delivery. +func (s *DeliverySettlement) Commit() { + if s == nil { + return + } + s.mu.Lock() + if s.committed { + s.mu.Unlock() + return + } + s.committed = true + callbacks := s.callbacks + s.callbacks = nil + s.mu.Unlock() + + for _, callback := range callbacks { + invokeDeliveryCommit(callback) + } +} + +// deferCommit queues a callback or invokes it immediately when settlement already committed. +func (s *DeliverySettlement) deferCommit(fn func()) { + s.mu.Lock() + if !s.committed { + s.callbacks = append(s.callbacks, fn) + s.mu.Unlock() + return + } + s.mu.Unlock() + invokeDeliveryCommit(fn) +} + +// invokeDeliveryCommit isolates deferred telemetry so it cannot invalidate an already committed broker acknowledgement. +func invokeDeliveryCommit(fn func()) { + defer func() { + _ = recover() + }() + fn() +} diff --git a/busruntime/delivery_settlement_test.go b/busruntime/delivery_settlement_test.go new file mode 100644 index 0000000..5522782 --- /dev/null +++ b/busruntime/delivery_settlement_test.go @@ -0,0 +1,115 @@ +package busruntime + +import ( + "context" + "sync/atomic" + "testing" +) + +// TestDeliverySettlementDefersAndCommitsOnce verifies settlement facts cannot precede the driver's positive commit. +func TestDeliverySettlementDefersAndCommitsOnce(t *testing.T) { + ctx, settlement := WithDeliverySettlement(context.Background()) + identity, ok := DeliverySettlementIdentityFromContext(ctx) + if !ok { + t.Fatal("settlement identity was absent from its context") + } + if same, sameOK := DeliverySettlementIdentityFromContext(ctx); !sameOK || same != identity { + t.Fatal("settlement identity changed for the same context") + } + otherCtx, _ := WithDeliverySettlement(context.Background()) + if other, otherOK := DeliverySettlementIdentityFromContext(otherCtx); !otherOK || other == identity { + t.Fatal("distinct settlements shared one identity") + } + var calls atomic.Int32 + if !DeferUntilDeliveryCommitted(ctx, func() { calls.Add(1) }) { + t.Fatal("settlement callback was not deferred") + } + if calls.Load() != 0 { + t.Fatal("settlement callback ran before commit") + } + settlement.Commit() + settlement.Commit() + if calls.Load() != 1 { + t.Fatalf("settlement callback calls = %d, want 1", calls.Load()) + } +} + +// TestDeliverySettlementLateRegistration verifies facts registered after a commit still observe that committed boundary. +func TestDeliverySettlementLateRegistration(t *testing.T) { + ctx, settlement := WithDeliverySettlement(nil) + settlement.Commit() + called := false + if !DeferUntilDeliveryCommitted(ctx, func() { called = true }) { + t.Fatal("late settlement callback was not recognized") + } + if !called { + t.Fatal("late settlement callback did not run") + } +} + +// TestDeliverySettlementAbsentAndPanickingCallbacks verifies optional settlement and telemetry panics remain isolated. +func TestDeliverySettlementAbsentAndPanickingCallbacks(t *testing.T) { + if identity, ok := DeliverySettlementIdentityFromContext(nil); ok || identity != (DeliverySettlementIdentity{}) { + t.Fatalf("nil context identity = %#v ok:%t, want zero/false", identity, ok) + } + if identity, ok := DeliverySettlementIdentityFromContext(context.Background()); ok || identity != (DeliverySettlementIdentity{}) { + t.Fatalf("plain context identity = %#v ok:%t, want zero/false", identity, ok) + } + if DeferUntilDeliveryCommitted(context.Background(), func() {}) { + t.Fatal("plain context unexpectedly exposed settlement") + } + ctx, settlement := WithDeliverySettlement(context.Background()) + var after atomic.Bool + DeferUntilDeliveryCommitted(ctx, func() { panic("observer failed") }) + DeferUntilDeliveryCommitted(ctx, func() { after.Store(true) }) + settlement.Commit() + if !after.Load() { + t.Fatal("panicking callback prevented later settlement facts") + } +} + +// TestDeliveryProvenanceContext distinguishes current generation identity from +// an earlier delivery whose settlement owner proved it remained unsettled. +func TestDeliveryProvenanceContext(t *testing.T) { + if _, ok := DeliveryProvenanceFromContext(nil); ok { + t.Fatal("nil context unexpectedly reports delivery provenance") + } + if _, ok := DeliveryProvenanceFromContext(context.Background()); ok { + t.Fatal("plain context unexpectedly reports delivery provenance") + } + want := DeliveryProvenance{ + GenerationID: "generation-current", + RecoveredGenerationID: "generation-earlier", + Recovered: true, + } + ctx := WithDeliveryProvenance(nil, want) + if got, ok := DeliveryProvenanceFromContext(ctx); !ok || got != want { + t.Fatalf("delivery provenance = %+v ok:%t, want %+v/true", got, ok, want) + } +} + +// TestDeliveryApplicationStateCommittedSignal keeps post-mutation provenance +// distinct from both deferred fact publication and physical settlement. +func TestDeliveryApplicationStateCommittedSignal(t *testing.T) { + if MarkDeliveryApplicationStateCommitted(nil) { + t.Fatal("nil context accepted an application-state signal") + } + if MarkDeliveryApplicationStateCommitted(context.Background()) { + t.Fatal("plain context accepted an application-state signal") + } + ctx, settlement := WithDeliverySettlement(context.Background()) + if settlement.ApplicationStateCommitted() { + t.Fatal("new settlement reports committed application state") + } + if !MarkDeliveryApplicationStateCommitted(ctx) || !settlement.ApplicationStateCommitted() { + t.Fatal("settlement did not retain committed application state") + } + settlement.Commit() + if !settlement.ApplicationStateCommitted() { + t.Fatal("physical settlement erased application-state provenance") + } + var nilSettlement *DeliverySettlement + if nilSettlement.ApplicationStateCommitted() { + t.Fatal("nil settlement reports committed application state") + } +} diff --git a/busruntime/optional_boundaries_test.go b/busruntime/optional_boundaries_test.go new file mode 100644 index 0000000..6ccb40d --- /dev/null +++ b/busruntime/optional_boundaries_test.go @@ -0,0 +1,40 @@ +package busruntime + +import ( + "context" + "testing" +) + +// TestDeliverySettlementNilInputsRemainOptional verifies callers can retain +// immediate legacy behavior without manufacturing a settlement boundary. +func TestDeliverySettlementNilInputsRemainOptional(t *testing.T) { + if DeferUntilDeliveryCommitted(nil, func() {}) { + t.Fatal("nil context accepted a deferred delivery callback") + } + ctx, settlement := WithDeliverySettlement(context.Background()) + if DeferUntilDeliveryCommitted(ctx, nil) { + t.Fatal("nil callback was reported as deferred") + } + settlement.Commit() + + var absent *DeliverySettlement + absent.Commit() +} + +// TestZeroContinuationScopeNeverAuthorizesDispatch proves nil and zero-value +// scopes preserve their documented no-permission behavior. +func TestZeroContinuationScopeNeverAuthorizesDispatch(t *testing.T) { + var nilScope *ContinuationScope + ctx, release := nilScope.Permit(nil) + release() + if ctx == nil || nilScope.Owns(ctx) { + t.Fatal("nil continuation scope granted permission") + } + + zeroScope := &ContinuationScope{} + ctx, release = zeroScope.Permit(context.Background()) + release() + if zeroScope.Owns(ctx) { + t.Fatal("zero continuation scope granted permission") + } +} diff --git a/busruntime/runtime.go b/busruntime/runtime.go index 2169ffd..b0b7552 100644 --- a/busruntime/runtime.go +++ b/busruntime/runtime.go @@ -2,17 +2,247 @@ package busruntime import ( "context" + "errors" + "sync/atomic" "time" ) +// DeliveryAttempt identifies one application attempt and its configured retry budget. +// Number is zero-based, and MaxRetry is the number of retries after the initial attempt. +type DeliveryAttempt struct { + Number int + MaxRetry int +} + +// Exhausted reports whether the current application attempt has consumed its retry budget. +func (a DeliveryAttempt) Exhausted() bool { + return a.Number >= a.MaxRetry +} + +// AttemptDecision describes how a worker must settle an application attempt. +type AttemptDecision uint8 + +const ( + // AttemptSucceeded commits successful handler execution. + AttemptSucceeded AttemptDecision = iota + // AttemptRetry schedules a later application attempt after the current attempt failed. + AttemptRetry + // AttemptFailed commits a permanent or exhausted application failure. + AttemptFailed + // AttemptRedeliver retries infrastructure work without consuming the application retry budget. + AttemptRedeliver +) + +type deliveryAttemptContextKey struct{} +type continuationDispatchContextKey struct{} + +type continuationScopeToken struct { + identity byte +} + +type continuationPermit struct { + scope *continuationScopeToken + active atomic.Bool +} + +// ContinuationScope owns short-lived permission for one runtime's handlers to enqueue descendants while that runtime drains. +// Its zero value grants no permission; use NewContinuationScope. +type ContinuationScope struct { + token *continuationScopeToken +} + +// WithDeliveryAttempt attaches physical delivery metadata for orchestration and middleware classification. +func WithDeliveryAttempt(ctx context.Context, attempt DeliveryAttempt) context.Context { + if ctx == nil { + ctx = context.Background() + } + return context.WithValue(ctx, deliveryAttemptContextKey{}, attempt) +} + +// DeliveryAttemptFromContext returns physical delivery metadata when a worker supplied it. +func DeliveryAttemptFromContext(ctx context.Context) (DeliveryAttempt, bool) { + if ctx == nil { + return DeliveryAttempt{}, false + } + attempt, ok := ctx.Value(deliveryAttemptContextKey{}).(DeliveryAttempt) + return attempt, ok +} + +// NewContinuationScope creates an unforgeable runtime-specific continuation scope. +func NewContinuationScope() *ContinuationScope { + return &ContinuationScope{token: &continuationScopeToken{}} +} + +// Permit marks ctx only until the returned release function runs. +// Runtime adapters release the permit as the originating handler returns so escaped contexts cannot enqueue during a later drain. +func (s *ContinuationScope) Permit(ctx context.Context) (context.Context, func()) { + if ctx == nil { + ctx = context.Background() + } + if s == nil || s.token == nil { + return ctx, func() {} + } + permit := &continuationPermit{scope: s.token} + permit.active.Store(true) + current := continuationPermits(ctx) + permits := append(make([]*continuationPermit, 0, len(current)+1), current...) + permits = append(permits, permit) + marked := context.WithValue(ctx, continuationDispatchContextKey{}, permits) + return marked, func() { permit.active.Store(false) } +} + +// Owns reports whether ctx carries a still-active permit issued by this scope. +func (s *ContinuationScope) Owns(ctx context.Context) bool { + if s == nil || s.token == nil { + return false + } + for _, permit := range continuationPermits(ctx) { + if permit != nil && permit.scope == s.token && permit.active.Load() { + return true + } + } + return false +} + +// PreserveDeliveryContext carries runtime-owned delivery state from source into +// a replacement context while retaining replacement cancellation, deadlines, +// and user values. Source runtime values override replacement values, and the +// same permit pointers are reused so handler return still expires permission. +func PreserveDeliveryContext(source context.Context, replacement context.Context) context.Context { + if replacement == nil { + replacement = context.Background() + } + preserved := replacement + if settlement, ok := deliverySettlementFromContext(source); ok { + preserved = context.WithValue(preserved, deliverySettlementContextKey{}, settlement) + } + if provenance, ok := DeliveryProvenanceFromContext(source); ok { + preserved = context.WithValue(preserved, deliveryProvenanceContextKey{}, provenance) + } + if source != nil { + if attempt, ok := source.Value(deliveryAttemptContextKey{}).(DeliveryAttempt); ok { + preserved = context.WithValue(preserved, deliveryAttemptContextKey{}, attempt) + } + if metadata, ok := source.Value(deliveryMetadataContextKey{}).(DeliveryMetadata); ok { + preserved = context.WithValue(preserved, deliveryMetadataContextKey{}, metadata) + } + } + sourcePermits := continuationPermits(source) + if len(sourcePermits) == 0 { + return preserved + } + replacementPermits := continuationPermits(preserved) + merged := append(make([]*continuationPermit, 0, len(sourcePermits)+len(replacementPermits)), replacementPermits...) + changed := false + for _, sourcePermit := range sourcePermits { + found := false + for _, replacementPermit := range replacementPermits { + if sourcePermit == replacementPermit { + found = true + break + } + } + if found { + continue + } + merged = append(merged, sourcePermit) + changed = true + } + if !changed { + return preserved + } + return context.WithValue(preserved, continuationDispatchContextKey{}, merged) +} + +// continuationPermits returns the immutable permit snapshot attached by nested runtime handlers. +func continuationPermits(ctx context.Context) []*continuationPermit { + if ctx == nil { + return nil + } + permits, _ := ctx.Value(continuationDispatchContextKey{}).([]*continuationPermit) + return permits +} + +// ClassifyAttempt maps a handler result to the settlement decision owned by its worker or driver. +func ClassifyAttempt(attempt DeliveryAttempt, err error) AttemptDecision { + switch { + case err == nil: + return AttemptSucceeded + case IsUncommitted(err): + return AttemptRedeliver + case IsPermanent(err), attempt.Exhausted(): + return AttemptFailed + default: + return AttemptRetry + } +} + +type permanentError struct { + cause error +} + +// Error describes the permanent application failure. +func (e permanentError) Error() string { + return e.cause.Error() +} + +// Unwrap preserves errors.Is and errors.As behavior for the application failure. +func (e permanentError) Unwrap() error { + return e.cause +} + +// Permanent marks an application error as terminal regardless of remaining retries. +func Permanent(err error) error { + if err == nil || IsPermanent(err) { + return err + } + return permanentError{cause: err} +} + +// IsPermanent reports whether an error requests terminal application settlement. +func IsPermanent(err error) bool { + var target permanentError + return errors.As(err, &target) +} + +type uncommittedError struct { + cause error +} + +// Error describes the infrastructure failure that prevented an outcome from being committed. +func (e uncommittedError) Error() string { + return e.cause.Error() +} + +// Unwrap preserves errors.Is and errors.As behavior for the infrastructure failure. +func (e uncommittedError) Unwrap() error { + return e.cause +} + +// Uncommitted marks an infrastructure error for redelivery without consuming application retries. +func Uncommitted(err error) error { + if err == nil || IsUncommitted(err) { + return err + } + return uncommittedError{cause: err} +} + +// IsUncommitted reports whether an application outcome still needs to be committed. +func IsUncommitted(err error) bool { + var target uncommittedError + return errors.As(err, &target) +} + // InboundJob is the minimal job view the orchestration runtime needs from the queue layer. type InboundJob interface { Bind(dst any) error PayloadBytes() []byte } +// Handler processes one inbound delivery for the orchestration runtime. type Handler func(ctx context.Context, job InboundJob) error +// JobOptions carries delivery policy from orchestration into the queue runtime. type JobOptions struct { Queue string Delay time.Duration @@ -22,6 +252,22 @@ type JobOptions struct { UniqueFor time.Duration } +// DeliveryMetadataVersion identifies the direct-delivery metadata understood +// by this version of the runtime and driver integration contract. +const DeliveryMetadataVersion = 1 + +// DeliveryMetadata carries correlation for an ordinary direct job without +// changing its application type or payload. Its JSON representation is the +// canonical persisted and transported driver metadata record. +type DeliveryMetadata struct { + SchemaVersion int `json:"schema_version"` + DispatchID string `json:"dispatch_id,omitempty"` + JobID string `json:"job_id,omitempty"` + ChainID string `json:"chain_id,omitempty"` + BatchID string `json:"batch_id,omitempty"` + Queue string `json:"queue,omitempty"` +} + // Runtime is the queue runtime surface required by the orchestration engine. type Runtime interface { BusRegister(jobType string, handler Handler) @@ -29,3 +275,36 @@ type Runtime interface { StartWorkers(ctx context.Context) error Shutdown(ctx context.Context) error } + +// DirectRuntime extends Runtime with canonical direct-job dispatch. Its +// embedded Runtime registration method handles both application types and +// retained legacy envelopes. +type DirectRuntime interface { + Runtime + // BusDispatchDirect submits application bytes with correlation kept in the metadata channel. + BusDispatchDirect(ctx context.Context, jobType string, payload []byte, metadata DeliveryMetadata, opts JobOptions) error +} + +type deliveryMetadataContextKey struct{} + +// WithDeliveryMetadata attaches direct-job correlation to one physical handler +// invocation without exposing transport framing to the application payload. +func WithDeliveryMetadata(ctx context.Context, metadata DeliveryMetadata) context.Context { + if ctx == nil { + ctx = context.Background() + } + return context.WithValue(ctx, deliveryMetadataContextKey{}, metadata) +} + +// DeliveryMetadataFromContext returns supported direct-job correlation supplied +// by a compatible worker runtime. Missing and unknown versions are untrusted. +func DeliveryMetadataFromContext(ctx context.Context) (DeliveryMetadata, bool) { + if ctx == nil { + return DeliveryMetadata{}, false + } + metadata, ok := ctx.Value(deliveryMetadataContextKey{}).(DeliveryMetadata) + if !ok || metadata.SchemaVersion != DeliveryMetadataVersion { + return DeliveryMetadata{}, false + } + return metadata, true +} diff --git a/busruntime/runtime_test.go b/busruntime/runtime_test.go new file mode 100644 index 0000000..8cd4a23 --- /dev/null +++ b/busruntime/runtime_test.go @@ -0,0 +1,259 @@ +package busruntime + +import ( + "context" + "errors" + "fmt" + "testing" + "time" +) + +// TestDeliveryAttemptContext verifies attempt metadata remains typed and nil-context safe. +func TestDeliveryAttemptContext(t *testing.T) { + if _, ok := DeliveryAttemptFromContext(nil); ok { + t.Fatal("nil context unexpectedly contained an attempt") + } + + want := DeliveryAttempt{Number: 2, MaxRetry: 4} + ctx := WithDeliveryAttempt(nil, want) + got, ok := DeliveryAttemptFromContext(ctx) + if !ok || got != want { + t.Fatalf("delivery attempt = %+v, %t; want %+v, true", got, ok, want) + } + if _, ok := DeliveryAttemptFromContext(context.Background()); ok { + t.Fatal("plain context unexpectedly contained an attempt") + } +} + +// TestDeliveryMetadataContext verifies direct correlation remains typed and +// independent from the physical attempt context. +func TestDeliveryMetadataContext(t *testing.T) { + if _, ok := DeliveryMetadataFromContext(nil); ok { + t.Fatal("nil context unexpectedly contained delivery metadata") + } + want := DeliveryMetadata{ + SchemaVersion: DeliveryMetadataVersion, + DispatchID: "dsp_1", + JobID: "job_1", + Queue: "critical", + } + ctx := WithDeliveryMetadata(nil, want) + got, ok := DeliveryMetadataFromContext(ctx) + if !ok || got != want { + t.Fatalf("delivery metadata = %+v, %t; want %+v, true", got, ok, want) + } + if _, ok := DeliveryAttemptFromContext(ctx); ok { + t.Fatal("delivery metadata unexpectedly invented an attempt") + } + future := WithDeliveryMetadata(ctx, DeliveryMetadata{ + SchemaVersion: DeliveryMetadataVersion + 1, + DispatchID: "spoofed", + }) + if metadata, ok := DeliveryMetadataFromContext(future); ok || metadata != (DeliveryMetadata{}) { + t.Fatalf("future metadata = %+v, %t; want zero, false", metadata, ok) + } +} + +// TestContinuationScope verifies drain permission is runtime-owned, explicit, expiring, and nil-context safe. +func TestContinuationScope(t *testing.T) { + first := NewContinuationScope() + second := NewContinuationScope() + if first.Owns(nil) || first.Owns(context.Background()) { + t.Fatal("unmarked context unexpectedly belonged to a continuation scope") + } + ctx, release := first.Permit(nil) + if !first.Owns(ctx) || second.Owns(ctx) { + t.Fatal("marked context did not preserve scoped continuation ownership") + } + release() + if first.Owns(ctx) { + t.Fatal("released or escaped context retained continuation permission") + } +} + +// TestPreserveDeliveryContext verifies a replacement retains driver-owned +// delivery state while using its own cancellation, deadline, and user values. +func TestPreserveDeliveryContext(t *testing.T) { + type contextKey struct{} + key := contextKey{} + first := NewContinuationScope() + second := NewContinuationScope() + replacementScope := NewContinuationScope() + wantProvenance := DeliveryProvenance{GenerationID: "generation-source", RecoveredGenerationID: "generation-old", Recovered: true} + wantAttempt := DeliveryAttempt{Number: 2, MaxRetry: 4} + wantMetadata := DeliveryMetadata{ + SchemaVersion: DeliveryMetadataVersion, + DispatchID: "dispatch-source", + JobID: "job-source", + ChainID: "chain-source", + Queue: "critical", + } + + source, cancelSource := context.WithCancel(context.WithValue(context.Background(), key, "source")) + cancelSource() + source, _ = WithDeliverySettlement(source) + sourceIdentity, sourceIdentityOK := DeliverySettlementIdentityFromContext(source) + if !sourceIdentityOK { + t.Fatal("source context did not retain its settlement identity") + } + source = WithDeliveryProvenance(source, wantProvenance) + source = WithDeliveryAttempt(source, wantAttempt) + source = WithDeliveryMetadata(source, wantMetadata) + source, releaseFirst := first.Permit(source) + source, releaseSecond := second.Permit(source) + replacementBase, cancelReplacement := context.WithTimeout(context.WithValue(context.Background(), key, "replacement"), time.Hour) + replacement, _ := WithDeliverySettlement(replacementBase) + replacementIdentity, replacementIdentityOK := DeliverySettlementIdentityFromContext(replacement) + if !replacementIdentityOK || replacementIdentity == sourceIdentity { + t.Fatal("replacement context did not begin with an independent settlement") + } + replacement = WithDeliveryProvenance(replacement, DeliveryProvenance{GenerationID: "spoofed"}) + replacement = WithDeliveryAttempt(replacement, DeliveryAttempt{Number: 99, MaxRetry: 99}) + replacement = WithDeliveryMetadata(replacement, DeliveryMetadata{SchemaVersion: DeliveryMetadataVersion, DispatchID: "spoofed"}) + replacement, releaseReplacement := replacementScope.Permit(replacement) + preserved := PreserveDeliveryContext(source, replacement) + + if !first.Owns(preserved) || !second.Owns(preserved) || !replacementScope.Owns(preserved) { + t.Fatal("replacement context did not retain every live continuation permit") + } + if got := preserved.Value(key); got != "replacement" { + t.Fatalf("replacement context value = %v, want replacement", got) + } + if err := preserved.Err(); err != nil { + t.Fatalf("preserved context inherited source cancellation: %v", err) + } + if got, ok := DeliverySettlementIdentityFromContext(preserved); !ok || got != sourceIdentity { + t.Fatalf("settlement identity = %+v, %t; want source identity", got, ok) + } + if got, ok := DeliveryProvenanceFromContext(preserved); !ok || got != wantProvenance { + t.Fatalf("delivery provenance = %+v, %t; want %+v", got, ok, wantProvenance) + } + if got, ok := DeliveryAttemptFromContext(preserved); !ok || got != wantAttempt { + t.Fatalf("delivery attempt = %+v, %t; want %+v", got, ok, wantAttempt) + } + if got, ok := DeliveryMetadataFromContext(preserved); !ok || got != wantMetadata { + t.Fatalf("delivery metadata = %+v, %t; want %+v", got, ok, wantMetadata) + } + wantDeadline, wantDeadlineOK := replacement.Deadline() + if gotDeadline, ok := preserved.Deadline(); ok != wantDeadlineOK || !gotDeadline.Equal(wantDeadline) { + t.Fatalf("replacement deadline = %v, %t; want %v, %t", gotDeadline, ok, wantDeadline, wantDeadlineOK) + } + + derived := context.WithValue(source, key, "derived") + derived = PreserveDeliveryContext(source, derived) + if !first.Owns(derived) || !second.Owns(derived) || derived.Value(key) != "derived" { + t.Fatal("source-derived replacement lost continuation authority or its replacement value") + } + releaseFirst() + releaseSecond() + if first.Owns(preserved) || second.Owns(preserved) { + t.Fatal("preserved continuation permits survived handler return") + } + if !replacementScope.Owns(preserved) { + t.Fatal("preserving source permits expired replacement-owned authority") + } + releaseReplacement() + if replacementScope.Owns(preserved) { + t.Fatal("replacement-owned permit survived its release") + } + cancelReplacement() + if !errors.Is(preserved.Err(), context.Canceled) { + t.Fatalf("preserved cancellation = %v, want replacement cancellation", preserved.Err()) + } + + plain := context.WithValue(context.Background(), key, "plain") + if got := PreserveDeliveryContext(context.Background(), plain); got != plain { + t.Fatal("permit-free source unnecessarily wrapped the replacement") + } + if got := PreserveDeliveryContext(nil, nil); got == nil || got.Err() != nil { + t.Fatalf("nil source and replacement produced invalid context: %v", got) + } + futureSource := WithDeliveryMetadata(context.Background(), DeliveryMetadata{ + SchemaVersion: DeliveryMetadataVersion + 1, + DispatchID: "future-source", + }) + trustedReplacement := WithDeliveryMetadata(context.Background(), DeliveryMetadata{ + SchemaVersion: DeliveryMetadataVersion, + DispatchID: "trusted-replacement", + }) + if metadata, ok := DeliveryMetadataFromContext(PreserveDeliveryContext(futureSource, trustedReplacement)); ok || metadata != (DeliveryMetadata{}) { + t.Fatalf("future source metadata became trusted replacement metadata: %+v, %t", metadata, ok) + } + permitOnly, releasePermitOnly := first.Permit(context.WithValue(context.Background(), key, "source-only")) + withoutReplacement := PreserveDeliveryContext(permitOnly, nil) + if !first.Owns(withoutReplacement) || withoutReplacement.Value(key) != nil { + t.Fatal("nil replacement did not preserve only runtime-owned source state") + } + releasePermitOnly() +} + +// TestDeliveryAttemptExhausted verifies MaxRetry counts retries after the initial attempt. +func TestDeliveryAttemptExhausted(t *testing.T) { + tests := []struct { + name string + attempt DeliveryAttempt + exhausted bool + }{ + {name: "initial with retries remaining", attempt: DeliveryAttempt{Number: 0, MaxRetry: 2}, exhausted: false}, + {name: "final configured retry", attempt: DeliveryAttempt{Number: 2, MaxRetry: 2}, exhausted: true}, + {name: "no retries configured", attempt: DeliveryAttempt{Number: 0, MaxRetry: 0}, exhausted: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := test.attempt.Exhausted(); got != test.exhausted { + t.Fatalf("Exhausted() = %t, want %t", got, test.exhausted) + } + }) + } +} + +// TestClassifyAttempt verifies application outcomes and infrastructure redelivery remain distinct. +func TestClassifyAttempt(t *testing.T) { + applicationErr := errors.New("application failed") + infrastructureErr := errors.New("store unavailable") + tests := []struct { + name string + attempt DeliveryAttempt + err error + want AttemptDecision + }{ + {name: "success", attempt: DeliveryAttempt{Number: 0, MaxRetry: 2}, want: AttemptSucceeded}, + {name: "retryable", attempt: DeliveryAttempt{Number: 0, MaxRetry: 2}, err: applicationErr, want: AttemptRetry}, + {name: "exhausted", attempt: DeliveryAttempt{Number: 2, MaxRetry: 2}, err: applicationErr, want: AttemptFailed}, + {name: "permanent", attempt: DeliveryAttempt{Number: 0, MaxRetry: 2}, err: Permanent(applicationErr), want: AttemptFailed}, + {name: "uncommitted", attempt: DeliveryAttempt{Number: 2, MaxRetry: 2}, err: Uncommitted(infrastructureErr), want: AttemptRedeliver}, + {name: "wrapped permanent", attempt: DeliveryAttempt{Number: 0, MaxRetry: 2}, err: fmt.Errorf("middleware: %w", Permanent(applicationErr)), want: AttemptFailed}, + {name: "wrapped uncommitted", attempt: DeliveryAttempt{Number: 2, MaxRetry: 2}, err: fmt.Errorf("workflow: %w", Uncommitted(infrastructureErr)), want: AttemptRedeliver}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := ClassifyAttempt(test.attempt, test.err); got != test.want { + t.Fatalf("ClassifyAttempt() = %v, want %v", got, test.want) + } + }) + } +} + +// TestAttemptErrorMarkers verifies markers are nil-safe, idempotent, and preserve error identity. +func TestAttemptErrorMarkers(t *testing.T) { + cause := errors.New("cause") + if Permanent(nil) != nil || Uncommitted(nil) != nil { + t.Fatal("nil marker input must stay nil") + } + + permanent := Permanent(cause) + if !IsPermanent(permanent) || IsUncommitted(permanent) || !errors.Is(permanent, cause) { + t.Fatalf("invalid permanent marker: %v", permanent) + } + if Permanent(permanent) != permanent { + t.Fatal("Permanent must be idempotent") + } + + uncommitted := Uncommitted(cause) + if !IsUncommitted(uncommitted) || IsPermanent(uncommitted) || !errors.Is(uncommitted, cause) { + t.Fatalf("invalid uncommitted marker: %v", uncommitted) + } + if Uncommitted(uncommitted) != uncommitted { + t.Fatal("Uncommitted must be idempotent") + } +} diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 0000000..9ed9b7e --- /dev/null +++ b/codecov.yml @@ -0,0 +1,18 @@ +codecov: + require_ci_to_pass: true + +# Go text profiles use repository module import paths. This deterministic fix +# maps every root, driver, and integration module record to its checked-in path. +fixes: + - "github.com/goforj/queue/::" + +coverage: + status: + project: + default: + target: auto + threshold: 1% + patch: + default: + target: auto + threshold: 1% diff --git a/database_config.go b/database_config.go index 3cf0a41..8fba03d 100644 --- a/database_config.go +++ b/database_config.go @@ -20,6 +20,7 @@ type DatabaseConfig struct { PollInterval time.Duration DefaultQueue string AutoMigrate bool + DisableAutoMigrate bool ProcessingRecoveryGrace time.Duration ProcessingLeaseNoTimeout time.Duration Observer Observer @@ -34,7 +35,9 @@ func (c DatabaseConfig) normalize() DatabaseConfig { if c.DefaultQueue == "" { c.DefaultQueue = "default" } - if !c.AutoMigrate { + if c.DisableAutoMigrate { + c.AutoMigrate = false + } else if !c.AutoMigrate { c.AutoMigrate = true } if c.ProcessingRecoveryGrace <= 0 { diff --git a/database_config_test.go b/database_config_test.go new file mode 100644 index 0000000..a46b5c4 --- /dev/null +++ b/database_config_test.go @@ -0,0 +1,53 @@ +package queue + +import ( + "testing" + "time" +) + +// TestDatabaseConfigNormalizeDefaultsAndAutoMigrationOptOut verifies the compatibility default and its explicit override. +func TestDatabaseConfigNormalizeDefaultsAndAutoMigrationOptOut(t *testing.T) { + defaults := (DatabaseConfig{}).normalize() + if defaults.Workers <= 0 { + t.Fatalf("default workers = %d, want a positive count", defaults.Workers) + } + if defaults.PollInterval != 50*time.Millisecond { + t.Fatalf("default poll interval = %v, want 50ms", defaults.PollInterval) + } + if defaults.DefaultQueue != "default" { + t.Fatalf("default queue = %q, want default", defaults.DefaultQueue) + } + if !defaults.AutoMigrate { + t.Fatal("AutoMigrate defaulted false, want compatibility default true") + } + if defaults.ProcessingRecoveryGrace != defaultProcessingRecoveryGrace { + t.Fatalf("default recovery grace = %v, want %v", defaults.ProcessingRecoveryGrace, defaultProcessingRecoveryGrace) + } + if defaults.ProcessingLeaseNoTimeout != defaultProcessingLeaseNoTimeout { + t.Fatalf("default no-timeout lease = %v, want %v", defaults.ProcessingLeaseNoTimeout, defaultProcessingLeaseNoTimeout) + } + + configured := (DatabaseConfig{ + Workers: 3, + PollInterval: time.Second, + DefaultQueue: "critical", + AutoMigrate: true, + DisableAutoMigrate: true, + ProcessingRecoveryGrace: 3 * time.Second, + ProcessingLeaseNoTimeout: 7 * time.Minute, + }).normalize() + if configured.Workers != 3 || configured.PollInterval != time.Second || configured.DefaultQueue != "critical" { + t.Fatalf("configured runtime values changed during normalization: %+v", configured) + } + if configured.AutoMigrate { + t.Fatal("DisableAutoMigrate did not override the compatibility default") + } + if configured.ProcessingRecoveryGrace != 3*time.Second || configured.ProcessingLeaseNoTimeout != 7*time.Minute { + t.Fatalf("configured recovery values changed during normalization: %+v", configured) + } + + enabled := (DatabaseConfig{AutoMigrate: true}).normalize() + if !enabled.AutoMigrate { + t.Fatal("explicit AutoMigrate true was not preserved") + } +} diff --git a/direct_delivery_test.go b/direct_delivery_test.go new file mode 100644 index 0000000..32097c9 --- /dev/null +++ b/direct_delivery_test.go @@ -0,0 +1,265 @@ +package queue + +import ( + "bytes" + "context" + "testing" + + "github.com/goforj/queue/busruntime" +) + +// TestQueueDirectDeliveryUsesApplicationTypeAndPayload pins the canonical +// physical boundary independently of synchronous handler execution. +func TestQueueDirectDeliveryUsesApplicationTypeAndPayload(t *testing.T) { + inner := &queueBackendRecorder{} + runtime := &nativeQueueRuntime{ + common: &queueCommon{ + inner: inner, + cfg: Config{DefaultQueue: "billing_default"}, + driver: DriverSync, + }, + runtime: &runtimeBackendStub{}, + nativeQueueRuntimeState: &nativeQueueRuntimeState{ + registered: make(map[string]Handler), + }, + } + q, err := newQueueFromRuntime(runtime) + if err != nil { + t.Fatalf("new queue: %v", err) + } + q.Register("reports:build", func(context.Context, Message) error { return nil }) + + payload := []byte{0, 1, 2, '{', 0xff} + result, err := q.Dispatch( + NewJob("reports:build"). + Payload(payload). + OnQueue("critical"), + ) + if err != nil { + t.Fatalf("dispatch: %v", err) + } + if len(inner.dispatched) != 1 { + t.Fatalf("physical dispatch count = %d, want 1", len(inner.dispatched)) + } + delivery := inner.dispatched[0] + if delivery.Type != "reports:build" { + t.Fatalf("physical type = %q, want reports:build", delivery.Type) + } + if !bytes.Equal(delivery.PayloadBytes(), payload) { + t.Fatalf("physical payload = %v, want %v", delivery.PayloadBytes(), payload) + } + metadata := DriverMetadata(delivery) + if metadata.SchemaVersion != DriverJobMetadataVersion || metadata.DispatchID != result.DispatchID || metadata.JobID == "" { + t.Fatalf("physical metadata = %+v, receipt = %+v", metadata, result) + } + if metadata.Queue != "critical" { + t.Fatalf("logical metadata queue = %q, want critical", metadata.Queue) + } + if got := DriverOptions(delivery).QueueName; got != "billing_critical" { + t.Fatalf("physical queue = %q, want billing_critical", got) + } + if _, ok := runtime.registered["reports:build"]; !ok { + t.Fatal("application type was not registered for direct delivery") + } + if _, ok := runtime.registered["bus:job"]; !ok { + t.Fatal("legacy direct-envelope handler was not retained") + } + + payload[0] = 99 + if bytes.Equal(delivery.PayloadBytes(), payload) { + t.Fatal("physical delivery retained the caller payload buffer") + } +} + +// TestQueueDirectDeliveryPreservesRawHandlerPayload proves the public message +// sees the canonical bytes even when they are not JSON and cannot be bound. +func TestQueueDirectDeliveryPreservesRawHandlerPayload(t *testing.T) { + q, err := NewSync() + if err != nil { + t.Fatalf("new sync queue: %v", err) + } + t.Cleanup(func() { + if shutdownErr := q.Shutdown(context.Background()); shutdownErr != nil { + t.Errorf("shutdown: %v", shutdownErr) + } + }) + + payload := []byte{0, 1, 2, 0xff} + var message Message + q.Register("reports:raw", func(_ context.Context, incoming Message) error { + message = incoming + return nil + }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + result, err := q.Dispatch(NewJob("reports:raw").Payload(payload)) + if err != nil { + t.Fatalf("dispatch raw payload: %v", err) + } + if !bytes.Equal(message.PayloadBytes(), payload) { + t.Fatalf("handler payload = %v, want %v", message.PayloadBytes(), payload) + } + if message.DispatchID != result.DispatchID || message.JobID == "" || message.JobType != "reports:raw" { + t.Fatalf("handler correlation = %+v, receipt = %+v", message, result) + } +} + +// TestQueueDirectDeliveryReservedTypesUseLegacyEnvelope prevents application +// names from replacing the frozen workflow protocol handlers. +func TestQueueDirectDeliveryReservedTypesUseLegacyEnvelope(t *testing.T) { + for _, jobType := range []string{"bus:job", "bus:chain:node", "bus:batch:job", "bus:callback"} { + t.Run(jobType, func(t *testing.T) { + inner := &queueBackendRecorder{} + runtime := &nativeQueueRuntime{ + common: &queueCommon{inner: inner, cfg: Config{DefaultQueue: "default"}, driver: DriverSync}, + runtime: &runtimeBackendStub{}, + nativeQueueRuntimeState: &nativeQueueRuntimeState{ + registered: make(map[string]Handler), + }, + } + q, err := newQueueFromRuntime(runtime) + if err != nil { + t.Fatalf("new queue: %v", err) + } + q.Register(jobType, func(context.Context, Message) error { return nil }) + + result, err := q.Dispatch(NewJob(jobType).Payload([]byte(`{"application":true}`))) + if err != nil { + t.Fatalf("dispatch reserved type: %v", err) + } + if len(inner.dispatched) != 1 { + t.Fatalf("physical dispatch count = %d, want 1", len(inner.dispatched)) + } + delivery := inner.dispatched[0] + if delivery.Type != "bus:job" { + t.Fatalf("physical type = %q, want retained bus:job", delivery.Type) + } + if DriverMetadata(delivery).SchemaVersion != 0 { + t.Fatalf("reserved delivery unexpectedly used direct metadata: %+v", DriverMetadata(delivery)) + } + observed := ResolveObservedJobMetadata(delivery.Type, delivery.PayloadBytes()) + if observed.DispatchID != result.DispatchID || observed.JobID == "" || observed.JobType != jobType { + t.Fatalf("legacy envelope metadata = %+v, receipt = %+v", observed, result) + } + }) + } +} + +// TestLegacyDirectEnvelopeOptionSupportsWorkerFirstRollout proves upgraded +// producers can keep emitting the frozen route until every worker is replaced. +func TestLegacyDirectEnvelopeOptionSupportsWorkerFirstRollout(t *testing.T) { + inner := &queueBackendRecorder{} + runtime := &nativeQueueRuntime{ + common: &queueCommon{inner: inner, cfg: Config{DefaultQueue: "default"}, driver: DriverSync}, + runtime: &runtimeBackendStub{}, + nativeQueueRuntimeState: &nativeQueueRuntimeState{ + registered: make(map[string]Handler), + }, + } + q, err := newQueueFromRuntime(runtime, WithLegacyDirectEnvelope()) + if err != nil { + t.Fatalf("new queue: %v", err) + } + q.Register("reports:legacy-rollout", func(context.Context, Message) error { return nil }) + + result, err := q.Dispatch(NewJob("reports:legacy-rollout").Payload([]byte(`{"id":1}`))) + if err != nil { + t.Fatalf("dispatch legacy rollout: %v", err) + } + if len(inner.dispatched) != 1 || inner.dispatched[0].Type != "bus:job" { + t.Fatalf("physical deliveries = %+v, want one bus:job", inner.dispatched) + } + if DriverMetadata(inner.dispatched[0]).SchemaVersion != 0 { + t.Fatalf("legacy delivery unexpectedly carried direct metadata: %+v", DriverMetadata(inner.dispatched[0])) + } + metadata := ResolveObservedJobMetadata(inner.dispatched[0].Type, inner.dispatched[0].PayloadBytes()) + if metadata.JobType != "reports:legacy-rollout" || metadata.DispatchID != result.DispatchID || metadata.JobID == "" { + t.Fatalf("legacy rollout metadata = %+v, receipt = %+v", metadata, result) + } +} + +// TestLegacyDirectEnvelopeExecutesOnUpgradedWorker proves an upgraded worker +// consumes old-envelope backlog through the same application handler and message. +func TestLegacyDirectEnvelopeExecutesOnUpgradedWorker(t *testing.T) { + q, err := NewSync(WithLegacyDirectEnvelope()) + if err != nil { + t.Fatalf("new legacy-emitting queue: %v", err) + } + t.Cleanup(func() { + if shutdownErr := q.Shutdown(context.Background()); shutdownErr != nil { + t.Errorf("shutdown: %v", shutdownErr) + } + }) + + received := make(chan Message, 1) + q.Register("reports:legacy-backlog", func(_ context.Context, message Message) error { + received <- message + return nil + }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start upgraded worker: %v", err) + } + result, err := q.Dispatch(NewJob("reports:legacy-backlog").Payload([]byte(`{"id":7}`))) + if err != nil { + t.Fatalf("dispatch legacy backlog job: %v", err) + } + message := <-received + if message.DispatchID != result.DispatchID || message.JobID == "" || message.JobType != "reports:legacy-backlog" { + t.Fatalf("legacy backlog message = %+v, receipt = %+v", message, result) + } + if !bytes.Equal(message.PayloadBytes(), []byte(`{"id":7}`)) { + t.Fatalf("legacy backlog payload = %q", message.PayloadBytes()) + } +} + +// TestNestedLegacyDeliveryShadowsParentMetadata prevents an inline reserved +// delivery from inheriting the direct job correlation that dispatched it. +func TestNestedLegacyDeliveryShadowsParentMetadata(t *testing.T) { + q, err := NewSync() + if err != nil { + t.Fatalf("new sync queue: %v", err) + } + t.Cleanup(func() { + if shutdownErr := q.Shutdown(context.Background()); shutdownErr != nil { + t.Errorf("shutdown: %v", shutdownErr) + } + }) + + metadataSeen := make(chan busruntime.DeliveryMetadata, 1) + q.Register("bus:callback", func(ctx context.Context, _ Message) error { + metadata, _ := busruntime.DeliveryMetadataFromContext(ctx) + metadataSeen <- metadata + return nil + }) + q.Register("reports:parent", func(ctx context.Context, _ Message) error { + _, dispatchErr := q.WithContext(ctx).Dispatch(NewJob("bus:callback").Payload([]byte(`{"nested":true}`))) + return dispatchErr + }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + if _, err := q.Dispatch(NewJob("reports:parent")); err != nil { + t.Fatalf("dispatch parent: %v", err) + } + if metadata := <-metadataSeen; metadata != (busruntime.DeliveryMetadata{}) { + t.Fatalf("nested legacy delivery inherited parent metadata: %+v", metadata) + } +} + +// TestDriverJobMetadataRejectsUnknownVersions ensures future transport metadata +// cannot spoof correlation on a worker that does not understand its semantics. +func TestDriverJobMetadataRejectsUnknownVersions(t *testing.T) { + job := DriverWithMetadata(NewJob("reports:build"), DriverJobMetadata{ + SchemaVersion: DriverJobMetadataVersion + 1, + DispatchID: "spoofed", + JobID: "spoofed", + }) + if metadata := DriverMetadata(job); metadata != (DriverJobMetadata{}) { + t.Fatalf("unknown metadata = %+v, want zero value", metadata) + } + observed := ResolveObservedJobMetadataFromJob(job) + if observed.DispatchID != "" || observed.JobID != "" || observed.JobType != "reports:build" { + t.Fatalf("unknown metadata affected observation: %+v", observed) + } +} diff --git a/dispatch_acceptance.go b/dispatch_acceptance.go new file mode 100644 index 0000000..cc987d4 --- /dev/null +++ b/dispatch_acceptance.go @@ -0,0 +1,107 @@ +package queue + +import ( + "context" + "sync" +) + +type dispatchAcceptanceContextKey struct{} + +type dispatchAcceptance struct { + mu sync.Mutex + accepted bool + callbacks []func() +} + +// ensureDispatchAcceptance shares one acceptance boundary across observation, delivery, and orchestration adapters. +func ensureDispatchAcceptance(ctx context.Context) (context.Context, *dispatchAcceptance) { + if ctx == nil { + ctx = context.Background() + } + if current := dispatchAcceptanceFromContext(ctx); current != nil { + return ctx, current + } + acceptance := &dispatchAcceptance{} + return context.WithValue(ctx, dispatchAcceptanceContextKey{}, acceptance), acceptance +} + +// newDispatchAcceptance starts an independent boundary so nested workflow dispatches cannot reuse their parent's accepted state. +func newDispatchAcceptance(ctx context.Context) (context.Context, *dispatchAcceptance) { + if ctx == nil { + ctx = context.Background() + } + acceptance := &dispatchAcceptance{} + return context.WithValue(ctx, dispatchAcceptanceContextKey{}, acceptance), acceptance +} + +// dispatchAcceptanceFromContext returns the current dispatch boundary when one has been installed. +func dispatchAcceptanceFromContext(ctx context.Context) *dispatchAcceptance { + if ctx == nil { + return nil + } + acceptance, _ := ctx.Value(dispatchAcceptanceContextKey{}).(*dispatchAcceptance) + return acceptance +} + +// onAccepted registers work that must occur after acceptance and before an inline delivery may begin. +func (a *dispatchAcceptance) onAccepted(callback func()) { + if a == nil || callback == nil { + return + } + a.mu.Lock() + if !a.accepted { + a.callbacks = append(a.callbacks, callback) + a.mu.Unlock() + return + } + a.mu.Unlock() + callback() +} + +// markAccepted commits the dispatch boundary exactly once before releasing inline delivery gates. +func (a *dispatchAcceptance) markAccepted() { + if a == nil { + return + } + a.mu.Lock() + if a.accepted { + a.mu.Unlock() + return + } + a.accepted = true + callbacks := append([]func(){}, a.callbacks...) + a.callbacks = nil + a.mu.Unlock() + for _, callback := range callbacks { + callback() + } +} + +// isAccepted reports whether the delivery backend crossed its acceptance boundary. +func (a *dispatchAcceptance) isAccepted() bool { + if a == nil { + return false + } + a.mu.Lock() + defer a.mu.Unlock() + return a.accepted +} + +type acceptedExecutionError struct { + cause error +} + +// Error preserves the application execution error text returned by synchronous dispatch. +func (e acceptedExecutionError) Error() string { + return e.cause.Error() +} + +// Unwrap preserves errors.Is and errors.As behavior for the application execution failure. +func (e acceptedExecutionError) Unwrap() error { + return e.cause +} + +// DispatchAccepted reports that enqueue acceptance preceded the synchronous execution failure. +func (e acceptedExecutionError) DispatchAccepted() bool { + return true +} diff --git a/dispatch_acceptance_edge_test.go b/dispatch_acceptance_edge_test.go new file mode 100644 index 0000000..49deed4 --- /dev/null +++ b/dispatch_acceptance_edge_test.go @@ -0,0 +1,51 @@ +package queue + +import ( + "errors" + "testing" +) + +// TestDispatchAcceptanceNilEdges verifies optional context and receiver boundaries remain safe. +func TestDispatchAcceptanceNilEdges(t *testing.T) { + ctx, acceptance := ensureDispatchAcceptance(nil) + if ctx == nil || acceptance == nil { + t.Fatalf("ensureDispatchAcceptance(nil) = %v, %p; want initialized values", ctx, acceptance) + } + if resolved := dispatchAcceptanceFromContext(ctx); resolved != acceptance { + t.Fatalf("resolved acceptance = %p, want %p", resolved, acceptance) + } + if resolved := dispatchAcceptanceFromContext(nil); resolved != nil { + t.Fatalf("nil context resolved acceptance %p", resolved) + } + + callbackCalled := false + acceptance.onAccepted(nil) + var absent *dispatchAcceptance + absent.onAccepted(func() { callbackCalled = true }) + absent.markAccepted() + if absent.isAccepted() { + t.Fatal("nil acceptance reported accepted") + } + if callbackCalled { + t.Fatal("nil acceptance invoked its callback") + } +} + +// TestAcceptedExecutionErrorPreservesCauseAndAcceptance verifies synchronous failures retain both error and settlement semantics. +func TestAcceptedExecutionErrorPreservesCauseAndAcceptance(t *testing.T) { + cause := errors.New("handler failed") + err := acceptedExecutionError{cause: cause} + if err.Error() != cause.Error() { + t.Fatalf("error text = %q, want %q", err.Error(), cause.Error()) + } + if !errors.Is(err, cause) { + t.Fatalf("errors.Is(%v, %v) = false", err, cause) + } + if !err.DispatchAccepted() { + t.Fatal("accepted execution error did not report dispatch acceptance") + } + + var _ interface { + DispatchAccepted() bool + } = err +} diff --git a/dispatch_acceptance_test.go b/dispatch_acceptance_test.go new file mode 100644 index 0000000..79623ca --- /dev/null +++ b/dispatch_acceptance_test.go @@ -0,0 +1,39 @@ +package queue + +import ( + "context" + "testing" +) + +// TestDispatchAcceptanceSeparatesNestedDispatches verifies child workflows cannot inherit an accepted parent boundary. +func TestDispatchAcceptanceSeparatesNestedDispatches(t *testing.T) { + parentCtx, parent := newDispatchAcceptance(context.Background()) + parent.markAccepted() + + childCtx, child := newDispatchAcceptance(parentCtx) + if child == parent || child.isAccepted() { + t.Fatalf("new child boundary reused accepted parent: parent=%p child=%p", parent, child) + } + resolvedCtx, resolved := ensureDispatchAcceptance(childCtx) + if resolvedCtx != childCtx || resolved != child { + t.Fatal("observer adapter did not reuse the current child boundary") + } + + child.markAccepted() + if !parent.isAccepted() || !child.isAccepted() { + t.Fatal("accepting child changed either boundary unexpectedly") + } +} + +// TestDispatchAcceptanceCallbacksRunOnce verifies multiple marks cannot duplicate enqueue facts. +func TestDispatchAcceptanceCallbacksRunOnce(t *testing.T) { + _, acceptance := newDispatchAcceptance(nil) + calls := 0 + acceptance.onAccepted(func() { calls++ }) + acceptance.markAccepted() + acceptance.markAccepted() + acceptance.onAccepted(func() { calls++ }) + if calls != 2 { + t.Fatalf("callback calls = %d, want one registered-before and one registered-after call", calls) + } +} diff --git a/dispatch_order_contract_test.go b/dispatch_order_contract_test.go new file mode 100644 index 0000000..ae25d9c --- /dev/null +++ b/dispatch_order_contract_test.go @@ -0,0 +1,81 @@ +package queue_test + +import ( + "context" + "errors" + "testing" + + "github.com/goforj/queue" +) + +// TestSyncDispatchObservationOrder verifies acceptance precedes inline execution on success. +func TestSyncDispatchObservationOrder(t *testing.T) { + recorder := &retryEventRecorder{} + q, err := queue.NewSync(queue.WithObserver(recorder)) + if err != nil { + t.Fatalf("new sync queue: %v", err) + } + q.Register("contract:order:success", func(context.Context, queue.Message) error { return nil }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + t.Cleanup(func() { _ = q.Shutdown(context.Background()) }) + + if _, err := q.Dispatch(queue.NewJob("contract:order:success")); err != nil { + t.Fatalf("dispatch: %v", err) + } + assertEventKinds(t, recorder.snapshot(), []queue.EventKind{ + queue.EventDispatchStarted, + queue.EventEnqueueAccepted, + queue.EventProcessStarted, + queue.EventJobStarted, + queue.EventJobSucceeded, + queue.EventProcessSucceeded, + queue.EventDispatchSucceeded, + }) +} + +// TestSyncExecutionFailureRemainsAccepted verifies business failure is not reported as enqueue rejection. +func TestSyncExecutionFailureRemainsAccepted(t *testing.T) { + recorder := &retryEventRecorder{} + q, err := queue.NewSync(queue.WithObserver(recorder)) + if err != nil { + t.Fatalf("new sync queue: %v", err) + } + wantErr := errors.New("business failure") + q.Register("contract:order:failure", func(context.Context, queue.Message) error { return wantErr }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + t.Cleanup(func() { _ = q.Shutdown(context.Background()) }) + + if _, err := q.Dispatch(queue.NewJob("contract:order:failure")); !errors.Is(err, wantErr) { + t.Fatalf("dispatch error = %v, want business failure", err) + } + assertEventKinds(t, recorder.snapshot(), []queue.EventKind{ + queue.EventDispatchStarted, + queue.EventEnqueueAccepted, + queue.EventProcessStarted, + queue.EventJobStarted, + queue.EventJobFailed, + queue.EventProcessFailed, + queue.EventDispatchSucceeded, + }) +} + +// assertEventKinds compares exact synchronous causality without coupling assertions to timestamps or IDs. +func assertEventKinds(t *testing.T, events []queue.Event, want []queue.EventKind) { + t.Helper() + got := make([]queue.EventKind, len(events)) + for index, event := range events { + got[index] = event.Kind + } + if len(got) != len(want) { + t.Fatalf("event kinds = %v, want %v", got, want) + } + for index := range want { + if got[index] != want[index] { + t.Fatalf("event kinds = %v, want %v", got, want) + } + } +} diff --git a/docs/backend-guarantees.md b/docs/backend-guarantees.md index 06f5f36..36d2d9a 100644 --- a/docs/backend-guarantees.md +++ b/docs/backend-guarantees.md @@ -2,21 +2,21 @@ This document defines the expected semantics and tested capability differences by backend. -All backends are expected to provide at-least-once delivery semantics. Handlers should be idempotent. +There is no blanket delivery guarantee shared by every backend. Handlers should be idempotent because durable backends may redeliver, while ephemeral backends may lose work at failure boundaries described below. ## Semantics Baseline (All Backends) -- Delivery: at-least-once +- Acceptance and delivery durability are backend-specific. - Duplicate processing: possible; handlers must be idempotent - Ordering: not guaranteed unless explicitly documented for a backend/runtime path -- Failure behavior: dispatch/processing errors should be surfaced; no silent loss is acceptable +- A successful dispatch means the backend-specific acceptance boundary was crossed; it does not imply handler success. +- Unsupported or unproven guarantees must remain explicit rather than being inferred from another backend's tests. ## Capability Matrix (Integration Fixture-Aligned) -The table below reflects the capability gates used in `integration/all/integration_scenarios_test.go` and the shared scenario suite. -Every capability/guarantee row should be justifiable by a concrete scenario or test. If a cell changes, update the linked scenario references in the same PR. +The table below reflects capability gates used by `integration/all/integration_scenarios_test.go`. A `Yes` means that fixture currently runs the linked scenario; it is evidence under that fixture's conditions, not by itself a production guarantee across crashes, producer/worker separation, or multiple processes. -| Backend | Backoff | Restart Recovery | Delayed/Retry Restart Durability | Poison Retry | Dispatch Context Cancel | Deterministic No-Dupes (suite) | Ordering Contract (suite) | Broker Fault Scenarios | Shutdown During Delay/Retry | +| Backend | Backoff | Restart Scenario | Delayed/Retry Restart Scenario | Poison Retry | Dispatch Context Cancel | Fixture No-Dupes | Ordering Scenario | Broker Fault Scenarios | Shutdown During Delay/Retry | | --- | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | :---: | | `redis` | No | Yes | Yes | No | No | Yes | Yes | Yes | Yes | | `mysql` | Yes | Yes | Yes | Yes | Yes | Yes | No | No | Yes | @@ -38,9 +38,9 @@ These are the primary shared-scenario proofs for the matrix above. | Delayed/retry restart durability | `scenario_worker_restart_delay_recovery`; `scenario_shutdown_during_delay_retry` | | Poison retry semantics | `scenario_poison_message_max_retry` | | Dispatch context cancellation | `scenario_dispatch_context_cancellation` parent with `scenario_dispatch_context_precanceled`, `scenario_dispatch_context_deadline_exceeded`, and `scenario_dispatch_context_followup_health` | -| Deterministic no-duplicate processing (suite-level capability) | `scenario_multi_worker_contention`; `scenario_duplicate_delivery_idempotency` | +| No-duplicate processing under fixture conditions | `scenario_multi_worker_contention`; `scenario_duplicate_delivery_idempotency`. Public logical `Job.UniqueFor` behavior is separately exercised through `TestIntegrationQueue_AllBackends` in `integration/all/runtime_integration_test.go`. | | Ordering contract (suite-level capability) | `scenario_ordering_contract` parent with `scenario_ordering_single_worker_fifo`; concurrent non-guarantee covered by `scenario_ordering_multi_worker_best_effort`; reordering behavior probed by `scenario_ordering_delayed_immediate_mix` and `scenario_ordering_retry_reorder_allowed` | -| Broker fault handling | `scenario_dispatch_during_broker_fault`; `scenario_consume_after_broker_recovery` | +| Broker fault handling | `scenario_dispatch_during_broker_fault`; `scenario_consume_after_broker_recovery`; Redis lost-ack recovery in `TestIntegrationChaos_RedisBrokerDisconnectRedelivery` | | Shutdown during delay/retry workloads | `scenario_shutdown_during_delay_retry` | | Pause/Resume capability behavior | `TestObservabilityIntegration_PauseResumeSupport_AllBackends` (`integration/root/observability_integration_test.go`) | | Native stats capability behavior | `TestObservabilityIntegration_AllBackends` (`integration/root/observability_integration_test.go`) | @@ -48,14 +48,51 @@ These are the primary shared-scenario proofs for the matrix above. | Workflow failure callback semantics (catch/finally + terminal state) | `TestIntegrationBus_AllBackends` -> `workflow_chain_failure_callbacks` and `workflow_batch_failure_callbacks` (`integration/bus/integration_test.go`) | | Workflow duplicate callback suppression (SQL runtime/store path) | `TestSQLStore_RuntimeChainFinallyDuplicateCallbackSuppressed`; `TestSQLStore_RuntimeChainCatchAndFinallyDuplicateCallbacksSuppressed`; `TestSQLStore_RuntimeChainFinallyCallbackReplayAfterDispatchFaultSuppressed`; existing batch duplicate callback suppression tests (`integration/bus/callback_sql_integration_test.go`) | | Workflow dispatch failure state consistency (SQL runtime/store path) | `TestSQLStore_RuntimeChainInitialDispatchFailureStateConsistent`; `TestSQLStore_RuntimeBatchPartialDispatchFailureStateConsistent` (`integration/bus/dispatch_failure_sql_integration_test.go`) | +| Externally managed queue-schema readiness | `TestDatabaseManagedSchemaValidationRejectsIncompleteBackends`, `TestDatabaseManagedSchemaRejectsNonBaseRelations`, and `TestDatabaseManagedSchemaPreflightAndStartupRetry` (`driver/sqlqueuecore/queue_database_failure_test.go`) cover missing operational tables, view-backed relations, and runtime-used columns without DDL. `TestDatabaseIntegration_SQLite/sqlite_managed_schema_fails_closed_then_recovers_after_provisioning` proves an empty schema fails both readiness and startup without mutation, then the same runtime succeeds after external provisioning. The `sqlite_managed_schema_dispatch_and_process`, `mysql_managed_schema_dispatch_and_process`, and `postgres_managed_schema_dispatch_and_process` scenarios prove canonical preprovisioned schemas support readiness, uniqueness, dispatch, and consumption on every SQL dialect (`integration/root/database_integration_test.go`). | +| Atomic workflow-store transitions across SQL dialects | `TestWorkflowStoreIntegration_SQLite`; `TestWorkflowStoreIntegration_MySQL`; `TestWorkflowStoreIntegration_MySQLAutoMigratesMissingReceiptAtLegacyWidths`; `TestWorkflowStoreIntegration_MySQLManagedWideKeys`; `TestWorkflowStoreIntegration_MySQLRejectsNonVARBINARYKeys`; `TestWorkflowStoreIntegration_Postgres` (`integration/root/workflow_store_integration_test.go`), plus shared memory/SQLite concurrency contracts (`internal/workflow/store_contract_test.go`). The MySQL upgrade fixture drops only the receipt table beside widened legacy state, verifies ordinary startup derives 512-byte receipt identities, and exercises identities above the fresh-schema defaults. The managed fixture separately covers a complete pre-existing wider schema. | +| Current-generation retention after a committed workflow transition | `TestDeliveryApplicationStateCommittedSignal` (`busruntime/delivery_settlement_test.go`), `TestDatabasePendingRecoveryTokenPreservesPendingRecovery` (`driver/sqlqueuecore/queue_database_impl_test.go`), and `TestChainPostTransitionFailureMarksCurrentGenerationForRecovery` (`internal/workflow/workflow_mutation_retry_test.go`) prove the direct signal, SQL token selection, and post-transition workflow handoff. The end-to-end receipt recovery row below covers all three SQL dialects. | +| Versioned transition-receipt interpretation | `TestTransitionReceiptUnknownVersionsFailClosed` and `TestUnknownTransitionReceiptVersionsBlockRecoveredApplicationExecution` (`internal/workflow`) prove unknown `receipt_version` or `event_schema_version` values return an uncommitted recovery error: the worker neither acknowledges the row, runs application code, marks application state committed, nor publishes reconstructed facts. The observer event schema is shared across queue, worker, and workflow layers and is versioned independently from the workflow-envelope protocol. The real SQL recovery scenarios below exercise the supported version columns through each dialect's physical schema and read/write path. | +| Receipt-backed workflow facts after definite SQL finalization failure | `TestDatabaseIntegration_SQLite/sqlite_workflow_success_facts_recover_after_finalization_failure` covers a terminal chain, a predecessor whose successor already completed, a batch member, and one-member aggregate completion on a real stale SQLite row. `TestRecoveredTransitionReceiptLogicalValidationSeparatesPhysicalOwnership` separates valid logical receipt proof from the exact physical fact owner. `TestChainSuccessorRejectionRecoversWithoutPredecessorReplay` proves exact-owner recovery repeatedly re-dispatches the immediate successor after definite enqueue rejection without re-running the predecessor. `TestChainRecoveryWithoutExactReceiptOwnershipPreservesOnlyLiveContinuation` covers a missing receipt, a decorated store without receipt capability, and different/legacy generation provenance. `TestChainSuccessRecoveryAllowsDifferentPhysicalDeliveryIdentity` adds valid duplicate chain deliveries with a different attempt or physical `JobID`: only the live immediate successor is dispatched, while the predecessor handler, facts, and callbacks are not replayed. Progressed or terminal state is a no-op, and a rejected replacement enqueue remains uncommitted for retry. `TestChainSuccessRecoveryRejectsInvalidReceiptShapeBeforeLiveness` proves a supported success receipt must reject cancellation and make completion exactly match final-node position before either facts or continuation dispatch. `TestBatchRecoverySettlesNonFactOwnersWithoutFacts` proves successful batch duplicates settle silently and failed duplicates return a generic permanent cause across different attempts, different recovered generations, and legacy provenance, without handlers, callbacks, or facts. Dispatch remains intentionally at-least-once because recovery cannot distinguish a missing successor from one queued but not yet progressed. `sqlite_terminal_batch_completion_recovers_from_completing_member` proves a two-member batch reconstructs `BatchCompleted` only for the recovered member whose receipt owns the aggregate transition. `sqlite_failed_batch_recovery_archives_without_reexecution` proves a failed receipt reaches SQL's `dead` archive without fabricating the unpersisted application cause or executing either handler again. `sqlite_later_workflow_attempt_wins_then_recovers_without_reexecution` proves that application retry does not inherit the earlier generation. `mysql_workflow_receipt_recovery` and `postgres_workflow_receipt_recovery` prove terminal-chain state/fact recovery through each server dialect (`integration/root/database_integration_test.go`). This does not cover other settlement owners, callbacks, durable continuation intent, or no-redelivery publication. | +| Receipt-backed terminal chain failure | `TestChainCommittedFailureRecoveryPreservesOneApplicationOccurrence` proves exact, different, and legacy recovered-generation provenance all return the first persisted permanent chain cause without another handler, Catch/Finally callback, or logical failure fact. `TestChainFailureRecoveryAllowsDifferentPhysicalDeliveryIdentity` extends the same archive classification to valid duplicates with a different attempt or physical `JobID`. `TestChainFailureRecoveryRejectsInvalidReceiptIdentity`, the transition-claim/incarnation contracts, and `TestTransitionReceiptAmbiguousCommitReadback/chain_failure` cover malformed logical identity, terminal shape, and commit readback; `TestChainFailureRecoveryWithoutPersistedCauseUsesTerminalDiagnostic` pins the empty-cause permanent diagnostic. `TestChainLegacyFailureRecoveryRetainsCurrentFailureClassification` records the weaker receipt-absent fallback, which reruns application code once for physical classification while suppressing duplicate facts/callbacks. `TestStoreContract_FailChainPreservesReceiptBackedCause` covers memory and SQLite first-cause preservation, and `TestSQLStoreChainFailureReceiptRollsBackParent` proves receipt-insert failure cannot commit parent failure without provenance. `TestDatabaseIntegration_SQLite/sqlite_failed_chain_recovery_archives_without_reexecution` then forces repeated real archive failure, preserves receipt owner, attempt zero, and cause, avoids duplicate application/workflow occurrences, and eventually reaches `dead` at attempt one with that persisted cause. Server-dialect failed-chain finalization fixtures remain open. | +| Repeated receipt-owner recovery and aggregate terminal ownership | `sqlite_workflow_receipt_owner_survives_repeated_finalization_failure`, `sqlite_failed_chain_recovery_archives_without_reexecution`, and the failed-batch recovery scenario each force at least two recovery finalization failures before a later settlement succeeds. The SQL driver performs a fenced, same-attempt, best-effort repair that restores inherited receipt lineage and adds a bounded delay; direct driver tests cover malformed, stale, and inapplicable repair branches. `mysql_concurrent_batch_receipt_owner` and `postgres_concurrent_batch_receipt_owner` race twelve fail-fast members through separate workers and prove the serialized parent transition produces twelve member receipts, one aggregate-owner receipt, and one terminal failed/cancelled fact pair. `TestSQLStoreBatchAggregateOwnershipMismatchFailsClosed` rejects an aggregate row with missing completion, success-owned cancellation, a missing claimed member, or owner/outcome disagreement with that member's receipt; `TestSQLStoreBatchAggregateIncarnationMismatchFailsClosed` rejects a stale aggregate incarnation. `TestBatchRecoveryRejectsInvalidAggregateReceiptShape` proves cancellation, completion, and live aggregate state must agree before recovery can acknowledge the delivery or produce effects. | \* `sqlite` is promoted to full restart/durability coverage in the shared suite when queue and worker use the same test-local DSN (see fixture override logic in `integration/all/integration_scenarios_test.go`). +## `UniqueFor` Identity and Scope + +Every driver receives one versioned logical identity composed from: + +- the effective physical queue name; +- the logical application job type; and +- the canonical serialized logical payload bytes. + +Absent payloads, zero-byte payloads, and the exact JSON `null` payload share one canonical absence identity so removing the legacy workflow envelope cannot change a job's key. All other payload bytes remain exact. Generated dispatch, job, chain, and batch IDs are correlation metadata and do not affect duplicate suppression. Retry, delay, timeout, and backoff options are also excluded. This lets two independently constructed workflow envelopes suppress the same logical work without conflating observability IDs with delivery policy. + +| Backend | Claim Scope | Failure Boundary | +| --- | --- | --- | +| `null`, `sync`, `workerpool` | One queue runtime instance | Claims live in memory and disappear when that runtime exits. Known pre-acceptance failures release their claim. | +| `mysql`, `postgres`, `sqlite` | All producers sharing the database | The uniqueness claims and queue row commit in one database transaction. | +| `redis` | All producers sharing Redis | TTL must be at least one second. A token-owned Redis claim is released after a definite Asynq physical duplicate. Other enqueue errors are ambiguous because Redis may have committed before the response was lost, so the claim remains until TTL to fail closed against duplicate retries. A producer crash before Asynq acceptance can therefore suppress work until the TTL expires. | +| `nats`, `sqs`, `rabbitmq` | One queue runtime instance | Claims live in memory and disappear when that runtime exits. Failures known to occur before publication release their claim; ambiguous server roundtrip, send, or confirmation failures retain it until TTL to avoid admitting an immediate duplicate. | + +Claims use a fixed acquisition-time TTL; successful processing does not shorten the window. During the SQL producer transition, each accepted unique dispatch atomically stores both the exact historical physical key and the canonical logical key with its queue row. An outstanding lock under either format rejects the transaction, and normal expiry pruning removes both formats. This makes mixed versions contend only when their physical queue, type, and payload bytes match exactly. The historical delimiter format can conflate distinct canonical jobs, such as type `a` with payload `b:c` and type `a:b` with payload `c` on the same queue, so those conservative false collisions remain until historical claiming is retired. + +Older public workflow producers embedded fresh correlation IDs in their physical envelopes. Independently dispatched equivalent work therefore does not share a historical key across versions. Stop those producers and wait for their longest live `UniqueFor` duration before starting new producers, or explicitly accept a transient logical duplicate window. Stable low-level SQL producers with unchanged physical identities may overlap. Redis separately retains Asynq's physical claim alongside the canonical claim for direct-job compatibility. + ## Backend Notes +### Local runtimes (`DriverSync`, `DriverWorkerpool`) + +- Sync executes work inline. Workerpool executes with the concurrency configured by `WithWorkers`; when no explicit buffer is supplied, its queue capacity scales with that worker count. +- Once shutdown begins, new public work is rejected. Workflow continuations are admitted only through a runtime-scoped permit held by an active handler; that permit expires when the handler returns and cannot authorize another runtime. +- Accepted delayed work and callback descendants remain part of the drain. A shutdown deadline bounds the current attempt; if it expires, shutdown returns the context error and a later call can continue cleanup. + ### Redis (`DriverRedis`) - Uses Asynq-backed runtime semantics. +- An accepted Asynq task is persisted in Redis, but handlers must tolerate redelivery around worker/process failures. +- Asynq v0.26 archives an exhausted task before consulting its non-failure predicate. New tasks with an explicit retry budget carry one header-marked transport reserve: workers expose the original application budget, explicitly archive terminal application errors, and reuse the reserve for uncommitted workflow or lease-recovery redelivery. Deploy workers before producers; already-queued legacy tasks remain subject to the upstream final-attempt behavior. +- Public workflow dispatch now preserves its zero retry policy instead of allowing Asynq to substitute its default of 25 retries. Applications that intentionally relied on that old fallback must set `.Retry(25)` explicitly. This is a runtime-behavior migration, separate from the worker-first transport-reserve rollout above. - Shared suite treats custom backoff as unsupported in this runtime path (`supportsBackoff=false`). - Ordering contract is tested in-suite (`supportsOrderingContract=true`) under the current shared scenario's constrained FIFO assumptions. - Do not generalize this to multi-worker, retry, or delayed/immediate mixed workloads unless explicitly documented and tested. @@ -63,6 +100,15 @@ These are the primary shared-scenario proofs for the matrix above. ### Database (`DriverDatabase`: MySQL/Postgres/SQLite) +- Acceptance is a committed queue-row insert. Claimed rows use stale-processing recovery, so application handlers must tolerate redelivery if finalization does not commit. +- Positive process and workflow facts wait for fenced row finalization matching the exact processing claim. Every claim receives an opaque generation ID. When a row remains pending for same-attempt infrastructure redelivery, SQL normally retains any inherited unsettled-generation ID. If the current generation durably writes a workflow transition receipt before later infrastructure work requests that redelivery, the workflow engine marks application state committed and SQL retains the current generation instead. The next claim can therefore match the actual receipt owner rather than an older recovered generation. This signal does not mean queue settlement or observer delivery succeeded. An application retry increments the attempt and clears every prior-generation link. Recovery flags, queue state, and error text are not workflow-transition proof by themselves. +- On the direct built-in SQL workflow-store path, a chain-node or batch-member outcome mutation writes an immutable transition receipt in the same database transaction. The receipt persists `receipt_version` for its ownership format and `event_schema_version` for the shared observer fact contract; both are currently `1`, and the event version is independent from the workflow-envelope protocol version. An unsupported value fails recovery closed with an uncommitted error rather than being collapsed into an absent receipt. Logical receipt proof requires supported versions; a persisted owner with nonempty generation, dispatch, physical job ID, and fingerprint plus a nonnegative owner attempt; matching workflow kind/ID/member/incarnation and logical dispatch; and the same immutable job fingerprint. The current delivery must carry nonempty dispatch and `JobID`, but its attempt is physical provenance and may differ from the owner or be negative. For a chain, the physical `JobID` may also differ from the receipt owner; for a batch, `JobID` is the logical member key and must match. This logical proof suppresses duplicate handler execution. Reconstructing member or aggregate facts is stricter: `RecoveredGenerationID`, current attempt, and physical `JobID` must all equal the receipt owner's tuple. A physical nonowner publishes no reconstructed facts. A failed chain receipt still returns the first persisted terminal cause as permanent, and a failed batch member still returns a generic permanent cause because its original cause is intentionally not persisted. Both failure paths drive SQL toward its `dead` archive without emitting replacement failure/member facts; built-in `FailChain` cannot replace the authoritative first chain cause. A SQL aggregate receipt must own completion; cancellation must own a failed outcome; and when it names the requested logical member, its workflow incarnation, complete physical owner, and outcome must match that member's receipt. Runtime recovery also requires its completion and cancellation flags to agree with live terminal state. Any contradiction fails uncommitted before acknowledgement or effects. `BatchCompleted` is reconstructed only when the exact physical fact owner also owns that validated aggregate transition. Server-dialect parent-row serialization ensures exactly one of concurrent distinct members owns that transition, with real twelve-worker MySQL and PostgreSQL race evidence. Built-in memory receipts are process-local. Receipt-absent legacy rows and custom/decorated/raw stores retain weaker fact and terminal-failure recovery guarantees; state-confirmed predecessor success still suppresses its handler and preserves the live continuation as described below. +- Any recovered predecessor whose validated durable state proves success re-dispatches the immediate successor while the chain is active and `NextIndex` still points to it, without re-running the predecessor. Exact physical fact ownership additionally reconstructs predecessor facts; a missing receipt, a store without receipt capability, or a logically valid receipt with different/legacy generation, different attempt, or different physical `JobID` dispatches only the successor. Progressed and terminal chains are no-ops. This closes definite enqueue rejection and the legacy/custom liveness gap but remains at-least-once: an already-enqueued successor that has not progressed is indistinguishable from a missing one, so duplicate enqueue remains possible. A transition receipt is not a durable continuation intent, settlement outbox, or observer outbox. It does not record callback delivery, `Progress` closures, batch fan-out, or whether a successor enqueue committed. If queue finalization commits and the process exits before deferred observer calls, no queue row remains to trigger recovery. Durable publication and continuation still require the open outbox/intent work. +- After a recovered SQL delivery exhausts its finalization retries without committing new application state, the driver makes one fenced best-effort repair: it returns that exact claim to `pending` on the same attempt, clears its processing timestamp, restores the inherited receipt-owner generation, and delays reclaim by at least the finalization-retry floor. Real SQLite scenarios prove multiple successive success, failed-chain, and failed-batch finalization failures retain lineage until a later delete or archive succeeds. If the repair itself cannot be proven or committed, `settlement_failed` reports the joined error and no stronger lineage guarantee is claimed. +- Startup migrations remain enabled by default. Set `DisableAutoMigrate: true` when an external deployment process owns schema changes; readiness and startup then perform no queue DDL and fail closed unless both `queue_jobs` and `queue_unique_locks` are base-table relations, including PostgreSQL partitioned tables, containing every column used by the current runtime. Views do not satisfy the managed-runtime contract. A failed managed-schema check does not consume startup, so the same runtime can become ready and start after deployment tooling provisions the complete schema. This is a presence gate, not certification of write permissions, exact SQL types, constraints, or performance indexes; externally managed deployments must still apply the canonical dialect schema. An automatic migration failure is also retryable after its lock, permission, or connectivity problem is corrected. +- Processing fencing adds one nullable `processing_token` column, so existing rows and producer-only old binaries remain readable. When migrations are externally managed, add that column before starting new workers. Do not overlap old and new SQL workers during rollout: old workers settle by row ID and cannot honor the new claim-generation fence. +- `queue.NewSQLStore` preserves the established migration-on-first-use behavior and additively creates `bus_workflow_transition_receipts`, including non-null `receipt_version` and `event_schema_version` columns. Real SQLite, MySQL, and PostgreSQL tests cover fresh auto-schema plus receipt-backed terminal-chain recovery. `SQLStoreConfig.AutoMigrate` remains in the compatibility shape, but its false zero value does not opt out. A wholly fresh MySQL schema uses `VARBINARY(255)` workflow/member and receipt identities. When established state tables exist but the receipt table is missing, startup validates their byte-exact `VARBINARY` keys and derives one shared receipt width: `workflow_id` is the larger effective chain-or-batch ID capacity, and `member_id` is the larger chain-node-or-batch-job capacity. `TestWorkflowStoreIntegration_MySQLAutoMigratesMissingReceiptAtLegacyWidths` proves this preserves a live 512-byte legacy schema and identities above the fresh defaults. Automatic startup never alters an existing receipt table; its capacities instead participate in the connected-schema intersection. An existing incompatible receipt table therefore requires a quiescent managed migration and a fresh store instance. If the derived composite primary key exceeds the MySQL server's indexed-key budget, receipt creation fails with both derived widths and schema-first guidance rather than narrowing or altering established tables. Operators must then quiesce workers, audit live identities, and precreate a compatible indexed receipt schema; if the server cannot represent the required capacities, identity limits and existing data need an explicit migration before rollout. Those exceptional cases are persisted-schema, runtime-behavior, and operational concerns, not source/API, configuration-file, wire, or minimum-Go-version changes. Externally managed workflow schemas must precreate every dialect-correct workflow table and use `queue.NewSQLStoreWithManagedSchema` before new workflow workers start. Rollback should quiesce new workers first and leave the additive receipt table in place; old binaries ignore it, while dropping it destroys recovery provenance and old pruning code may otherwise leave receipt rows behind. Managed-schema migration/rollback, real cross-dialect pruning, and physical commit/readback ambiguity when the database or context is unavailable remain open. +- The additive opt-out does not yet replace a versioned rollout policy. Operators that keep automatic migration enabled must grant the required DDL permissions and coordinate concurrent startup, especially for the uniqueness expiry index; MySQL and PostgreSQL concurrency/permission evidence remains open. - Supports retry/backoff, poison retry, dispatch context cancellation, and deterministic duplicate prevention in the shared suite. - Broker fault injection scenarios are not enabled in the shared suite for DB backends. - DB backends rely on stale-`processing` recovery behavior for crash recovery (`process_recovered` event visibility is important operationally). @@ -70,21 +116,32 @@ These are the primary shared-scenario proofs for the matrix above. ### NATS (`DriverNATS`) -- Supports core dispatch/processing and backoff/poison retry semantics in the shared suite. +- The current implementation uses Core NATS publish/subscribe, not JetStream. It is an ephemeral broker adapter: there is no durable consumer acknowledgement, retained work queue, or crash recovery boundary. +- Plain subscriptions are broadcast semantics, not competing-consumer queue semantics: every worker subscription on the same subject, whether in one process or several, can receive a copy. +- Worker startup reports success only after the server has observed the subscription, and failed startup can be retried. Shutdown waits for callbacks and delayed work already admitted to that worker before closing its producer connection. +- Initial and replacement publishes flush through a server roundtrip before reporting success. This proves only that the Core NATS server observed the ephemeral publish; it is not a durable queue acknowledgement. +- Retry is republish-based. Shutdown drains the worker's subscription before admitted handlers finish replacement publication; without another subscriber, Core NATS can accept and then discard that replacement. A process or connection failure can likewise lose the original or replacement message, so this backend does not currently conform to a durable committed-retry contract. +- The shared suite exercises core dispatch/processing and backoff/poison behavior only while the fixture remains available. - Shared suite does not claim restart durability guarantees for delayed/retried work (`supportsRestart=false`, `supportsRestartDelayedDurability=false`). - Deterministic no-duplicate and ordering guarantees are not claimed in the suite. - Users should treat ordering as non-guaranteed unless a stronger constrained contract is explicitly added and tested. ### SQS (`DriverSQS`) +- Initial and replacement acceptance require a successful SQS `SendMessage` response with a non-empty service-generated message ID. SQS delivery is redeliverable through visibility timeout, so handlers must be idempotent. +- Retry republishes before deleting the original. Positive process and workflow facts wait for `DeleteMessage`; missing receipts or delete failures emit `settlement_failed` and leave the original eligible for redelivery, so duplicate handling remains mandatory. - Supports restart recovery in the shared suite, but not delayed/retry restart durability guarantees (`supportsRestartDelayedDurability=false`). - Broker fault scenarios are not deterministically exercised in the shared suite. - Ordering guarantees are not claimed. +- Workers do not yet extend message visibility while a handler runs, and each receive can return several messages that one worker processes sequentially. Configure visibility for the worst-case interval from receive through completion of the last message in that batch, and expect duplicate delivery if processing exceeds it; M4-03 tracks heartbeat/extension support. - Local integration validation uses LocalStack. ### RabbitMQ (`DriverRabbitMQ`) +- Initial and replacement persistent publishes require a positive publisher confirmation before dispatch succeeds or the original delivery is acknowledged. +- Worker retry publishes before acknowledging the original. Positive process and workflow facts wait for Ack. A negative confirmation permits safe claim compensation; a missing, canceled, or failed confirmation is treated as ambiguous and does not trigger reconnect-republish or uniqueness release. Ack/Nack failures emit `settlement_failed` because the original may redeliver. - Supports restart recovery and delayed/retry restart durability in the shared suite. +- A worker does not yet reconnect after its delivery channel closes; reconstruct the queue runtime after connection loss. Dial retries and AMQP channel/connection closure are not fully context-aware and can overrun a caller's lifecycle deadline. - Broker fault scenarios are not deterministically exercised in the shared suite. - Ordering guarantees are not claimed. @@ -99,6 +156,7 @@ Until the shared ordering contract is split into condition-specific scenarios, t ## Production Guidance Notes - Treat this matrix as the contract for what the shared integration suite validates today. +- Do not infer cross-process `UniqueFor` behavior from an instance-scoped backend. Use the scope and failure boundaries documented above. - If you change a capability flag in `integration/all/integration_scenarios_test.go`, update this document in the same PR. - If you want to claim a stronger backend guarantee publicly, add or unskip the corresponding shared scenario first. diff --git a/docs/bus-design.md b/docs/bus-design.md index 316b951..cdf68b1 100644 --- a/docs/bus-design.md +++ b/docs/bus-design.md @@ -1,4 +1,38 @@ -# Bus Library Design (Handoff) +# Workflow Architecture (Historical Bus Design) + +> **Status:** This is the original bus design record, retained to explain the +> version-one wire and API constraints. It is not the current architecture or +> implementation roadmap; use `plan.md` for both. + +## Current Ownership + +- `*queue.Queue` is the sole canonical application facade for dispatch, handlers, middleware, chains, batches, workflow state, stores, and observation. +- `internal/workflow.Engine` owns orchestration implementation. It depends on the neutral `busruntime.Runtime` transport seam and does not import root `queue`, public `bus`, or `queuecore`. +- Public messages, results, middleware, persisted workflow records, and store contracts are physical root `queue` types. Private adapters translate them at the engine boundary so GoDoc, reflection, generators, and custom stores never expose `internal/workflow` as their apparent owner. +- Public `bus` is a deprecated compatibility package. `bus.New(existingQueue)` returns an option-free adapter over that queue's existing engine; it does not register a second engine. Construction options and `NewWithStore` are rejected for an already-built queue and must instead be supplied through root queue options. +- The legacy raw-`busruntime.Runtime` construction route remains temporarily supported for integrations and preserves its observer, store, clock, and middleware options. +- `bus.Job` remains a boundary DTO because its public fields, composite literals, deferred JSON encoding, and raw string/byte semantics cannot alias `queue.Job` compatibly. `bus.JobOptions` is a source-compatible alias of the root persisted-options shape, and the facade converts the job once into the canonical root path. +- The self-returning `bus.ChainBuilder` and `bus.BatchBuilder` interfaces remain physical deprecated contracts. Keeping them distinct avoids breaking downstream type switches and custom implementations; their adapters still delegate every operation to the canonical engine. +- The legacy `bus.Event` observer shape remains only at that compatibility boundary. Root `queue.Observer` is the canonical event model and the internal engine has one event producer. +- `queue.FakeQueue` owns the only fake state and runs chain/batch construction through the production workflow engine and memory-store contract. Deprecated `bus.Fake` and `queuefake.Fake` values are typed compatibility views over that same concurrency-safe recorder; they do not own independent dispatch, builder, or assertion models. +- Version-one physical names and JSON envelopes remain readable compatibility contracts despite the historical prefix. Root direct dispatch now uses the application job type and payload; `bus:job` remains registered for old backlog, reserved-name collisions, the migration option, and the raw-runtime compatibility route. Chain, batch, and callback deliveries retain `bus:chain:node`, `bus:batch:job`, and `bus:callback`. + +## Compatibility Migration + +Ordinary source forms remain supported: custom `bus.Bus`, store, middleware, observer, and builder implementations; keyed and unkeyed legacy DTO literals; the Temporal adapter; and the legacy fake all compile against the facade. The following runtime/tooling identity migration is intentional: + +- compatible `bus` message, result, middleware, workflow-record, and store aliases now have the root package identity `github.com/goforj/queue`; +- `bus.Job`, `bus.Event`, `bus.Observer`, `bus.Bus`, `bus.Option`, `bus.ChainBuilder`, and `bus.BatchBuilder` retain their legacy `github.com/goforj/queue/bus` identity; +- code that keys behavior on `%T`, `reflect.Type.PkgPath`, gob/interface registration names, generated registries, dependency-injection keys, or a custom type-sensitive persistence format must map the applicable old `bus` names to the root `queue` names. + +One configuration and runtime-behavior incompatibility is intentional: every option-free `bus.New(existingQueue)` facade now shares the root queue's handler registry, store, observer, middleware, and lifecycle instead of constructing independent state over the same physical runtime. Code that deliberately relied on isolated root and bus state must use distinct queue runtimes; ordinary callers should register and configure one root queue and treat `bus` only as a compatibility view. `bus.New(existingQueue, nonNilOption...)` and `bus.NewWithStore(existingQueue, ...)` now return `bus.ErrQueueOptionsUnsupported` because those options cannot configure only the shared view. Supply `queue.WithObserver`, `queue.WithStore`, `queue.WithClock`, and `queue.WithMiddleware` when constructing the root queue, then call `bus.New(existingQueue)` without options. The retained raw-`busruntime.Runtime` route continues to accept legacy bus options. + +Fake runtime behavior is also intentionally corrected: abandoned builders no longer satisfy chain or batch assertions, invalid or canceled dispatches remain absent, builder options are retained, returned workflow IDs identify lookup state, effective default queues are assertion-visible, and Reset clears direct plus workflow records from every compatibility view. Recording fakes accept closure callbacks for fluent compatibility but do not retain them in fake runtime state or execute them. Tests that asserted the old constant `fake-chain` or `fake-batch` identifiers must instead treat returned IDs as opaque and may use `FindChain` or `FindBatch`; tests that deliberately depended on separate queue/workflow direct histories must migrate to the unified direct assertions. Constructor signatures, the zero value, value copies after initialization, and physical `bus.Fake`/`bus.BatchSpec` identities remain source-compatible; configuration, persisted data, wire formats, operations, and the minimum Go version are unchanged. + +The type-identity migration itself did not change wire or persistence contracts. The later direct-delivery cutover deliberately does; see [Direct Delivery Migration](direct-delivery-migration.md) for the exact wire, SQL, runtime, and rollout boundary. Literal legacy-wire and legacy-SQL fixtures continue to guard backward reading. + +The remainder of this document describes the superseded proposal. Examples that +construct or configure `bus` directly should not be treated as current guidance. This document defines a `bus` package for GoForj that composes on top of `github.com/goforj/queue` and provides workflow orchestration primitives: dispatch, chain, batch, callbacks, middleware, events, and test fakes. @@ -85,8 +119,8 @@ type JobOptions struct { Constructors: ```go -func New(q queue.Queue, opts ...Option) (Bus, error) -func NewWithStore(q queue.Queue, store Store, opts ...Option) (Bus, error) +func New(q any, opts ...Option) (Bus, error) +func NewWithStore(q any, store Store, opts ...Option) (Bus, error) func NewFake() *Fake ``` @@ -190,8 +224,8 @@ Bus emits internal lifecycle events: Observer API: ```go -type Observer interface { Observe(Event) } -type ObserverFunc func(Event) +type Observer interface { Observe(context.Context, Event) } +type ObserverFunc func(context.Context, Event) func MultiObserver(observers ...Observer) Observer ``` @@ -204,10 +238,12 @@ Event fields (minimum): - `occurred_at`, `duration` - `error` (optional) +The observer event schema and the workflow-envelope protocol are separate version domains even though both currently start at `1`. Envelope `schema_version` governs internal workflow dispatch decoding. `Event.SchemaVersion` governs the shared queue/worker/workflow observer contract, and transition-receipt `event_schema_version` pins only that observer contract. + ## State Model and Store ```go -type Store interface { +type WorkflowStore interface { CreateChain(ctx context.Context, rec ChainRecord) error AdvanceChain(ctx context.Context, chainID string, completedNode string) (next *ChainNode, done bool, err error) FailChain(ctx context.Context, chainID string, cause error) error @@ -219,6 +255,19 @@ type Store interface { MarkBatchJobFailed(ctx context.Context, batchID, jobID string, cause error) (BatchState, done bool, err error) CancelBatch(ctx context.Context, batchID string) error GetBatch(ctx context.Context, batchID string) (BatchState, error) + + MarkCallbackInvoked(ctx context.Context, key string) (bool, error) + Prune(ctx context.Context, before time.Time) error +} +``` + +Stores that execute across competing workers can add the compatible outcome capability: + +```go +type WorkflowOutcomeStore interface { + WorkflowStore + FailChainNode(ctx context.Context, chainID, nodeID string, cause error) (ChainState, bool, error) + SettleBatchJob(ctx context.Context, batchID, jobID string, outcome BatchJobOutcome, cause error) (BatchState, bool, error) } ``` @@ -227,6 +276,28 @@ Implementations: - `MemoryStore` (local/test default) - `SQLStore` (recommended production) +Workflow creation requires a non-empty workflow ID, at least one chain node or batch member, and a non-empty unique ID for every node or member. Builders already produce records with those properties; applications that call `WorkflowStore` directly must do the same. The built-in memory store snapshots chain nodes and payload bytes during creation and returns isolated copies, so mutating an input record, successor, or `ChainState` does not mutate persisted state. + +Both implementations claim a chain node or batch member before changing its parent state. SQL performs that claim and an arithmetic parent update in one transaction, so duplicate delivery cannot advance twice and concurrent batch members cannot overwrite one another's counters. The same concurrency contract runs against SQLite, MySQL, and PostgreSQL. + +Built-in stores also implement the additive `WorkflowOutcomeStore` capability. It gives successful and failed deliveries of one chain node or batch member a single first-writer settlement boundary. A contradictory late delivery is acknowledged without changing the committed outcome or aggregate counters; it emits no losing logical job/workflow fact, advances no application progress, and invokes no callback. Chain transitions compare the persisted node order and `NextIndex`; batch transitions report whether the requested outcome category owns the already-claimed member. The established batch schema and `BatchState` do not retain a per-member failure cause, so the `cause` argument remains delivery-local metadata rather than part of first-writer ownership. Persisted chain failures retain their first authoritative cause; built-in `FailChain` is now a no-op for an already-terminal chain instead of overwriting that cause. The base `WorkflowStore` remains source-compatible for established custom stores, but a custom implementation must add `WorkflowOutcomeStore` to provide the public atomic contradictory-category guarantee across processes. + +The built-in engine store has a narrower private contract as well. Its `claimedNow` result means only that the current store call performed the transition; it is response-local and is not durable owner proof. For durable ownership, memory and SQL built-ins record an immutable transition receipt containing its receipt format version, reconstructed observer-event schema version, workflow incarnation, member outcome, physical delivery generation and attempt, and correlated job identity. SQL writes that receipt in the same transaction as the workflow mutation; memory retains it only for the life of the process. Both version fields are currently `1`. A runtime fails recovery closed when either `receipt_version` or `event_schema_version` is unsupported: it returns an uncommitted error and neither acknowledges the delivery, executes application code, marks application state committed, nor reconstructs facts from a format it does not understand. Logical receipt validation requires a complete valid persisted owner, including a nonnegative owner attempt; nonempty current dispatch/`JobID`; matching workflow kind/ID/member/incarnation and owner dispatch; and an immutable job fingerprint match. The current attempt is only physical provenance and may differ from the owner or be negative. Chain duplicates may also carry a different physical `JobID`; batch `JobID` remains its logical member key and must match. A logically valid receipt proves the application transition and suppresses handler replay. Reconstructing successful member or aggregate facts additionally requires exact recovered generation, current attempt, and physical `JobID` ownership. Queue provenance alone cannot prove either boundary. + +After the current delivery both claims a built-in transition and obtains its receipt, the engine marks application state committed on that delivery's settlement boundary. This is a provenance handoff, not queue settlement: if later workflow infrastructure requires same-attempt redelivery, SQL retains the current generation as the receipt owner instead of continuing to carry an older recovered generation. A numbered application retry still clears the link. Focused settlement, SQL-token, and chain post-transition tests cover this handoff, while real SQLite, MySQL, and PostgreSQL tests now cover receipt-backed terminal-chain recovery after forced finalization failure. + +A logically valid failed chain receipt uses the authoritative `ChainState.Failure` to return a permanent physical outcome across exact, different, or legacy recovered-generation provenance and across different attempts or physical `JobID`s. An empty persisted cause returns a permanent diagnostic rather than success. Recovery does not re-run the handler, Catch/Finally callbacks, or occurrence-based `JobFailed`/`ChainFailed` facts. Invalid receipt/event version, incomplete owner, logical dispatch/job-content mismatch, workflow incarnation, outcome, or aggregate flags return an uncommitted error before application code; physical nonownership alone does not. A receipt-absent legacy built-in row and application-defined/decorated stores retain the compatibility fallback, which may execute the handler once to preserve terminal physical classification while still suppressing duplicate facts and callbacks. Failure-receipt insertion and parent failure are one SQL transaction, and a receipt-insert fault rolls both back. A real SQLite queue fixture forces the initial archive plus multiple recovery archives to fail, verifies first-cause and generation lineage survive at attempt zero, and then reaches `dead` at attempt one with the persisted cause and only one application/workflow failure occurrence. Equivalent MySQL/PostgreSQL failed-chain finalization fixtures remain open. + +For batches, the terminal member's transaction also records aggregate completion ownership. Built-in memory settlement holds one mutex; MySQL and PostgreSQL lock the parent row after the member compare-and-swap, so only the first false-to-true terminal transition can create the aggregate-owner receipt. Real twelve-member, twelve-worker fail-fast races on both server dialects prove one aggregate receipt and one failed/cancelled terminal fact pair while every member retains its own receipt. Because batch `JobID` is the logical member key, it must match for replay suppression. A valid duplicate with a different attempt, recovered generation, or legacy provenance still suppresses its handler; successful duplicates settle without facts, while failed duplicates return a generic permanent cause because the original application cause is not persisted. A SQL aggregate row is valid only as a completed transition; cancellation must own failure; and an aggregate row that names the requested member must match that member receipt's workflow incarnation, complete owner, and outcome. Recovery also checks those flags against live terminal state. Missing or contradictory proof fails uncommitted before handlers, callbacks, state-commit signaling, or facts. `BatchCompleted` may be reconstructed for a batch of any size only when this validated aggregate receipt names the exact recovered generation, current attempt, and `JobID`; completed aggregate state without that exact fact owner is never sufficient. `TestSQLStoreBatchAggregateOwnershipMismatchFailsClosed`, `TestSQLStoreBatchAggregateIncarnationMismatchFailsClosed`, and `TestBatchRecoveryRejectsInvalidAggregateReceiptShape` pin these corruption boundaries. `TestDatabaseIntegration_SQLite/sqlite_terminal_batch_completion_recovers_from_completing_member` proves the two-member recovery case without executing either handler again or attributing completion to the earlier member. The SQL delivery reaches its terminal `dead` archive on failed recovery rather than being acknowledged as success, and no fabricated failure or member fact is published. The normal `queue.WithStore(queue.NewSQLStore(...))` path unwraps the built-in store so this private contract remains available, and an option-free `bus.New(existingQueue)` shares that same engine. Application-defined stores, decorators around built-ins, and the retained raw-runtime bus route expose only public store capabilities. They remain source-compatible and can retain first-writer outcome-category semantics through `WorkflowOutcomeStore`, but they do not receive the exact built-in generation/receipt or `claimedNow` guarantees. + +Any recovered predecessor whose validated durable state proves success checks live `NextIndex` and re-dispatches the immediate successor while it has not progressed, without re-running the predecessor. Exact recovered generation, attempt, and physical `JobID` ownership can also reconstruct the predecessor's deferred facts. A missing receipt, an application-defined/decorated store without receipt capability, or a logically valid receipt with different/legacy generation, different attempt, or different physical `JobID` restores only the live continuation; it emits no predecessor facts or callbacks. A supported success receipt is logically validated before this liveness fallback: it cannot own cancellation, and its completion flag must exactly match whether the predecessor is final. Corruption returns an uncommitted outcome with no dispatch or effects. A successor enqueue rejection is likewise uncommitted so recovery can try again. This repairs definite enqueue rejection and legacy/custom liveness, but it is deliberately at-least-once: the predecessor row cannot distinguish a missing successor from one already queued but not yet reflected in workflow state, so recovery may enqueue a duplicate. Once the successor has progressed or the chain is terminal, recovery does not dispatch it again. + +Transition receipts are not observer or continuation outboxes. They neither prove callback delivery nor retain `Progress` closures, successor-enqueue acceptance, batch fan-out, or deferred observer invocation after queue settlement. After a recovered SQL delivery exhausts finalization retries, the driver makes a fenced best-effort repair that restores its inherited receipt lineage on the same attempt and returns it to `pending` with a bounded delay. Real SQLite success, failed-chain, and failed-batch scenarios force multiple recovery finalization failures before later settlement and prove no handler replay while the repair succeeds. The repair cannot cover a failed repair transaction, physical commit/readback ambiguity, or a row already removed by successful queue settlement. Durable continuation intents and a settlement outbox remain roadmap work. + +MySQL key validation follows the capacities discovered from every identity column used by a workflow and its receipt. A wholly fresh auto-schema uses 255-byte workflow/member and receipt identities plus 512-byte callback keys. When only the receipt table is missing beside established state, ordinary startup derives its `workflow_id` width as the larger of the effective chain and batch ID capacities and its `member_id` width as the larger of the chain-node and batch-job capacities. A real upgrade fixture widens legacy state to 512 bytes, drops only the receipt table, and proves startup recreates it at 512/512 while accepting identities above the fresh defaults. Existing tables are never altered: a pre-existing receipt instead intersects the accepted capacities discovered from the complete live schema. An incompatible existing receipt therefore requires a quiescent managed migration and a new store instance. If the derived three-column primary key exceeds the server's indexed-key budget, creation fails with the derived widths and schema-first guidance; it does not silently narrow established identities. Operators must precreate a compatible indexed receipt schema or explicitly migrate supported identity limits and existing data before rollout. + +Caller-managed workflow identity columns, including `bus_workflow_transition_receipts.workflow_id` and `.member_id`, must use `VARBINARY`; `VARCHAR`, `TEXT`, and fixed-width `BINARY` are rejected because they do not provide the same byte-exact round-trip contract. `queue.NewSQLStore` retains the legacy behavior of enabling schema creation even when compatibility field `SQLStoreConfig.AutoMigrate` is false; use `queue.NewSQLStoreWithManagedSchema` only after provisioning every required table and both receipt version columns. Before upgrading an incompatible schema, quiesce workflow writers, audit case- or padding-equivalent keys for collisions, convert all identity columns and align their receipt widths in one maintenance window, then restart workers with a new store instance. This is a MySQL persisted-schema, runtime-behavior, and operational rollout concern, not a source/API, configuration-file, wire, or minimum-Go-version change. Real MySQL and PostgreSQL tests exercise fresh auto-created receipt tables, serialized aggregate ownership, and receipt-backed recovery. On rollback, quiesce new workers and leave the receipt table in place for a later re-upgrade; old binaries ignore it, while dropping it discards transition provenance. Managed-schema migration and physical commit/readback ambiguity when a post-commit receipt read cannot reach the database remain open. Conservative chain re-dispatch does not make successor enqueue exactly-once, and batch fan-out still requires persisted dispatch-intent work. + ## Failure, Idempotency, Retry Ownership Chain: diff --git a/docs/bus-implementation-checklist.md b/docs/bus-implementation-checklist.md index 3b0fc0f..0b784da 100644 --- a/docs/bus-implementation-checklist.md +++ b/docs/bus-implementation-checklist.md @@ -1,6 +1,9 @@ # Bus Implementation Checklist -Use this as the source of truth for issue creation and implementation order. +> **Status:** Historical checklist for the original independent `bus` +> implementation. `plan.md` is the current source of truth. The engine now lives +> in `internal/workflow`, root `queue` owns the application surface, and public +> `bus` is a deprecated forwarding/raw-runtime compatibility facade. ## Scope diff --git a/docs/compatibility-policy.md b/docs/compatibility-policy.md index 4881968..8a524dc 100644 --- a/docs/compatibility-policy.md +++ b/docs/compatibility-policy.md @@ -6,6 +6,8 @@ This document defines how queue backend compatibility is communicated for releas - The project documents backend/runtime compatibility expectations in release notes and supporting docs. - Capability differences are documented separately (see `docs/backend-guarantees.md`). +- Release notes classify source/API, configuration, persisted-data, runtime-behavior, operational-rollout, and minimum-Go-version changes independently. +- A worker-first wire rollout or uniqueness quiescence window is an operational migration even when source APIs remain compatible. ## What "supported" means here diff --git a/docs/direct-delivery-migration.md b/docs/direct-delivery-migration.md new file mode 100644 index 0000000..753723c --- /dev/null +++ b/docs/direct-delivery-migration.md @@ -0,0 +1,83 @@ +# Direct Delivery Migration + +## Contract + +`Queue.Dispatch` sends an ordinary job using its application `Job.Type` and exact `Job.PayloadBytes()`. It does not create a workflow envelope. Correlation travels beside the job through versioned driver metadata: + +```json +{ + "schema_version": 1, + "dispatch_id": "dsp_...", + "job_id": "job_...", + "queue": "critical" +} +``` + +`chain_id` and `batch_id` are reserved in the same record for correlated delivery kinds. Direct jobs leave them empty. The application payload is never nested inside this record. + +Drivers carry the record using their native transport boundary: + +- Sync and Workerpool retain it privately on the in-memory `Job` value. +- Redis stores JSON in the `goforj-queue-driver-job-metadata` Asynq header. +- NATS, SQS, and RabbitMQ add an optional `metadata` member to their existing transport message. +- SQL stores JSON in the nullable `queue_jobs.metadata_json` column. + +Missing metadata is valid for legacy and low-level deliveries. Version 1 is trusted. Malformed or unknown versions never block application delivery and never supply correlation IDs; workers fall back to the physical application identity or decode a supported version-one workflow envelope. + +Chains, batches, and ephemeral callbacks continue to use the version-one workflow envelope because their durable state transitions require orchestration fields. The raw `bus.New(busruntime.Runtime)` compatibility route also retains its exact version-one `bus:job` bytes. New workers keep all four legacy handlers registered, so old backlog remains readable. + +Application job types equal to `bus:job`, `bus:chain:node`, `bus:batch:job`, or `bus:callback` continue through the legacy direct envelope. This prevents an application registration from replacing a reserved workflow handler. + +## Deployment Order + +Compatibility is intentionally one-way: a new worker reads old envelopes and new direct deliveries, while an old worker does not understand a new application-type delivery. Use this expand-and-contract rollout: + +1. Prepare SQL deployments without overlapping worker versions. New SQL workers fence every processing claim with `processing_token`, but old workers settle by row ID and cannot honor that generation fence. Quiesce every old SQL worker before starting any new SQL worker, even while producers still use `queue.WithLegacyDirectEnvelope()`. + + For externally managed schemas, add both nullable columns while the old SQL worker fleet is quiesced: + + | Dialect | `metadata_json` | `processing_token` | + | --- | --- | --- | + | PostgreSQL | `TEXT NULL` | `TEXT NULL` | + | SQLite | `TEXT NULL` | `TEXT NULL` | + | MySQL | `TEXT NULL` | `VARCHAR(64) NULL` | + + For example, PostgreSQL and SQLite use: + + ```sql + ALTER TABLE queue_jobs ADD COLUMN metadata_json TEXT NULL; + ALTER TABLE queue_jobs ADD COLUMN processing_token TEXT NULL; + ``` + + MySQL uses: + + ```sql + ALTER TABLE queue_jobs ADD COLUMN metadata_json TEXT NULL; + ALTER TABLE queue_jobs ADD COLUMN processing_token VARCHAR(64) NULL; + ``` + + When automatic migration is enabled, quiesce the old SQL worker fleet before starting the new binary; new-worker startup adds both columns before polling. Automatic migration does not make mixed old/new SQL workers safe. When migration is disabled, readiness and worker startup validate that both queue objects are base-table relations, including PostgreSQL partitioned tables, and that every column used by the current runtime is present. An empty, view-backed, or incomplete schema fails closed without queue DDL. After deployment tooling installs the complete schema, the same runtime can retry readiness and startup. This validation checks presence, not write permissions, exact SQL types, constraints, or performance indexes, so deployment tooling must still apply the dialect-correct canonical schema. +2. Deploy the new worker-capable release while producers include `queue.WithLegacyDirectEnvelope()`. This keeps all producers on `bus:job` until every consumer has been replaced. For SQL, this is a fleet replacement after the old workers have stopped, not a rolling overlap. +3. Verify no old consumers remain for the target queues. +4. Remove `WithLegacyDirectEnvelope` from producers to enable canonical direct delivery. +5. Before rolling workers back, restore legacy producer emission and drain every direct-delivery backlog with the new workers. For SQL, quiesce every new worker after the drain and before starting any old worker binary. The two additive columns can remain in place; old binaries ignore them. Only then may old workers return. + +For non-SQL backends, do not run old and new consumers after step 4. The failure mode differs by backend: SQS and RabbitMQ can delete or acknowledge an unknown application type; Redis can retry and archive it; Core NATS can drop it and its broadcast model can also duplicate work during overlapping consumer cutovers. Core NATS therefore requires a coordinated consumer/producer switch rather than a durability claim. SQL has the stricter boundary described above: old and new SQL workers must never overlap during rollout or rollback because generation fencing is independent of the direct-delivery envelope format. + +SQL producers have a separate `UniqueFor` rolling-upgrade boundary. Older releases stored an unprefixed digest of the physical queue, type, and payload, while this release uses the canonical version-one logical identity. A current SQL dispatch atomically claims both identities in the same transaction as its queue row. An outstanding exact historical lock or canonical lock rejects the dispatch, an expired historical lock is renewed, and any collision or storage failure rolls back both claims and the queue row. This protects mixed producer versions when their physical queue, type, and payload bytes match exactly; it does not relax the no-overlap rule for SQL worker generations. + +Pre-version high-level workflow envelopes contained freshly generated correlation values. Independently dispatched equivalent work therefore does not share an exact historical key, even when its logical type and payload match. Stop those old producers and wait at least their longest live `UniqueFor` duration before starting new producers, or explicitly accept a transient logical duplicate window. Stable low-level producers whose physical identity is unchanged may overlap because both versions contend on the historical key. + +The current release retains dual claims unconditionally. A future release may stop writing historical keys only after the last old SQL producer has been removed and at least the longest `UniqueFor` duration it could have written has elapsed. Operators do not need to delete existing lock rows; normal expiry pruning removes both formats. The historical delimiter format can conflate distinct canonical jobs, such as type `a` with payload `b:c` and type `a:b` with payload `c` on the same queue, so the transition conservatively retains those false collisions until historical claiming is retired. Canonical version-one identity provides stable logical uniqueness for new work. + +## Compatibility Classification + +- **Source/API:** Existing root and `bus` calls retain their signatures. Driver metadata helpers and `WithLegacyDirectEnvelope` are additive advanced APIs. +- **Configuration:** Existing configuration remains valid. The migration option is temporary and opt-in. +- **Persisted data:** SQL adds two nullable columns, `metadata_json` and `processing_token`. Existing rows remain readable and retain `NULL` in both columns; old producers can continue inserting rows that omit them after migration. During the uniqueness transition, each accepted SQL `UniqueFor` dispatch stores both historical and canonical lock rows in the existing table. Existing locks are not rewritten, and expiry pruning removes both formats. +- **Wire:** New root direct deliveries use the application type and payload plus optional transport metadata. Legacy workflow envelopes remain readable and raw-runtime `bus` emission remains byte-stable. +- **Runtime behavior:** Exact `Job.PayloadBytes()` are delivered without a JSON re-marshal. Arbitrary bytes now reach the handler; `Message.Bind` reports a JSON error only if the application chooses to bind non-JSON bytes. An absent payload remains absent instead of becoming literal JSON `null`. +- **Operations:** Worker-first rollout and backlog-aware rollback are required as described above. SQL additionally requires complete old/new worker-fleet separation in both directions. +- **Minimum Go version:** Unchanged. + +Logical uniqueness remains on the version-one queue/type/payload identity. Direct and legacy-envelope forms therefore collide within the same declared backend scope, and dispatch/job correlation does not change the key. SQL's additional historical claim exists only to preserve outstanding pre-version locks during producer upgrades. diff --git a/docs/events.md b/docs/events.md index b1d024c..a4a0c4a 100644 --- a/docs/events.md +++ b/docs/events.md @@ -1,6 +1,6 @@ # Queue Events Contract -This document defines the public observability event contract emitted through `Observer`. +This document defines the root application facade's unified observability contract emitted through `queue.Observer`. `Event.Layer` distinguishes queue, worker, and workflow facts without requiring separate observer models on the normal `*queue.Queue` path. The deprecated `bus` package retains its legacy event shape only as an adapter at the compatibility boundary; it no longer owns a second event producer or orchestration engine. ## Goals @@ -10,8 +10,11 @@ This document defines the public observability event contract emitted through `O ## Event kinds -Dispatch lifecycle: +Queue dispatch lifecycle: +- `EventDispatchStarted`: public dispatch began. +- `EventDispatchSucceeded`: public dispatch crossed the backend acceptance boundary. A synchronous handler can still return an application error after this fact. +- `EventDispatchFailed`: public dispatch failed before acceptance. - `EventEnqueueAccepted`: job accepted for dispatch. - `EventEnqueueRejected`: dispatch failed with error. - `EventEnqueueDuplicate`: dispatch rejected as duplicate (`UniqueFor`). @@ -20,22 +23,48 @@ Dispatch lifecycle: Processing lifecycle: - `EventProcessStarted`: handler attempt started. -- `EventProcessSucceeded`: handler attempt succeeded. -- `EventProcessFailed`: handler attempt failed. -- `EventProcessRetried`: failed attempt scheduled for retry. -- `EventProcessArchived`: terminal failure (no retries left). +- `EventProcessSucceeded`: handler attempt succeeded. SQL, SQS, and RabbitMQ emit this only after durable row finalization, deletion, or acknowledgement respectively; backends without a post-handler settlement hook retain their documented weaker boundary. +- `EventProcessFailed`: handler attempt returned an error or panicked. A panic is reported before the original panic value is rethrown so backend recovery and retry semantics remain unchanged. +- `EventProcessRetried`: processing began for a numbered application retry attempt. Infrastructure redelivery of that same attempt may repeat the fact. +- `EventProcessArchived`: the driver confirmed terminal settlement for a failed attempt. +- `EventRepublishFailed`: an internal delay or retry replacement could not be published. +- `EventSettlementFailed`: durable SQL finalization, broker acknowledgement, or broker deletion failed after handler or replacement work completed, so redelivery remains possible. Queue control lifecycle: - `EventQueuePaused`: queue consumption paused. - `EventQueueResumed`: queue consumption resumed. +Workflow lifecycle: + +- `EventJobStarted`, `EventJobSucceeded`, `EventJobFailed` +- `EventChainStarted`, `EventChainAdvanced`, `EventChainCompleted`, `EventChainFailed` +- `EventBatchStarted`, `EventBatchProgressed`, `EventBatchCompleted`, `EventBatchFailed`, `EventBatchCancelled` +- `EventCallbackStarted`, `EventCallbackSucceeded`, `EventCallbackFailed` + +Positive job, chain, batch, and callback facts use the same SQL/SQS/RabbitMQ settlement boundary as `EventProcessSucceeded`. The SQL queue gives every processing claim an opaque generation ID. Same-attempt infrastructure redelivery normally retains inherited unsettled-generation provenance. When the current generation commits a receipt-backed workflow transition before later infrastructure work requests redelivery, the workflow engine marks application state committed and SQL retains that current generation instead. The signal selects the truthful receipt owner; it does not commit deferred facts or prove observer delivery. An application retry increments the attempt and clears the link, while recovery flags, aggregate state, and application error text do not supply equivalent authority. + +The direct built-in workflow-store path records a separate transition receipt in the same mutation that commits a chain-node or batch-member outcome. `receipt_version` identifies the durable ownership format and `event_schema_version` identifies the shared observer fact contract it can reconstruct; both are currently `1`. The event schema is independent from the workflow-envelope protocol even while their current numeric values match. An unsupported receipt or event-schema version fails recovery closed with an uncommitted error: the worker does not acknowledge the delivery, run application code, mark application state committed, or publish reconstructed facts. Logical receipt proof requires supported versions; a complete valid persisted owner, including a nonnegative owner attempt; matching workflow kind/ID/member/incarnation and logical dispatch; nonempty current dispatch/`JobID`; and matching immutable job content/fingerprint. The current attempt is physical provenance and may differ from the owner or be negative. Chain physical `JobID` may also differ; batch `JobID` is its logical member key and must match. That proof suppresses duplicate handler execution. Successful fact reconstruction additionally requires exact prior `RecoveredGenerationID`, current attempt, and physical `JobID` ownership. A physical nonowner publishes no recovered success fact. An exact owner defers the already-committed `EventJobSucceeded` plus `EventChainAdvanced`, `EventChainCompleted`, or `EventBatchProgressed` fact to the reclaimed delivery's new settlement. Reconstructed successes use zero duration because the original handler timing is not persisted. Their deterministic `EventID` permits consumer deduplication, but observer invocation may repeat. Real SQLite, MySQL, and PostgreSQL finalization-failure scenarios cover successful terminal-chain recovery through their physical receipt schemas. + +A logically valid failed chain receipt returns the first persisted `ChainState.Failure` as a permanent physical outcome across exact, different, or legacy recovered-generation provenance and across different attempts or physical `JobID`s; an empty cause becomes a permanent terminal diagnostic. Recovery does not re-run the handler, Catch/Finally callbacks, `EventJobFailed`, or `EventChainFailed`; those facts remain occurrence-based and require the still-open settlement outbox if they must survive a post-transition finalization crash. Invalid version, incomplete owner, logical dispatch/job-content mismatch, workflow incarnation, outcome, or terminal flags fail closed before application code; physical nonownership alone does not. Receipt-absent legacy rows and stores without the private capability retain weaker replay behavior: application code may run once to recover terminal physical classification, but duplicate workflow facts and callbacks remain suppressed. The built-in stores preserve the first terminal chain cause so a later `FailChain` call cannot change what recovery returns. A real SQLite fixture proves repeated archive failure retains the cause and receipt lineage before one final `dead` settlement; server-dialect failed-chain fixtures remain open. + +`EventBatchCompleted` is reconstructed only when a validated aggregate receipt says the exact recovered generation, current attempt, and `JobID` made that batch terminal and live state agrees. Batch `JobID` is also the logical member key, so it must match even when only suppressing replay. Every SQL aggregate row must own completion, cancellation must own a failed outcome, and a row naming the requested member must match that member receipt's workflow incarnation, complete owner, and outcome. Cancellation and completion flags must also agree with live terminal state; corruption fails uncommitted before any partial member or aggregate fact. The aggregate proof works independently of batch size. Built-in memory settlement serializes the parent transition under one mutex; MySQL and PostgreSQL lock the parent batch row after each member claim, so only the first false-to-true terminal transition owns aggregate facts. `TestSQLStoreBatchAggregateOwnershipMismatchFailsClosed`, `TestSQLStoreBatchAggregateIncarnationMismatchFailsClosed`, and `TestBatchRecoveryRejectsInvalidAggregateReceiptShape` pin these constraints. Real twelve-member fail-fast races on both server dialects prove one aggregate-owner receipt and one `EventBatchFailed`/`EventBatchCancelled` pair. The real SQLite two-member terminal-owner scenario verifies that recovery emits completion with the completing member's `JobID`, does not re-execute either handler, and never infers ownership from the earlier member or aggregate state alone. A built-in store's `claimedNow` result suppresses duplicate facts and effects only for the immediate call and is not persisted proof. Custom stores, decorators around built-ins, and the retained raw bus construction path do not expose the private receipt contract, so they do not have this exact fact-recovery guarantee. + +A recovered chain predecessor whose validated durable state proves success re-dispatches its immediate successor while live state still points to that successor, without re-running the predecessor. Exact recovered generation, attempt, and physical `JobID` ownership can reconstruct the predecessor facts above; a missing receipt, a store without receipt capability, or a logically valid physical nonowner dispatches only the successor and emits no predecessor facts or callbacks. A supported success receipt is logically validated before liveness recovery: cancellation is invalid and completion must exactly match final-node position, otherwise recovery is uncommitted with no dispatch or effects. Progressed and terminal state is a no-op. This closes definite enqueue rejection and the legacy/custom liveness gap but does not make continuation delivery exactly-once: a successor already queued but not yet progressed is indistinguishable from a missing successor, so duplicate enqueue remains possible under the queue's at-least-once contract. + +A workflow transition receipt is not a settlement receipt or observer outbox. Fact recovery does not retain `Progress` closures, successor-enqueue acceptance, batch fan-out, or terminal callbacks. If a recovered SQL delivery's finalization fails again, the driver makes a fenced best-effort attempt to restore the inherited receipt-owner generation on the same attempt, return the row to `pending`, and delay the next reclaim. Real SQLite tests force multiple such failures before later success, failed-chain archive, or failed-batch archive without replaying application code. If queue finalization commits and the process exits before deferred observer calls, no recoverable queue row may remain. A durable settlement outbox with restart draining and persisted continuation/callback intents are still required; observers remain best-effort telemetry rather than workflow continuation machinery. Callback redelivery after an at-most-once marker emits no second success fact, and closure callbacks remain explicitly ephemeral. + +An allowed batch item failure emits `EventBatchProgressed` with `Err` and does not emit `EventBatchFailed`. The aggregate can later emit `EventBatchCompleted` after every item reaches an allowed terminal outcome. `EventBatchFailed` is reserved for a non-allowed failure or cancellation path that makes the aggregate fail. Batch settlement durably owns the member's success-or-failure category, but the established batch state does not persist a per-member error string. A same-call ambiguous workflow-store commit can resolve category ownership from a matching receipt while retaining that physical attempt's error detail when readback remains available. On restart, a logically valid failed receipt returns a generic permanent cause even across a different attempt, different or legacy recovered-generation provenance, so SQL archives the delivery as `dead` instead of acknowledging it as success. Batch `JobID` must still match the logical member. The recovery does not fabricate the missing original cause or emit replacement failure/member facts. Physical commit/readback ambiguity that cannot reach the database remains unresolved. + ## Required fields Present on all events whenever known: - `Kind` +- `Layer` - `Time` +- `SchemaVersion` +- `EventID` - `Driver` - `Queue` - `JobType` @@ -51,13 +80,18 @@ Failure/cancel/reject events additionally include: - `Err` +Every layer includes the applicable `DispatchID`, `JobID`, `ChainID`, and `BatchID` correlation fields when the delivery carries supported metadata. Queue and worker facts read the versioned direct-driver sidecar or decode a retained workflow envelope, so they can be joined to workflow facts without inspecting payloads in application observers. + ## Semantics and guarantees - Events are per-attempt, not aggregated. -- `EventProcessRetried` is emitted only when another attempt will occur (`Attempt < MaxRetry`). -- `EventProcessArchived` is emitted when retries are exhausted. -- `JobKey` is a deterministic hash key for correlation. It is not guaranteed globally unique. -- `Queue` defaults to `"default"` when not explicitly set. +- Dispatch, enqueue, and queue-control events use `EventLayerQueue`; physical attempt events use `EventLayerWorker`; logical job, chain, batch, and callback transitions use `EventLayerWorkflow`. +- `EventProcessRetried` is emitted when processing begins with `Attempt > 0`. It is intentionally not emitted merely because a handler returned an error, and consumers must tolerate a repeated fact when infrastructure redelivers the same numbered attempt. +- `EventProcessArchived` is reserved for a driver-confirmed terminal settlement; drivers that cannot yet confirm that boundary omit it rather than emitting a prediction. +- `JobKey` is a deterministic hash of the logical job type and payload. Volatile dispatch/workflow IDs are excluded, and the value is not guaranteed globally unique. +- Correlated recoverable job successes and emitted positive chain or batch transition facts use a deterministic `EventID` for the same logical fact across settlement recovery. Failure EventIDs remain occurrence-based. Deterministic identity supports deduplication; it does not prove that an observer received the fact or make every event exactly-once. +- `Queue` is the effective physical backend name carried by the dispatch. With a namespaced default such as `billing_default`, an explicit logical queue such as `critical` is reported as `billing_critical`. Jobs that omit a queue continue to report `default`; changing how `Config.DefaultQueue` routes empty targets is a separate targeting decision. Correlated queue, worker, workflow, aggregate, and callback facts always report the same name. +- Aggregate and callback workflow facts retain the triggering job's effective queue, logical job type, and `JobKey`, so observers can join them to queue and worker facts without reading payloads. ## Cross-driver support @@ -74,13 +108,42 @@ Driver-specific capabilities: ## Observer behavior contract -- Observers are side-effect hooks only; they must not control queue execution. -- Queue processing must continue even if an observer is slow, fails, or panics. -- Implementations should prefer non-blocking observer behavior in hot paths. +- Observers are best-effort telemetry hooks only; they must not control queue execution or implement workflow continuations. +- Observer calls are synchronous and causally ordered on an individual execution path. Slow observers therefore delay that path. +- Dispatchers and workers may invoke the same observer concurrently. Observer implementations must synchronize mutable state they own. +- Observer panics are isolated and do not change queue or workflow outcomes. +- Handler panics emit `EventProcessFailed` before being rethrown; observation does not convert them into returned errors or otherwise change backend panic handling. +- Use `ChannelObserver` when asynchronous delivery or an explicit drop-if-full policy is required. - Logging adapters should avoid raw payload logging by default. ## Versioning - `EventKind` names and baseline semantics are public API. +- `Event.SchemaVersion` identifies the canonical observer envelope shared by queue, worker, and workflow layers. It is not the workflow-envelope protocol version and may evolve independently. - Additive changes are allowed (new event kinds, new optional fields). -- Breaking changes require a major version bump. +- Breaking changes require an explicit compatibility release and migration guide; after v1 they require a major version bump. + +## Unified observer migration + +The observer collapse is an explicit pre-v1 compatibility boundary: + +- `queue.WithObserver` accepts `queue.Observer` and receives queue, worker, and workflow layers. +- `queue.WorkflowEvent`, `queue.WorkflowEventKind`, `queue.WorkflowObserver`, and `queue.WorkflowObserverFunc` are deprecated aliases of the root event model. +- Code that used unkeyed `queue.Event` or `bus.Event` literals must switch to keyed literals because the envelopes now include correlation fields. +- Adapt custom `bus.Observer` implementations with `queue.ObserverFunc` when constructing a root queue. `bus.WithObserver` remains supported only on the retained raw-`busruntime.Runtime` construction route; an already-built `*queue.Queue` must receive observation options when it is constructed. +- Sinks that only need logical job, chain, batch, and callback transitions can return early unless `event.Layer == queue.EventLayerWorkflow`. +- Legacy `queue.WorkflowObserver` and `bus.Observer` sinks also received `EventDispatchStarted`, `EventDispatchSucceeded`, and `EventDispatchFailed`. Those dispatch facts deliberately belong to `EventLayerQueue` in the unified model because they describe public queue acceptance, not a committed workflow transition. To retain the full legacy scope, accept the workflow layer plus those three event kinds: + +```go +if event.Layer != queue.EventLayerWorkflow { + switch event.Kind { + case queue.EventDispatchStarted, + queue.EventDispatchSucceeded, + queue.EventDispatchFailed: + default: + return + } +} +``` + +This migration changes Go source compatibility and observer volume/concurrency. It does not change persisted workflow records or queue wire envelopes. diff --git a/docs/flake-log.md b/docs/flake-log.md index 559b4b0..25539d3 100644 --- a/docs/flake-log.md +++ b/docs/flake-log.md @@ -16,7 +16,7 @@ Use this document to review and record timing/concurrency flake evidence from th - Includes: - `integration-flake--summary.md` - `integration-flake--attempts.tsv` - - per-attempt `go test` logs + - per-attempt `go test -json` logs ## Current repeated probes (default) @@ -40,8 +40,8 @@ For a release candidate: 1. Review the most recent `integration-flake-repeat` workflow run. 2. Open each backend artifact summary (`integration-flake--summary.md`). -3. Check per-scenario fail counts / flake rates. -4. If any failures occurred: +3. Check per-scenario pass, fail, capability-skip, and missing-event counts plus flake rates. Skips and missing events are excluded from the flake-rate denominator, and any missing expected scenario event fails the job. +4. If any failures or missing events occurred: - inspect corresponding per-attempt logs - classify root cause (`test assumption`, `timing budget`, `driver/backend behavior`, `real regression`) - file/fix before GA, or explicitly document/waive with rationale diff --git a/docs/ga-readiness.md b/docs/ga-readiness.md index e9c8d1e..52e2551 100644 --- a/docs/ga-readiness.md +++ b/docs/ga-readiness.md @@ -65,7 +65,8 @@ What is not yet sufficient for a GA claim: - Scenarios: - `scenario_dispatch_during_broker_fault` - `scenario_consume_after_broker_recovery` - - Acceptance: failures are surfaced during broker outage and recovery path processes jobs after broker restoration. + - `TestIntegrationChaos_RedisBrokerDisconnectRedelivery` + - Acceptance: failures are surfaced during broker outage, recovery processes work after restoration, and Redis redelivers a handler-time lost acknowledgement without consuming the application retry budget. - [x] Duplicate-delivery idempotency scenario remains green. - Scenario: `scenario_duplicate_delivery_idempotency` @@ -137,8 +138,8 @@ What is not yet sufficient for a GA claim: - [ ] Canonical metrics/events contract is documented. - Acceptance: names, labels/fields, and semantics are documented and versioned. - - Must include recovery/failure events (for example `republish_failed`, `process_recovered`). - - Progress (2026-02-23): baseline contract added in `docs/metrics-contract.md` and baseline ops guidance added in `docs/ops-alerts.md` (includes `republish_failed` and `process_recovered` coverage). + - Must include recovery/failure events (for example `republish_failed`, `settlement_failed`, `process_recovered`). + - Progress (2026-02-23): baseline contract added in `docs/metrics-contract.md` and baseline ops guidance added in `docs/ops-alerts.md` (includes `republish_failed`, `settlement_failed`, and `process_recovered` coverage). - Remaining: pin a contract version, define required fields precisely, and align emitted metrics/log labels in production instrumentation. - [ ] Alerts and dashboards exist for core operations. @@ -195,6 +196,8 @@ What is not yet sufficient for a GA claim: ## 7. Coverage and Test Debt (should complete) +Coverage reporting now includes the root module, every buildable driver/example/integration module, and the tagged integration suite across the parallel backend matrix. CI rejects incomplete or ambiguous profile fan-in before its single required Codecov upload. This fixes reporting scope; it does not by itself close the test-debt items below. + - [ ] Close remaining low-value 0% helper branches where practical. - Acceptance: no easy/uncontroversial 0% branches remain in core runtime paths. diff --git a/docs/integration-scenarios.md b/docs/integration-scenarios.md index 483ac20..1e293e4 100644 --- a/docs/integration-scenarios.md +++ b/docs/integration-scenarios.md @@ -41,6 +41,9 @@ Named scenarios currently enforced: Optional long-run scenario (enabled with `RUN_SOAK=1`): - `scenario_soak_mixed_load` +Scheduled Redis fault scenario (enabled with `RUN_CHAOS=1`): +- `TestIntegrationChaos_RedisBrokerDisconnectRedelivery` + ## Latest trust snapshot Last full run (local, Docker/testcontainers): @@ -79,6 +82,7 @@ What this proves today: - Multi-worker contention is validated for deterministic backends to ensure single successful processing per job. - Duplicate-delivery idempotency patterns are validated under forced retry with single side-effect commit. - Broker fault injection and consume-after-recovery flow is validated on supported backends. +- Redis broker disconnect during an active handler is validated through failed acknowledgement, same-task lease recovery, and redelivery without consuming the application's zero-retry budget. The handler's idempotency key keeps its side-effect commit count at one. - FIFO ordering is validated only in the constrained `scenario_ordering_single_worker_fifo` sub-scenario for backends marked ordering-capable. - `scenario_ordering_multi_worker_best_effort` explicitly treats concurrent-worker ordering as a non-guarantee and only asserts completion/correctness. - Delayed/immediate and retry-based reordering behavior is explicitly exercised to avoid over-claiming FIFO semantics. @@ -131,7 +135,7 @@ Observability contract ## Execution model - `smoke`: always-on integration scenarios (current baseline). -- `chaos`: scheduled fault-injection and recovery scenarios (`scenario_dispatch_during_broker_fault`, `scenario_consume_after_broker_recovery`, `scenario_worker_restart_recovery`, `scenario_worker_restart_delay_recovery`, and race-heavy scenarios). +- `chaos`: scheduled fault-injection and recovery scenarios (`TestIntegrationChaos_RedisBrokerDisconnectRedelivery`, `scenario_dispatch_during_broker_fault`, `scenario_consume_after_broker_recovery`, `scenario_worker_restart_recovery`, `scenario_worker_restart_delay_recovery`, and race-heavy scenarios). - `soak`: extended runtime scenarios; isolated from normal CI when needed. ## Timing guardrails diff --git a/docs/metrics-contract.md b/docs/metrics-contract.md index f3adbe9..c431bd5 100644 --- a/docs/metrics-contract.md +++ b/docs/metrics-contract.md @@ -2,10 +2,7 @@ This document defines the baseline observability contract for `queue` before GA. -It covers two event streams: - -- Queue runtime events (`queue.Event`) via `queue.Observer` -- Workflow/runtime orchestration events (`queue.WorkflowEvent`) via `queue.WithObserver(...)` +For the normal root facade, it covers one event stream: `queue.Event` values delivered to the `queue.Observer` installed with `queue.WithObserver(...)`. `Event.Layer` identifies whether a fact came from queueing, worker execution, or workflow orchestration. The deprecated `bus` package translates the same internal producer into its frozen legacy event shape only for compatibility consumers; it no longer owns a second event stream. This is a baseline contract. Before GA, pin a version and treat field/label changes as compatibility-impacting. @@ -13,18 +10,22 @@ This is a baseline contract. Before GA, pin a version and treat field/label chan - Common field names across logs/metrics/traces - Predictable event semantics across backends -- Explicit handling of recovery/failure internals (`republish_failed`, `process_recovered`) +- Explicit handling of recovery/failure internals (`republish_failed`, `settlement_failed`, `process_recovered`) - Stable labels for dashboards and alerts -## Event Streams +## Unified Event Stream -### 1. Queue Runtime Events (`queue.Event`) +### Queue and Worker Layers Source: - `queue.Observer` - `queue.ObserverFunc` -- `queue.Config.Observer` +- `queue.WithObserver(...)` + +`queue.Config.Observer` is a deprecated compatibility path into the same stream. + +Queue, worker, and workflow events carry the same applicable dispatch/job/chain/batch correlation IDs. Internal envelope IDs are excluded from `Event.JobKey`, so telemetry grouping follows the logical application job rather than a one-off wrapper delivery. `Event.JobKey` remains an observability field rather than a persisted uniqueness key; both contracts resolve the same logical job type and payload, while uniqueness additionally includes version and effective queue framing. Event kind type: @@ -32,6 +33,10 @@ Event kind type: Current runtime event kinds include: +- public dispatch lifecycle: + - `dispatch_started` + - `dispatch_succeeded` + - `dispatch_failed` - enqueue lifecycle: - `enqueue_accepted` - `enqueue_rejected` @@ -49,15 +54,19 @@ Current runtime event kinds include: - internal recovery/failure: - `process_recovered` - `republish_failed` + - `settlement_failed` Recommended required fields (when available): | Field | Type | Notes | | --- | --- | --- | | `kind` | string | `queue.EventKind` value | +| `layer` | string | `queue`, `worker`, or `workflow` | | `time` | timestamp | event timestamp | +| `schema_version` | integer | event envelope schema version | +| `event_id` | string | unique event identifier | | `driver` | string | backend/runtime (`redis`, `sqs`, etc.) | -| `queue` | string | logical/physical queue name in runtime context | +| `queue` | string | effective physical backend queue name; omitted targets currently report `default` | | `job_type` | string | job type identifier | | `job_key` | string | stable job key/idempotency key when available | | `attempt` | integer | current attempt number | @@ -66,24 +75,23 @@ Recommended required fields (when available): | `duration_ms` | number | processing duration for completion/failure events | | `error` | string | normalized error string/class for failures | -### 2. Workflow Events (`queue.WorkflowEvent`) +### Workflow Layer Source: - `queue.WithObserver(...)` -- `queue.WorkflowObserver` -- `queue.WorkflowObserverFunc` +- events where `event.Layer == queue.EventLayerWorkflow` Event kind type: -- `queue.WorkflowEventKind` +- `queue.EventKind` + +The deprecated `WorkflowEvent`, `WorkflowEventKind`, `WorkflowObserver`, and `WorkflowObserverFunc` names are aliases of the canonical root model rather than a second stream. + +Public dispatch lifecycle facts are intentionally queue-layer events because they bracket queue acceptance. They are not logical workflow transitions, even when a workflow dispatch produced them. A sink migrated from the legacy workflow observer contract must accept `dispatch_started`, `dispatch_succeeded`, and `dispatch_failed` in addition to `EventLayerWorkflow` events to retain its former scope. A filter that accepts only `EventLayerWorkflow` is the narrower job, chain, batch, and callback stream. Current workflow event kinds (via internal orchestration engine) include: -- dispatch: - - `dispatch_started` - - `dispatch_succeeded` - - `dispatch_failed` - job orchestration: - `job_started` - `job_succeeded` @@ -108,7 +116,8 @@ Recommended required fields (when available): | Field | Type | Notes | | --- | --- | --- | -| `kind` | string | `queue.WorkflowEventKind` value | +| `kind` | string | `queue.EventKind` value | +| `layer` | string | `workflow` | | `time` | timestamp | event timestamp | | `schema_version` | integer | workflow event schema version | | `event_id` | string | unique workflow event identifier | @@ -116,7 +125,7 @@ Recommended required fields (when available): | `job_id` | string | workflow job record ID | | `chain_id` | string | chain workflow ID | | `batch_id` | string | batch workflow ID | -| `queue` | string | target queue | +| `queue` | string | effective physical backend queue name; omitted targets currently report `default` | | `job_type` | string | job type | | `attempt` | integer | attempt number for job events | | `duration_ms` | number | duration for succeeded/failed events | @@ -148,18 +157,21 @@ For metrics dimensions, keep cardinality bounded: These may be implemented via logs, counters, histograms, or OTel metrics. -### Queue Runtime Metrics +### Queue and Worker Metrics - `queue_events_total{kind,driver,queue}` +- `queue_dispatch_total{kind,driver,queue}` - `queue_process_duration_ms` histogram `{driver,queue,job_type}` - `queue_enqueue_failures_total{driver,queue}` - `queue_republish_failed_total{driver,queue}` (from `republish_failed`) +- `queue_settlement_failed_total{driver,queue}` (from `settlement_failed`) - `queue_process_recovered_total{driver,queue}` (from `process_recovered`) +For `StatsCollector`, an identity-bearing `settlement_failed` closes the exact active attempt because handler execution has ended, but it increments neither `Processed` nor application `Failed`: the delivery outcome remains unresolved and may redeliver. Current built-in drivers carry the same opaque physical identity through start, process, and settlement facts. An identity-less terminal fact cannot close an identity-bearing start, and an identity-less settlement fact leaves `Active` unchanged, because event fields cannot distinguish a late settlement from a newer execution of the same job. Upgrade settlement-aware driver modules with root, and require custom drivers to forward the handler's settlement context consistently, when exact gauges matter. This deliberately conservative compatibility behavior can overcount outdated drivers instead of undercounting unrelated live work. Track `queue_settlement_failed_total` separately rather than folding it into either terminal counter. + ### Workflow Metrics - `queue_workflow_events_total{kind,queue}` -- `queue_workflow_dispatch_total{kind,queue}` - `queue_workflow_job_duration_ms` histogram `{kind,queue,job_type}` - `queue_workflow_chain_events_total{kind}` - `queue_workflow_batch_events_total{kind}` @@ -178,8 +190,7 @@ After GA (target): ## Cross-References -- `observability.go` (queue runtime event kinds and `queue.Event`) -- `runtime.go` (workflow aliases: `queue.WorkflowEvent`, `queue.WorkflowEventKind`) -- `bus/events.go` (underlying workflow event schema) +- `observability.go` (canonical `queue.Event`, event layers, kinds, and observer helpers) +- `runtime.go` (one `queue.WithObserver(...)` attachment path and deprecated workflow aliases) - `docs/ops-alerts.md` (dashboard/alert baseline) - `docs/runbooks/` (incident response) diff --git a/docs/ops-alerts.md b/docs/ops-alerts.md index 8a1b8c5..c1ff73a 100644 --- a/docs/ops-alerts.md +++ b/docs/ops-alerts.md @@ -31,7 +31,7 @@ Why: - Jobs started/sec - Jobs succeeded/sec - Jobs failed/sec -- Retries scheduled/sec +- Retry-attempt deliveries/sec (`process_retried`; infrastructure redelivery may repeat a numbered attempt) Why: @@ -68,11 +68,13 @@ Track counts/rates for runtime events (from `queue.Observer`), including: - `process_retried` - `process_archived` - `republish_failed` +- `settlement_failed` - `process_recovered` Why: - `republish_failed` exposes hidden delay/retry republish churn +- `settlement_failed` exposes broker acknowledgements or deletions that left original work redeliverable - `process_recovered` shows DB stale-processing recovery activity ## Minimum Alerts (Baseline) @@ -129,7 +131,11 @@ Suggested default posture: - warning on any sustained non-zero rate - critical if rising during backlog growth or broker instability -### E. Stale Processing Recovery Alert (DB Backends) +### E. Settlement Failure Alert + +Fire when `settlement_failed` is non-zero. Treat a sustained rate as duplicate-delivery and broker-health risk because handler or replacement work may already have completed while the original remained unsettled. + +### F. Stale Processing Recovery Alert (DB Backends) Fire when `process_recovered` rate exceeds expected baseline. @@ -137,7 +143,7 @@ Why: - A rising rate indicates worker crashes, DB finalization failures, or unhealthy pods causing stale `processing` rows -### F. Worker Crash Loop / Restart Alert +### G. Worker Crash Loop / Restart Alert Fire when worker restart count exceeds threshold over a short window. @@ -159,6 +165,7 @@ At minimum, document fields/labels for: Events that must be covered: - `republish_failed` +- `settlement_failed` - `process_recovered` See also: diff --git a/docs/production-config.md b/docs/production-config.md index 46cdf3b..4e129fd 100644 --- a/docs/production-config.md +++ b/docs/production-config.md @@ -6,7 +6,7 @@ Treat these values as starting points. Tune them using workload measurements, in ## General Principles -- Handlers should be idempotent (at-least-once delivery semantics). +- Handlers should be idempotent. Durable backends may redeliver around settlement failures; Core NATS is currently ephemeral and does not provide an at-least-once queue guarantee. - Start with conservative concurrency, then increase while watching: - processing latency - retry/failure rate @@ -35,6 +35,14 @@ Watch for: - downstream rate limiting - DB connection pool exhaustion (DB backends) +### Shutdown deadlines + +Shutdown drains operations and handler work already admitted to the current process. For local runtimes that includes accepted delayed workflow descendants; durable database or broker backlog remains stored for another worker or process restart. Supply a context deadline that covers the longest in-flight handler, local accepted delay, and settlement time. If the deadline expires, shutdown returns the context error; call it again with a fresh context to continue retryable cleanup. + +Root operations admitted before draining hold a lifecycle lease, while new public work is rejected after draining begins. Continuations may cross that gate only while an active handler owns the same runtime-scoped permit. + +Successful shutdown is terminal for that queue instance. Construct a new queue to restart processing; repeated shutdown calls remain idempotent, while a failed cleanup attempt can be retried with a fresh context. + ## Job Retry / Backoff Guidance ### Retries (`Job.Retry(n)`) @@ -88,14 +96,57 @@ Starting points: - keep worker concurrency modest initially (`1-4` per process) - monitor query latency and stale-processing recovery events (`process_recovered`) -Important DB recovery knobs (`queue.Config`): +Important DB recovery settings (`mysqlqueue.Config`, `postgresqueue.Config`, or `sqlitequeue.Config`, passed to the corresponding `NewWithConfig` constructor): + +- `ProcessingRecoveryGrace` (default `2s` when non-positive) + - added to a job's positive timeout before its `processing` row is eligible for stale recovery + - increase only if timed-out handlers may still be settling when recovery begins +- `ProcessingLeaseNoTimeout` (default `5m` when non-positive) + - minimum processing age before a job with no positive timeout is eligible for stale recovery + - increase for longer-running jobs that intentionally omit a timeout + +The same fields are available on the advanced `queue.DatabaseConfig`; they are not fields of `queue.Config`. + +Every SQL processing claim has an opaque generation ID. When infrastructure keeps the row pending for the same numbered attempt, SQL normally retains inherited recovery provenance. If that delivery durably owns a new built-in workflow transition receipt and subsequent workflow infrastructure still requires redelivery, the workflow engine marks application state committed on the delivery-settlement boundary. SQL then retains the current generation rather than the older inherited generation, so the next claimant can match the receipt that actually owns the transition. The signal does not mean the queue row settled, observer callbacks ran, or continuation work completed. An application retry advances the attempt and clears every old link. Generation provenance is not an application error, does not redefine the admin-visible `last_error` field, and cannot be forged through error text. + +For receipt-backed workflow recovery, pass a built-in `queue.NewSQLStore(...)` directly to `queue.WithStore`. The store writes `bus_workflow_transition_receipts` in the same transaction as a chain-node or batch-member outcome transition. Each row includes non-null integer `receipt_version` and `event_schema_version` columns, both currently `1`. The first versions the durable ownership record; the second pins the shared observer event contract and is independent from the workflow-envelope protocol version. A worker fails recovery closed on either unknown value: it returns an uncommitted error and does not acknowledge the row, execute application code, mark application state committed, or emit reconstructed facts. On stale recovery, logical receipt proof requires a complete valid persisted owner, including a nonnegative owner attempt; matching workflow kind/ID/member/incarnation and dispatch; nonempty current dispatch/`JobID`; and matching immutable job content/fingerprint. The current attempt is physical provenance and may differ from the owner or be negative. Chain physical `JobID` may also differ; batch `JobID` is its logical member key and must match. That proof suppresses handler replay. Successful member or aggregate fact reconstruction additionally requires exact `RecoveredGenerationID`, current attempt, and physical `JobID` ownership; a physical nonowner publishes no recovered success facts. `BatchCompleted` also requires that exact fact owner to own a validated aggregate terminal transition; aggregate state alone is insufficient for both one-member and multi-member batches. Built-in memory receipts survive only within the current process. + +MySQL and PostgreSQL serialize the parent batch row after a member claim, and the memory store uses its existing mutex, so only one concurrent false-to-true parent transition owns the aggregate receipt and terminal effects. Every SQL aggregate row must own completion; cancellation must own a failed outcome; and a row naming the requested member must match that member receipt's workflow incarnation, complete owner, and outcome. Recovery also requires its completion and cancellation flags to agree with live terminal state. Inconsistency fails uncommitted before acknowledgement, handlers, callbacks, state-commit signaling, or facts. `TestSQLStoreBatchAggregateOwnershipMismatchFailsClosed`, `TestSQLStoreBatchAggregateIncarnationMismatchFailsClosed`, and `TestBatchRecoveryRejectsInvalidAggregateReceiptShape` cover those fail-closed branches. Real twelve-worker fail-fast races on both server dialects verify twelve member receipts, one aggregate receipt, and one failed/cancelled terminal fact pair. A separate SQLite two-member recovery scenario verifies only the completing receipt owner receives `BatchCompleted` after finalization failure. + +When validated durable state proves a recovered chain predecessor succeeded and still points to its immediate successor, the runtime re-dispatches that successor without re-running the predecessor. Exact recovered generation, attempt, and physical `JobID` ownership may also reconstruct predecessor facts. A missing receipt, a custom/decorated store without receipt capability, or a logically valid receipt with different/legacy generation, different attempt, or different physical `JobID` dispatches only the successor and repeats no predecessor facts or callbacks. Supported success receipts are logically validated first: cancellation is invalid and completion must exactly match final-node position; corruption fails uncommitted with no dispatch or effects. A rejected successor enqueue is also uncommitted so recovery can retry. Treat this as at-least-once continuation recovery, not exactly-once dispatch: the runtime cannot distinguish a missing successor from one already queued but not yet progressed, so a duplicate enqueue is possible. Once the successor progresses or the chain becomes terminal, predecessor recovery does not enqueue it again. + +For a receipt-backed terminal chain failure, logical receipt proof returns the first persisted `ChainState.Failure` as permanent across exact, different, or legacy recovered-generation provenance and across different attempts or physical `JobID`s; an empty persisted cause becomes a permanent diagnostic. It does not execute the handler, Catch/Finally callbacks, or logical failure facts again. Invalid version, incomplete owner, logical dispatch/job-content mismatch, workflow incarnation, outcome, or terminal flags fail closed with an uncommitted outcome; physical nonownership alone does not. Built-in `FailChain` preserves the first terminal cause, so direct store callers can no longer use a later failure call to replace it; retain secondary diagnostics separately. Receipt-absent legacy rows and custom/decorated stores may still execute application code once to preserve terminal physical classification, while duplicate workflow facts/callbacks remain suppressed. A real SQLite fixture proves repeated archive failure retains attempt zero, receipt lineage, and cause before a later `dead` settlement at attempt one; equivalent MySQL/PostgreSQL failed-chain fixtures remain open. -- `DatabaseProcessingRecoveryGrace` - - grace period before reclaiming stale `processing` jobs - - start with default unless you have proven false-positive recovery under your handler latencies -- `DatabaseProcessingLeaseNoTimeout` - - fallback lease for jobs without explicit timeout - - increase for very long-running jobs if you observe premature stale recovery +Receipt-backed failed batch recovery uses a generic permanent cause because the original application error is not persisted in batch state. This applies when the current delivery has a different or negative attempt, different recovered generation, or legacy provenance: the logically valid receipt still proves application failure and suppresses replay, while no replacement failure/member facts are emitted. Batch `JobID` must still match the logical member. SQL therefore drives that physical delivery to its terminal `dead` archive rather than deleting it as success, without inventing the original cause or executing the handler again. A logically valid successful batch duplicate settles without reconstructed facts under the same physical-nonowner conditions. + +If a recovered SQL delivery exhausts its bounded finalization retries, the driver makes one fenced best-effort repair when the delivery did not commit new application state. A successful repair preserves the numbered attempt, restores the inherited receipt-owner generation, clears `processing_started_at`, returns the row to `pending`, and delays reclaim by the greater of the polling interval and finalization-retry floor. Real SQLite success, failed-chain, and failed-batch scenarios force at least two recovery finalization failures before a later delete or archive succeeds without handler replay. A rejected or unavailable repair is joined into `settlement_failed`; it is not a stronger durability guarantee. + +Decorating a built-in store, supplying an application-defined store, or using the retained raw bus construction path hides the private receipt and response-local `claimedNow` capabilities. Those routes retain the public `WorkflowStore` contract and, when implemented, `WorkflowOutcomeStore` first-writer semantics, but they have weaker duplicate-effect and exact fact-recovery guarantees. + +Transition receipts are not settlement outboxes or durable continuation intents. They do not retain observer callbacks, `Progress` closures, successor-enqueue acceptance, callback dispatch, or batch fan-out, and they cannot repair a process exit after queue finalization removed the row. Keep handlers idempotent and treat observation as best-effort until the separate durable outbox and continuation/callback-intent work lands. Custom and decorated stores still need explicit fallback contracts before they can claim the built-in receipt guarantees. + +Schema migration ownership: + +- queue-table startup migrations are enabled by default; set `DisableAutoMigrate: true` on `sqlitequeue.Config`, `mysqlqueue.Config`, `postgresqueue.Config`, or the advanced `queue.DatabaseConfig` when deployment tooling owns queue tables +- in managed queue mode, readiness and startup perform no queue DDL and require `queue_jobs` and `queue_unique_locks` to be base-table relations, including PostgreSQL partitioned tables, with every column the current runtime reads or writes; empty, view-backed, and incomplete schemas fail before workers poll +- managed queue validation checks presence, not write permissions, exact SQL types, constraints, or performance indexes; install the complete dialect-correct canonical schema rather than treating a successful check as a schema-lint or query-performance guarantee +- a failed managed queue check is retryable on the same runtime after deployment tooling installs or repairs the schema; canonical preprovisioned schemas are exercised through readiness, uniqueness, dispatch, and consumption on SQLite, MySQL, and PostgreSQL +- workflow-store migration policy is constructor-selected: `queue.NewSQLStore` preserves legacy migration-on-first-use behavior, including when compatibility field `SQLStoreConfig.AutoMigrate` is false; `queue.NewSQLStoreWithManagedSchema` performs no workflow DDL +- when deployment tooling owns workflow schema, create every dialect-correct workflow table before constructing the store with `queue.NewSQLStoreWithManagedSchema`; `bus_workflow_transition_receipts` must include non-null `receipt_version` and `event_schema_version` integer columns as well as its ownership fields +- keep either migration-on-start default only when the runtime identity has DDL permission and concurrent application startup is coordinated +- a failed queue migration can be retried by a later `Start`; a workflow-store first-use migration failure remains attached to that store instance, so correct the lock, permission, or connectivity issue and construct a new store +- a wholly fresh MySQL auto-schema creates workflow/member and receipt identities as `VARBINARY(255)` and callback keys as `VARBINARY(512)` +- when legacy MySQL state tables exist but the receipt table is missing, ordinary `queue.NewSQLStore` validates their `VARBINARY` keys and derives receipt `workflow_id` from the larger effective chain-or-batch ID capacity and `member_id` from the larger chain-node-or-batch-job capacity; `TestWorkflowStoreIntegration_MySQLAutoMigratesMissingReceiptAtLegacyWidths` proves the real 512/512 upgrade path with long identities +- automatic startup never alters an existing receipt table; its live widths participate in capacity discovery, so quiesce workflow writers and use an operator-managed migration before starting a new store when that table is missing columns, uses incompatible identity types, or is narrower than the capacities the deployment must retain +- a derived receipt primary key can exceed the MySQL server's indexed-key budget when established identity widths are extreme; startup then fails with both derived widths and schema-first guidance instead of narrowing or altering live tables, and operators must precreate a compatible indexed receipt schema or explicitly migrate supported identity limits and existing data +- all MySQL workflow identity columns, including receipt identities, must use byte-exact `VARBINARY`; incompatible types fail schema-capacity discovery, while a managed SQLite/PostgreSQL schema encounters a missing receipt table when a receipt operation first runs +- the current built-in pruner removes transition receipts with their terminal parent workflow +- use a schema-first, quiescent worker rollout for the receipt table until cross-dialect migration concurrency evidence is complete +- for rollback, quiesce new workers before starting old binaries and leave the additive table in place; old code ignores it, dropping it destroys provenance, and an old pruner can leave receipt rows that it does not know how to delete +- real SQLite, MySQL, and PostgreSQL finalization-failure tests cover auto-schema receipt creation, supported-version read/write, exact-owner recovery, and no handler re-execution; MySQL and PostgreSQL additionally cover concurrent aggregate ownership +- managed-schema rollout/rollback, real cross-dialect pruning, and physical commit/readback ambiguity when the database or context is unavailable still require separate gates + +Deriving a missing MySQL receipt table preserves established wider `VARBINARY` capacities without a source/API, configuration-file, workflow-envelope, or minimum-Go-version change. A pre-existing incompatible receipt table, or live widths whose derived primary key exceeds the server budget, still requires a persisted-schema and operational migration; quiesce workers and audit existing identities before changing those limits. When tuning: @@ -111,8 +162,9 @@ Good default when: Starting points: +- use Core NATS only where ephemeral broadcast delivery is acceptable; this adapter is not a durable competing-consumer work queue - use conservative concurrency while validating duplicate/ordering expectations -- confirm delayed/retry durability expectations against your workload (see capability matrix) +- do not rely on delayed/retry survival across disconnect, process shutdown, or periods with no subscriber ### SQS (`DriverSQS`) @@ -124,6 +176,7 @@ Starting points: - partition critical and bulk jobs into separate queues - validate handler duration vs SQS visibility timeout behavior in your environment +- size visibility for sequential processing of a received batch; workers do not yet extend visibility while handlers run - monitor duplicate deliveries and end-to-end latency under retries Operational note: @@ -142,6 +195,11 @@ Starting points: - validate restart/retry behavior and throughput under your expected publish/consume rate - watch connection/channel health and reconnect churn +Operational note: + +- workers do not currently reconnect after their delivery channel closes; replace the runtime after connection loss +- AMQP dialing and resource closure may exceed a lifecycle context deadline, so supervise shutdown at the process level + ## Queue Layout Recommendations Use multiple queues when workloads differ materially by: @@ -163,14 +221,16 @@ Run dedicated workers (or worker pools) per queue class when needed. At minimum, wire: -- runtime observer (`queue.Config.Observer`) -- workflow observer (`queue.WithObserver(...)`) when using chains/batches/callbacks +- one observer with `queue.WithObserver(...)` for queue, worker, and workflow events + +`queue.Config.Observer` remains a compatibility path and feeds the same event stream, but new applications should prefer the constructor option consistently across drivers. Track and alert on: - `process_failed` - `process_retried` - `republish_failed` +- `settlement_failed` - `process_recovered` (DB backends) See: diff --git a/docs/readme/main.go b/docs/readme/main.go index 01d43ca..f0cd0a6 100644 --- a/docs/readme/main.go +++ b/docs/readme/main.go @@ -81,7 +81,7 @@ type Example struct { } func includeInReadmeAPI(fd *FuncDoc) bool { - if fd.Package == "queue" && fd.Owner == "FakeQueue" && (fd.Name == "BusRegister" || fd.Name == "BusDispatch") { + if fd.Package == "queue" && fd.Owner == "FakeQueue" && (fd.Name == "BusRegister" || fd.Name == "BusDispatch" || fd.Name == "BusDispatchDirect") { return false } if fd.Package == "queue" && (fd.Group == "Queue Runtime" || fd.Group == "Driver Integration") { @@ -111,10 +111,6 @@ func parseFuncs(root string) ([]*FuncDoc, error) { category string } targets := []parseTarget{{dir: root, category: "Core"}} - queuefakeDir := filepath.Join(root, "queuefake") - if st, err := os.Stat(queuefakeDir); err == nil && st.IsDir() { - targets = append(targets, parseTarget{dir: queuefakeDir, category: "Testing"}) - } driverDirs, err := filepath.Glob(filepath.Join(root, "driver", "*queue")) if err != nil { return nil, err @@ -130,6 +126,11 @@ func parseFuncs(root string) ([]*FuncDoc, error) { if err != nil { return nil, err } + for _, function := range funcs { + if function.Package == "queue" && function.Group == "Testing" { + function.Category = "Testing" + } + } out = append(out, funcs...) } @@ -257,7 +258,7 @@ func parseFuncsInDir(dir string, category string) ([]*FuncDoc, error) { } for _, spec := range d.Specs { typeSpec, ok := spec.(*ast.TypeSpec) - if !ok { + if !ok || !ast.IsExported(typeSpec.Name.Name) { continue } iface, ok := typeSpec.Type.(*ast.InterfaceType) @@ -625,9 +626,9 @@ func renderAPI(funcs []*FuncDoc) string { if testingPkgs, ok := byCategoryPackageGroup["Testing"]; ok && len(testingPkgs) > 0 { var links []string - if queuefakeGroups, ok := testingPkgs["queuefake"]; ok { + if queueGroups, ok := testingPkgs["queue"]; ok { var fns []*FuncDoc - for _, groupFns := range queuefakeGroups { + for _, groupFns := range queueGroups { fns = append(fns, groupFns...) } sort.Slice(fns, func(i, j int) bool { @@ -663,6 +664,7 @@ func renderAPI(funcs []*FuncDoc) string { } else { buf.WriteString("## " + category + " API\n\n") if category == "Testing" { + buf.WriteString("`queue.NewFake` is a recording fake with its established `Dispatch(any) error` surface. Inject it where `*queue.FakeQueue` or that recording contract is accepted; it is not a drop-in `*queue.Queue`.\n\n") buf.WriteString("Examples in this section assume they are used inside tests and `t` is a `*testing.T` (or `testing.TB`).\n\n") } } @@ -676,7 +678,7 @@ func renderAPI(funcs []*FuncDoc) string { if category == "Core" && pkg == "queue" { // Preserve original root API detail layout (no extra package heading). } else if category == "Testing" { - // Testing API currently renders queuefake only; skip redundant package heading. + // The canonical testing API is rooted in queue, so its section needs no redundant package heading. } else { buf.WriteString("### " + packageCategoryLabel(category, pkg) + "\n\n") } diff --git a/docs/readme/testcounts/integration_count.json b/docs/readme/testcounts/integration_count.json new file mode 100644 index 0000000..d997d7b --- /dev/null +++ b/docs/readme/testcounts/integration_count.json @@ -0,0 +1,5 @@ +{ + "count": 631, + "source_hash": "sha256:886715b5a5184f7770cc7e723aa6cd424cd882478ea37d11f6b756fa696264c0", + "backend_scope": "all" +} diff --git a/docs/readme/testcounts/main.go b/docs/readme/testcounts/main.go index 5563215..6a12633 100644 --- a/docs/readme/testcounts/main.go +++ b/docs/readme/testcounts/main.go @@ -1,10 +1,12 @@ -//go:build ignore -// +build ignore +//go:build ignore || testcounts +// +build ignore testcounts package main import ( "bytes" + "context" + "crypto/sha256" "encoding/json" "fmt" "go/ast" @@ -16,36 +18,61 @@ import ( "regexp" "sort" "strings" + "time" ) const ( - testCountStart = "" - testCountEnd = "" + testCountStart = "" + testCountEnd = "" + integrationCountTimeout = 30 * time.Minute + integrationManifestName = "integration_count.json" ) +// Counts records the executed unit and integration test counts rendered in the README. type Counts struct { Unit int Integration int } +type integrationCountManifest struct { + Count int `json:"count"` + SourceHash string `json:"source_hash"` + BackendScope string `json:"backend_scope"` +} + +// main updates README test-count badges from executed evidence or a verified integration manifest. func main() { if err := run(); err != nil { fmt.Println("Error:", err) os.Exit(1) } - fmt.Println("✔ Test badges updated from executed test runs") + fmt.Println(testCountSuccessMessage(os.Getenv("TESTCOUNT_USE_INTEGRATION_MANIFEST") == "1")) +} + +// testCountSuccessMessage describes the evidence source used for a successful badge update. +func testCountSuccessMessage(useIntegrationManifest bool) string { + if useIntegrationManifest { + return "✔ Test badges updated from unit execution and verified integration evidence" + } + return "✔ Test badges updated from executed unit and integration runs" } +// run calculates test evidence and updates the checked-in README and integration manifest. func run() error { root, err := findRoot() if err != nil { return err } + return runAtRoot(root) +} - integrationRoot := filepath.Join(root, "integration") - integrationNames, err := integrationTopLevelTests(integrationRoot) - if err != nil { - return fmt.Errorf("integration top-level tests: %w", err) +// runAtRoot calculates test evidence for one repository root and updates its generated artifacts. +func runAtRoot(root string) error { + useIntegrationManifest := os.Getenv("TESTCOUNT_USE_INTEGRATION_MANIFEST") == "1" + if !useIntegrationManifest { + if err := validateFullIntegrationScope(os.Getenv("INTEGRATION_BACKEND")); err != nil { + return err + } } unitCount, err := countRunEvents(root, nil) @@ -53,21 +80,51 @@ func run() error { return fmt.Errorf("count unit test runs: %w", err) } - integrationCount, err := countIntegrationRunEvents(integrationRoot, integrationNames) + integrationRoot := filepath.Join(root, "integration") + integrationSourceHash, err := integrationTestSourceHash(root, integrationRoot) if err != nil { - fmt.Printf("warn: integration executed count unavailable (%v); leaving integration badge unchanged if present\n", err) - integrationCount = -1 + return fmt.Errorf("hash integration test sources: %w", err) + } + manifestPath := filepath.Join(root, "docs", "readme", "testcounts", integrationManifestName) + + var integrationCount int + if useIntegrationManifest { + manifest, manifestErr := loadIntegrationCountManifest(manifestPath, integrationSourceHash) + if manifestErr != nil { + return manifestErr + } + integrationCount = manifest.Count + } else { + rootIntegrationNames, namesErr := integrationTopLevelTests(root) + if namesErr != nil { + return fmt.Errorf("root integration top-level tests: %w", namesErr) + } + rootIntegrationCount, countErr := countRunEvents(root, rootIntegrationNames) + if countErr != nil { + return fmt.Errorf("count root integration test runs: %w", countErr) + } + integrationNames, namesErr := integrationTopLevelTests(integrationRoot) + if namesErr != nil { + return fmt.Errorf("integration top-level tests: %w", namesErr) + } + integrationCount, err = countIntegrationRunEvents(integrationRoot, integrationNames) + if err != nil { + return fmt.Errorf("count integration test runs: %w", err) + } + integrationCount += rootIntegrationCount + if err := writeIntegrationCountManifest(manifestPath, integrationCountManifest{ + Count: integrationCount, + SourceHash: integrationSourceHash, + BackendScope: "all", + }); err != nil { + return err + } } readmePath := filepath.Join(root, "README.md") data, err := os.ReadFile(readmePath) if err != nil { - return err - } - - existing, _ := existingCountsFromReadme(string(data)) - if integrationCount < 0 { - integrationCount = existing.Integration + return fmt.Errorf("read README: %w", err) } out, err := updateTestsSection(string(data), Counts{ @@ -75,12 +132,125 @@ func run() error { Integration: integrationCount, }) if err != nil { - return err + return fmt.Errorf("update README test counts: %w", err) } - return os.WriteFile(readmePath, []byte(out), 0o644) + return writeTestCountREADME(readmePath, out) } +// writeTestCountREADME persists the rendered badge block with a contextual failure. +func writeTestCountREADME(path, contents string) error { + if err := os.WriteFile(path, []byte(contents), 0o644); err != nil { + return fmt.Errorf("write README test counts: %w", err) + } + return nil +} + +// loadIntegrationCountManifest returns full-run evidence only when it covers the current integration sources. +func loadIntegrationCountManifest(path, expectedSourceHash string) (integrationCountManifest, error) { + data, err := os.ReadFile(path) + if err != nil { + return integrationCountManifest{}, fmt.Errorf("read integration count manifest: %w", err) + } + var manifest integrationCountManifest + if err := json.Unmarshal(data, &manifest); err != nil { + return integrationCountManifest{}, fmt.Errorf("decode integration count manifest: %w", err) + } + if manifest.Count <= 0 { + return integrationCountManifest{}, fmt.Errorf("integration count manifest has nonpositive count %d", manifest.Count) + } + if manifest.SourceHash == "" { + return integrationCountManifest{}, fmt.Errorf("integration count manifest has empty source hash") + } + if manifest.BackendScope != "all" { + return integrationCountManifest{}, fmt.Errorf("integration count manifest has backend scope %q, want %q", manifest.BackendScope, "all") + } + if manifest.SourceHash != expectedSourceHash { + return integrationCountManifest{}, fmt.Errorf("integration test sources changed: manifest has %s, current sources have %s; run `cd docs && go run ./readme/testcounts/main.go` with integration services available", manifest.SourceHash, expectedSourceHash) + } + return manifest, nil +} + +// validateFullIntegrationScope prevents a partial backend selection from replacing full-suite evidence. +func validateFullIntegrationScope(value string) error { + scope := strings.ToLower(strings.TrimSpace(value)) + if scope == "" || scope == "all" { + return nil + } + return fmt.Errorf("full integration count requires INTEGRATION_BACKEND=all or unset, got %q", value) +} + +// writeIntegrationCountManifest records the source identity behind a full integration count. +func writeIntegrationCountManifest(path string, manifest integrationCountManifest) error { + // The fixed scalar fields cannot contain values unsupported by encoding/json. + data, _ := json.MarshalIndent(manifest, "", " ") + data = append(data, '\n') + if err := os.WriteFile(path, data, 0o644); err != nil { + return fmt.Errorf("write integration count manifest: %w", err) + } + return nil +} + +// integrationTestSourceHash fingerprints the integration module plus root integration-tagged sources and module inputs. +func integrationTestSourceHash(root, integrationRoot string) (string, error) { + type sourceFile struct { + path string + src []byte + } + var sources []sourceFile + integrationRoot = filepath.Clean(integrationRoot) + integrationInfo, err := os.Stat(integrationRoot) + if err != nil { + return "", fmt.Errorf("inspect integration module: %w", err) + } + if !integrationInfo.IsDir() { + return "", fmt.Errorf("integration module path is not a directory: %s", integrationRoot) + } + err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + if info.Name() == ".git" || info.Name() == "vendor" { + return filepath.SkipDir + } + if path != root && path != integrationRoot && fileExists(filepath.Join(path, "go.mod")) { + return filepath.SkipDir + } + return nil + } + // Walk only yields descendants of root, so the relative path cannot cross filesystem volumes. + relativePath, _ := filepath.Rel(root, path) + relativePath = filepath.ToSlash(relativePath) + inIntegrationModule := path == integrationRoot || strings.HasPrefix(path, integrationRoot+string(filepath.Separator)) + isModuleInput := relativePath == "go.mod" || relativePath == "go.sum" || relativePath == "integration/go.mod" || relativePath == "integration/go.sum" + if !strings.HasSuffix(path, ".go") && !isModuleInput { + return nil + } + src, err := os.ReadFile(path) + if err != nil { + return err + } + if !inIntegrationModule && !isModuleInput && !hasIntegrationBuildTag(src) { + return nil + } + sources = append(sources, sourceFile{path: relativePath, src: src}) + return nil + }) + if err != nil { + return "", err + } + sort.Slice(sources, func(i, j int) bool { return sources[i].path < sources[j].path }) + hash := sha256.New() + for _, source := range sources { + _, _ = fmt.Fprintf(hash, "%s%c", source.path, byte(0)) + _, _ = hash.Write(source.src) + _, _ = hash.Write([]byte{0}) + } + return fmt.Sprintf("sha256:%x", hash.Sum(nil)), nil +} + +// countRunEvents executes the selected Go tests and counts their run events. func countRunEvents(root string, integrationPrefixes map[string]struct{}) (int, error) { args := []string{"test", "./...", "-run", "Test", "-count=1", "-json"} if integrationPrefixes != nil { @@ -134,6 +304,7 @@ func countRunEvents(root string, integrationPrefixes map[string]struct{}) (int, return total, nil } +// countIntegrationRunEvents executes integration-tagged tests within a bounded count-generation window. func countIntegrationRunEvents(integrationRoot string, integrationPrefixes map[string]struct{}) (int, error) { if integrationPrefixes == nil || len(integrationPrefixes) == 0 { return 0, nil @@ -144,14 +315,24 @@ func countIntegrationRunEvents(integrationRoot string, integrationPrefixes map[s } args := []string{"test", "-tags=integration", "./...", "-run", runPattern, "-count=1", "-json"} - cmd := exec.Command("go", args...) + ctx, cancel := context.WithTimeout(context.Background(), integrationCountTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, "go", args...) cmd.Dir = integrationRoot + cmd.Env = append(cmd.Environ(), + "INTEGRATION_BACKEND=all", + "RUN_CHAOS=0", + "RUN_SOAK=0", + ) var out bytes.Buffer cmd.Stdout = &out cmd.Stderr = &out if err := cmd.Run(); err != nil { + if ctx.Err() != nil { + return 0, fmt.Errorf("go %s (in %s): %w", strings.Join(args, " "), integrationRoot, ctx.Err()) + } return 0, fmt.Errorf("go %s (in %s): %w\n%s", strings.Join(args, " "), integrationRoot, err, out.String()) } @@ -179,6 +360,7 @@ func countIntegrationRunEvents(integrationRoot string, integrationPrefixes map[s return total, nil } +// buildTopLevelRunPattern builds an exact pattern for selected top-level tests and their subtests. func buildTopLevelRunPattern(names map[string]struct{}) string { if len(names) == 0 { return "" @@ -188,10 +370,10 @@ func buildTopLevelRunPattern(names map[string]struct{}) string { for _, k := range keys { parts = append(parts, regexp.QuoteMeta(k)) } - // Match the top-level integration test and any subtests beneath it. return "^(" + strings.Join(parts, "|") + ")(/.*)?$" } +// integrationTopLevelTests discovers integration-tagged top-level test functions. func integrationTopLevelTests(root string) (map[string]struct{}, error) { names := map[string]struct{}{} @@ -204,6 +386,9 @@ func integrationTopLevelTests(root string) (map[string]struct{}, error) { if name == ".git" || name == "vendor" { return filepath.SkipDir } + if path != root && fileExists(filepath.Join(path, "go.mod")) { + return filepath.SkipDir + } return nil } if !strings.HasSuffix(path, "_test.go") { @@ -241,6 +426,7 @@ func integrationTopLevelTests(root string) (map[string]struct{}, error) { return names, nil } +// updateTestsSection replaces only the generated README badge block. func updateTestsSection(readme string, counts Counts) (string, error) { start := strings.Index(readme, testCountStart) end := strings.Index(readme, testCountEnd) @@ -264,22 +450,7 @@ func updateTestsSection(readme string, counts Counts) (string, error) { return before + leading + strings.Join(lines, "\n") + "\n" + after, nil } -var ( - unitBadgeCountRE = regexp.MustCompile(`unit_tests-(\d+)-`) - integrationBadgeCountRE = regexp.MustCompile(`integration_tests-(\d+)-`) -) - -func existingCountsFromReadme(readme string) (Counts, error) { - var c Counts - if m := unitBadgeCountRE.FindStringSubmatch(readme); len(m) == 2 { - fmt.Sscanf(m[1], "%d", &c.Unit) - } - if m := integrationBadgeCountRE.FindStringSubmatch(readme); len(m) == 2 { - fmt.Sscanf(m[1], "%d", &c.Integration) - } - return c, nil -} - +// hasIntegrationBuildTag reports whether a source file opts into the integration suite. func hasIntegrationBuildTag(src []byte) bool { lines := strings.Split(string(src), "\n") for _, line := range lines { @@ -300,6 +471,7 @@ func hasIntegrationBuildTag(src []byte) bool { return false } +// findRoot locates the queue module from any supported generator working directory. func findRoot() (string, error) { wd, _ := os.Getwd() candidates := []string{wd, filepath.Join(wd, ".."), filepath.Join(wd, "..", ".."), filepath.Join(wd, "..", "..", "..")} @@ -312,11 +484,13 @@ func findRoot() (string, error) { return "", fmt.Errorf("could not find project root from %s", wd) } +// fileExists reports whether a generator input path exists. func fileExists(p string) bool { _, err := os.Stat(p) return err == nil } +// _sortedKeys returns map keys in deterministic lexical order. func _sortedKeys(m map[string]struct{}) []string { out := make([]string, 0, len(m)) for k := range m { diff --git a/docs/readme/testcounts/main_test.go b/docs/readme/testcounts/main_test.go new file mode 100644 index 0000000..18c57a7 --- /dev/null +++ b/docs/readme/testcounts/main_test.go @@ -0,0 +1,406 @@ +//go:build testcounts +// +build testcounts + +package main + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +// TestTestCountSuccessMessage verifies generator output identifies whether integration evidence was executed or verified. +func TestTestCountSuccessMessage(t *testing.T) { + if got := testCountSuccessMessage(false); !strings.Contains(got, "executed unit and integration") { + t.Fatalf("full success message = %q", got) + } + if got := testCountSuccessMessage(true); !strings.Contains(got, "verified integration evidence") { + t.Fatalf("manifest success message = %q", got) + } +} + +// TestRunAtRootGeneratesAndConsumesFullEvidence verifies both full and manifest-backed badge paths on a minimal multi-module repository. +func TestRunAtRootGeneratesAndConsumesFullEvidence(t *testing.T) { + root := newTestCountRepository(t) + t.Setenv("GOWORK", "off") + t.Setenv("TESTCOUNT_USE_INTEGRATION_MANIFEST", "") + t.Setenv("INTEGRATION_BACKEND", "all") + t.Setenv("RUN_CHAOS", "1") + t.Setenv("RUN_SOAK", "1") + + if err := runAtRoot(root); err != nil { + t.Fatalf("generate full evidence: %v", err) + } + manifestPath := filepath.Join(root, "docs", "readme", "testcounts", integrationManifestName) + data, err := os.ReadFile(manifestPath) + if err != nil { + t.Fatalf("read generated manifest: %v", err) + } + if !strings.Contains(string(data), `"count": 3`) || !strings.Contains(string(data), `"backend_scope": "all"`) { + t.Fatalf("generated manifest = %s, want three all-backend run events", data) + } + assertTestCountBadges(t, root, 1, 3) + + t.Setenv("TESTCOUNT_USE_INTEGRATION_MANIFEST", "1") + t.Setenv("INTEGRATION_BACKEND", "null") + t.Chdir(root) + if err := run(); err != nil { + t.Fatalf("consume verified manifest: %v", err) + } + assertTestCountBadges(t, root, 1, 3) +} + +// TestRunAtRootRejectsPartialFullEvidence verifies scope validation happens before any test execution or artifact write. +func TestRunAtRootRejectsPartialFullEvidence(t *testing.T) { + root := newTestCountRepository(t) + t.Setenv("TESTCOUNT_USE_INTEGRATION_MANIFEST", "") + t.Setenv("INTEGRATION_BACKEND", "redis") + err := runAtRoot(root) + if err == nil || !strings.Contains(err.Error(), "requires INTEGRATION_BACKEND=all") { + t.Fatalf("partial full evidence error = %v", err) + } + if _, err := os.Stat(filepath.Join(root, "docs", "readme", "testcounts", integrationManifestName)); !os.IsNotExist(err) { + t.Fatalf("partial generation wrote a manifest: %v", err) + } +} + +// TestRunAtRootRejectsInvalidManifest verifies the manifest-backed path propagates evidence validation failures. +func TestRunAtRootRejectsInvalidManifest(t *testing.T) { + root := newTestCountRepository(t) + t.Setenv("GOWORK", "off") + t.Setenv("TESTCOUNT_USE_INTEGRATION_MANIFEST", "1") + t.Setenv("INTEGRATION_BACKEND", "null") + manifestPath := filepath.Join(root, "docs", "readme", "testcounts", integrationManifestName) + writeTestFile(t, manifestPath, `{"count":3,"source_hash":"sha256:stale","backend_scope":"all"}`) + err := runAtRoot(root) + if err == nil || !strings.Contains(err.Error(), "integration test sources changed") { + t.Fatalf("invalid manifest error = %v", err) + } +} + +// TestRunAtRootReportsEvidenceFailures verifies each generation boundary preserves a specific diagnostic. +func TestRunAtRootReportsEvidenceFailures(t *testing.T) { + tests := []struct { + name string + mutate func(*testing.T, string) + want string + }{ + { + name: "unit execution", + mutate: func(t *testing.T, root string) { + writeTestFile(t, filepath.Join(root, "unit_test.go"), "package testcounts\n\nimport \"testing\"\n\nfunc TestUnit(t *testing.T) { missing() }\n") + }, + want: "count unit test runs", + }, + { + name: "missing integration module", + mutate: func(t *testing.T, root string) { + if err := os.Rename(filepath.Join(root, "integration"), filepath.Join(root, "integration-missing")); err != nil { + t.Fatalf("hide integration module: %v", err) + } + }, + want: "hash integration test sources", + }, + { + name: "root discovery", + mutate: func(t *testing.T, root string) { + writeTestFile(t, filepath.Join(root, "tagged_integration_test.go"), "//go:build integration\n\npackage testcounts\n\nfunc TestBroken(") + }, + want: "root integration top-level tests", + }, + { + name: "root execution", + mutate: func(t *testing.T, root string) { + writeTestFile(t, filepath.Join(root, "tagged_integration_test.go"), "//go:build integration\n\npackage testcounts\n\nimport \"testing\"\n\nfunc TestRootIntegration(t *testing.T) { missing() }\n") + }, + want: "count root integration test runs", + }, + { + name: "integration discovery", + mutate: func(t *testing.T, root string) { + writeTestFile(t, filepath.Join(root, "integration", "suite_integration_test.go"), "//go:build integration\n\npackage integration\n\nfunc TestBroken(") + }, + want: "integration top-level tests", + }, + { + name: "integration execution", + mutate: func(t *testing.T, root string) { + writeTestFile(t, filepath.Join(root, "integration", "suite_integration_test.go"), "//go:build integration\n\npackage integration\n\nimport \"testing\"\n\nfunc TestIntegration(t *testing.T) { missing() }\n") + }, + want: "count integration test runs", + }, + { + name: "manifest write", + mutate: func(t *testing.T, root string) { + path := filepath.Join(root, "docs", "readme", "testcounts", integrationManifestName) + if err := os.Mkdir(path, 0o755); err != nil { + t.Fatalf("create manifest directory collision: %v", err) + } + }, + want: "write integration count manifest", + }, + { + name: "README read", + mutate: func(t *testing.T, root string) { + if err := os.Remove(filepath.Join(root, "README.md")); err != nil { + t.Fatalf("remove README: %v", err) + } + }, + want: "read README", + }, + { + name: "README anchors", + mutate: func(t *testing.T, root string) { + writeTestFile(t, filepath.Join(root, "README.md"), "missing generated anchors\n") + }, + want: "update README test counts", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + root := newTestCountRepository(t) + t.Setenv("GOWORK", "off") + t.Setenv("TESTCOUNT_USE_INTEGRATION_MANIFEST", "") + t.Setenv("INTEGRATION_BACKEND", "all") + t.Setenv("RUN_CHAOS", "") + t.Setenv("RUN_SOAK", "") + test.mutate(t, root) + err := runAtRoot(root) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("generation error = %v, want containing %q", err, test.want) + } + }) + } +} + +// TestIntegrationTestSourceHashTracksExecutionInputs verifies the manifest identity follows code and module inputs but ignores documentation. +func TestIntegrationTestSourceHashTracksExecutionInputs(t *testing.T) { + root := t.TempDir() + integrationRoot := filepath.Join(root, "integration") + if err := os.MkdirAll(filepath.Join(root, "bus"), 0o755); err != nil { + t.Fatalf("create root package: %v", err) + } + if err := os.MkdirAll(integrationRoot, 0o755); err != nil { + t.Fatalf("create integration module: %v", err) + } + if err := os.MkdirAll(filepath.Join(root, ".git"), 0o755); err != nil { + t.Fatalf("create ignored metadata directory: %v", err) + } + if err := os.MkdirAll(filepath.Join(root, "nested"), 0o755); err != nil { + t.Fatalf("create ignored nested module: %v", err) + } + testPath := filepath.Join(root, "bus", "hooks_integration_test.go") + helperPath := filepath.Join(integrationRoot, "helper.go") + readmePath := filepath.Join(root, "README.md") + writeTestFile(t, testPath, "//go:build integration\n\npackage fixture\n\nfunc TestCounted() {}\n") + writeTestFile(t, helperPath, "package fixture\n\nvar backends = []string{\"one\"}\n") + writeTestFile(t, filepath.Join(root, "go.mod"), "module fixture\n") + writeTestFile(t, filepath.Join(integrationRoot, "go.mod"), "module fixture/integration\n") + writeTestFile(t, filepath.Join(root, "nested", "go.mod"), "module fixture/nested\n") + writeTestFile(t, filepath.Join(root, "nested", "ignored.go"), "package nested\n") + writeTestFile(t, readmePath, "initial docs\n") + + initial, err := integrationTestSourceHash(root, integrationRoot) + if err != nil { + t.Fatalf("initial source hash: %v", err) + } + writeTestFile(t, readmePath, "changed docs\n") + afterDocumentationChange, err := integrationTestSourceHash(root, integrationRoot) + if err != nil { + t.Fatalf("hash after documentation change: %v", err) + } + if afterDocumentationChange != initial { + t.Fatalf("documentation changed hash from %q to %q", initial, afterDocumentationChange) + } + + writeTestFile(t, helperPath, "package fixture\n\nvar backends = []string{\"one\", \"two\"}\n") + afterHelperChange, err := integrationTestSourceHash(root, integrationRoot) + if err != nil { + t.Fatalf("hash after helper change: %v", err) + } + if afterHelperChange == initial { + t.Fatalf("integration source hash remained %q after a helper change", initial) + } + + writeTestFile(t, testPath, "//go:build integration\n\npackage fixture\n\nfunc TestCountedChanged() {}\n") + afterRootTaggedChange, err := integrationTestSourceHash(root, integrationRoot) + if err != nil { + t.Fatalf("hash after root tagged change: %v", err) + } + if afterRootTaggedChange == afterHelperChange { + t.Fatalf("integration source hash remained %q after a root tagged change", afterHelperChange) + } +} + +// TestIntegrationTestSourceHashRejectsInvalidInputs verifies missing modules and unreadable source entries fail closed. +func TestIntegrationTestSourceHashRejectsInvalidInputs(t *testing.T) { + root := t.TempDir() + integrationPath := filepath.Join(root, "integration") + writeTestFile(t, integrationPath, "not a directory") + if _, err := integrationTestSourceHash(root, integrationPath); err == nil || !strings.Contains(err.Error(), "not a directory") { + t.Fatalf("file integration path error = %v", err) + } + + root = t.TempDir() + integrationPath = filepath.Join(root, "integration") + if err := os.MkdirAll(integrationPath, 0o755); err != nil { + t.Fatalf("create integration directory: %v", err) + } + if err := os.Symlink(filepath.Join(root, "missing-target"), filepath.Join(integrationPath, "broken.go")); err != nil { + t.Fatalf("create broken source link: %v", err) + } + if _, err := integrationTestSourceHash(root, integrationPath); err == nil { + t.Fatal("broken integration source link was accepted") + } +} + +// TestIntegrationTopLevelTestsSkipsNestedModules verifies root evidence cannot double-count the dedicated integration module. +func TestIntegrationTopLevelTestsSkipsNestedModules(t *testing.T) { + root := t.TempDir() + nested := filepath.Join(root, "integration") + if err := os.MkdirAll(nested, 0o755); err != nil { + t.Fatalf("create nested module: %v", err) + } + writeTestFile(t, filepath.Join(root, "root_test.go"), "//go:build integration\n\npackage fixture\n\nfunc TestRoot() {}\n") + writeTestFile(t, filepath.Join(nested, "go.mod"), "module fixture/integration\n") + writeTestFile(t, filepath.Join(nested, "nested_test.go"), "//go:build integration\n\npackage integration\n\nfunc TestNested() {}\n") + + names, err := integrationTopLevelTests(root) + if err != nil { + t.Fatalf("discover root tests: %v", err) + } + if _, ok := names["TestRoot"]; !ok { + t.Fatal("root integration test was not discovered") + } + if _, ok := names["TestNested"]; ok { + t.Fatal("nested integration module test was double-counted") + } +} + +// TestIntegrationCountManifestValidation verifies malformed, invalid, and stale evidence fails closed. +func TestIntegrationCountManifestValidation(t *testing.T) { + _, err := loadIntegrationCountManifest(filepath.Join(t.TempDir(), "missing.json"), "sha256:current") + if err == nil || !strings.Contains(err.Error(), "read integration count manifest") { + t.Fatalf("missing manifest error = %v, want read failure", err) + } + + tests := []struct { + name string + contents string + want string + }{ + {name: "malformed", contents: "{", want: "decode integration count manifest"}, + {name: "missing count", contents: `{"source_hash":"sha256:current","backend_scope":"all"}`, want: "nonpositive count"}, + {name: "null count", contents: `{"count":null,"source_hash":"sha256:current","backend_scope":"all"}`, want: "nonpositive count"}, + {name: "zero count", contents: `{"count":0,"source_hash":"sha256:current","backend_scope":"all"}`, want: "nonpositive count"}, + {name: "negative count", contents: `{"count":-1,"source_hash":"sha256:current","backend_scope":"all"}`, want: "nonpositive count"}, + {name: "empty hash", contents: `{"count":1,"source_hash":"","backend_scope":"all"}`, want: "empty source hash"}, + {name: "empty scope", contents: `{"count":1,"source_hash":"sha256:current"}`, want: "backend scope"}, + {name: "partial scope", contents: `{"count":1,"source_hash":"sha256:current","backend_scope":"redis"}`, want: "backend scope"}, + {name: "stale hash", contents: `{"count":1,"source_hash":"sha256:old","backend_scope":"all"}`, want: "integration test sources changed"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + path := filepath.Join(t.TempDir(), integrationManifestName) + writeTestFile(t, path, test.contents) + _, err := loadIntegrationCountManifest(path, "sha256:current") + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("load manifest error = %v, want containing %q", err, test.want) + } + }) + } +} + +// TestIntegrationCountManifestRoundTrip verifies full-run evidence remains readable by the unit-only guard. +func TestIntegrationCountManifestRoundTrip(t *testing.T) { + path := filepath.Join(t.TempDir(), integrationManifestName) + want := integrationCountManifest{Count: 612, SourceHash: "sha256:current", BackendScope: "all"} + if err := writeIntegrationCountManifest(path, want); err != nil { + t.Fatalf("write manifest: %v", err) + } + got, err := loadIntegrationCountManifest(path, want.SourceHash) + if err != nil { + t.Fatalf("load manifest: %v", err) + } + if got != want { + t.Fatalf("manifest = %#v, want %#v", got, want) + } + if err := writeIntegrationCountManifest(t.TempDir(), want); err == nil || !strings.Contains(err.Error(), "write integration count manifest") { + t.Fatalf("write manifest to directory error = %v, want write failure", err) + } +} + +// TestWriteTestCountREADME verifies successful persistence and contextual write failures. +func TestWriteTestCountREADME(t *testing.T) { + path := filepath.Join(t.TempDir(), "README.md") + if err := writeTestCountREADME(path, "rendered\n"); err != nil { + t.Fatalf("write README: %v", err) + } + data, err := os.ReadFile(path) + if err != nil || string(data) != "rendered\n" { + t.Fatalf("written README = %q, %v", data, err) + } + if err := writeTestCountREADME(t.TempDir(), "rendered\n"); err == nil || !strings.Contains(err.Error(), "write README test counts") { + t.Fatalf("write README directory error = %v", err) + } +} + +// TestValidateFullIntegrationScope verifies only the complete backend selection can refresh full-run evidence. +func TestValidateFullIntegrationScope(t *testing.T) { + for _, value := range []string{"", "all", " ALL "} { + if err := validateFullIntegrationScope(value); err != nil { + t.Fatalf("scope %q rejected: %v", value, err) + } + } + for _, value := range []string{"null", "redis", "redis,sqs"} { + if err := validateFullIntegrationScope(value); err == nil { + t.Fatalf("partial scope %q accepted", value) + } + } +} + +// writeTestFile writes a test fixture or fails the current test. +func writeTestFile(t *testing.T, path, contents string) { + t.Helper() + if err := os.WriteFile(path, []byte(contents), 0o644); err != nil { + t.Fatalf("write %s: %v", path, err) + } +} + +// newTestCountRepository creates a minimal root plus nested integration module with deterministic run counts. +func newTestCountRepository(t *testing.T) string { + t.Helper() + root := t.TempDir() + integrationRoot := filepath.Join(root, "integration") + manifestDir := filepath.Join(root, "docs", "readme", "testcounts") + if err := os.MkdirAll(integrationRoot, 0o755); err != nil { + t.Fatalf("create integration module: %v", err) + } + if err := os.MkdirAll(manifestDir, 0o755); err != nil { + t.Fatalf("create manifest directory: %v", err) + } + writeTestFile(t, filepath.Join(root, "go.mod"), "module example.com/testcounts\n\ngo 1.24.4\n") + writeTestFile(t, filepath.Join(root, "unit_test.go"), "package testcounts\n\nimport \"testing\"\n\nfunc TestUnit(t *testing.T) {}\n") + writeTestFile(t, filepath.Join(root, "tagged_integration_test.go"), "//go:build integration\n\npackage testcounts\n\nimport \"testing\"\n\nfunc TestRootIntegration(t *testing.T) {}\n") + writeTestFile(t, filepath.Join(integrationRoot, "go.mod"), "module example.com/testcounts/integration\n\ngo 1.24.4\n") + writeTestFile(t, filepath.Join(integrationRoot, "suite_integration_test.go"), "//go:build integration\n\npackage integration\n\nimport (\n\t\"os\"\n\t\"testing\"\n)\n\nfunc TestIntegration(t *testing.T) {\n\tif os.Getenv(\"INTEGRATION_BACKEND\") != \"all\" || os.Getenv(\"RUN_CHAOS\") != \"0\" || os.Getenv(\"RUN_SOAK\") != \"0\" {\n\t\tt.Fatalf(\"count environment = %q/%q/%q\", os.Getenv(\"INTEGRATION_BACKEND\"), os.Getenv(\"RUN_CHAOS\"), os.Getenv(\"RUN_SOAK\"))\n\t}\n\tt.Run(\"child\", func(t *testing.T) {})\n}\n") + writeTestFile(t, filepath.Join(root, "README.md"), testCountStart+"\nold\n"+testCountEnd+"\n") + return root +} + +// assertTestCountBadges verifies the generated README contains the expected executed counts. +func assertTestCountBadges(t *testing.T, root string, unit, integration int) { + t.Helper() + data, err := os.ReadFile(filepath.Join(root, "README.md")) + if err != nil { + t.Fatalf("read generated README: %v", err) + } + wantUnit := "unit_tests-" + fmt.Sprint(unit) + "-" + wantIntegration := "integration_tests-" + fmt.Sprint(integration) + "-" + if !strings.Contains(string(data), wantUnit) || !strings.Contains(string(data), wantIntegration) { + t.Fatalf("generated README = %s, want %s and %s", data, wantUnit, wantIntegration) + } +} diff --git a/docs/runbooks/backlog-growth.md b/docs/runbooks/backlog-growth.md index bbb807f..ecb29f3 100644 --- a/docs/runbooks/backlog-growth.md +++ b/docs/runbooks/backlog-growth.md @@ -10,7 +10,7 @@ - Confirm affected queue(s) and backend (`redis`, `mysql`, `postgres`, `rabbitmq`, `sqs`, etc.) - Check worker liveness and recent deploys/restarts -- Check error/retry rate and `republish_failed` / `process_recovered` event spikes +- Check error/retry rate and `republish_failed` / `settlement_failed` / `process_recovered` event spikes - Check whether the backlog is mostly: - pending - scheduled/delayed @@ -43,7 +43,7 @@ - Queue depth trend reverses within expected recovery window - Processing throughput increases and retry/failure rate drops -- No sustained `republish_failed` spikes +- No sustained `republish_failed` or `settlement_failed` spikes ## Follow-up diff --git a/driver/mysqlqueue/mysqlqueue.go b/driver/mysqlqueue/mysqlqueue.go index bc4b35c..dabe1f3 100644 --- a/driver/mysqlqueue/mysqlqueue.go +++ b/driver/mysqlqueue/mysqlqueue.go @@ -10,10 +10,12 @@ import ( "github.com/goforj/queue/queueconfig" ) +// Config configures the MySQL queue driver. type Config struct { queueconfig.DriverBaseConfig DB *sql.DB DSN string + DisableAutoMigrate bool ProcessingRecoveryGrace time.Duration ProcessingLeaseNoTimeout time.Duration } @@ -48,6 +50,7 @@ func New(dsn string, opts ...queue.Option) (*queue.Queue, error) { // }, // DB: nil, // optional; provide *sql.DB instead of DSN // DSN: "user:pass@tcp(127.0.0.1:3306)/queue?parseTime=true", // optional if DB is set +// DisableAutoMigrate: false, // set true when schema migrations are managed externally // ProcessingRecoveryGrace: 2 * time.Second, // default if <=0: 2s // ProcessingLeaseNoTimeout: 5 * time.Minute, // default if <=0: 5m // }, @@ -62,6 +65,7 @@ func NewWithConfig(cfg Config, opts ...queue.Option) (*queue.Queue, error) { DriverBaseConfig: cfg.DriverBaseConfig, DB: cfg.DB, DSN: cfg.DSN, + DisableAutoMigrate: cfg.DisableAutoMigrate, ProcessingRecoveryGrace: cfg.ProcessingRecoveryGrace, ProcessingLeaseNoTimeout: cfg.ProcessingLeaseNoTimeout, }, opts...) diff --git a/driver/natsqueue/natsqueue.go b/driver/natsqueue/natsqueue.go index 7b661d0..b5872ad 100644 --- a/driver/natsqueue/natsqueue.go +++ b/driver/natsqueue/natsqueue.go @@ -54,17 +54,18 @@ func NewWithConfig(cfg Config, opts ...queue.Option) (*queue.Queue, error) { if cfg.URL == "" { return nil, fmt.Errorf("nats url is required") } + observer := driverbridge.NewObserverSink(cfg.Observer) rootCfg := queue.Config{ Driver: queue.DriverNATS, DefaultQueue: cfg.DefaultQueue, - Observer: cfg.Observer, + Observer: observer, } return driverbridge.NewQueueFromDriver(rootCfg, newNATSQueue(cfg.URL), func(workers int) (any, error) { return newNATSWorkerWithConfig(natsWorkerConfig{ URL: cfg.URL, DefaultQueue: queue.PhysicalQueueName(cfg.DefaultQueue, cfg.DefaultQueue), Workers: workers, - Observer: cfg.Observer, + Observer: observer, }), nil }, opts...) } diff --git a/driver/natsqueue/queue_nats_impl.go b/driver/natsqueue/queue_nats_impl.go index 114ff0c..b7777c3 100644 --- a/driver/natsqueue/queue_nats_impl.go +++ b/driver/natsqueue/queue_nats_impl.go @@ -8,28 +8,62 @@ import ( "time" "github.com/goforj/queue" + "github.com/goforj/queue/internal/uniqueness" "github.com/goforj/queue/queuecore" "github.com/nats-io/nats.go" ) +const natsRoundTripTimeout = 5 * time.Second + type natsMessage struct { - Type string `json:"type"` - Payload []byte `json:"payload,omitempty"` - Queue string `json:"queue"` - Attempt int `json:"attempt,omitempty"` - MaxRetry int `json:"max_retry,omitempty"` - BackoffMillis int64 `json:"backoff_millis,omitempty"` - TimeoutMillis int64 `json:"timeout_millis,omitempty"` - AvailableAtMS int64 `json:"available_at_ms,omitempty"` - PublishedAtMS int64 `json:"published_at_ms,omitempty"` + Type string `json:"type"` + Payload []byte `json:"payload,omitempty"` + Queue string `json:"queue"` + Metadata json.RawMessage `json:"metadata,omitempty"` + Attempt int `json:"attempt,omitempty"` + MaxRetry int `json:"max_retry,omitempty"` + BackoffMillis int64 `json:"backoff_millis,omitempty"` + TimeoutMillis int64 `json:"timeout_millis,omitempty"` + AvailableAtMS int64 `json:"available_at_ms,omitempty"` + PublishedAtMS int64 `json:"published_at_ms,omitempty"` +} + +type natsConnection interface { + Publish(subject string, data []byte) error + FlushWithContext(ctx context.Context) error + Drain() error + Close() +} + +type synchronousNATSConnection struct { + *nats.Conn +} + +// Drain waits for the asynchronous Core NATS drain to close the connection. +func (c *synchronousNATSConnection) Drain() error { + if c == nil || c.Conn == nil || c.IsClosed() { + return nil + } + closed := c.StatusChanged(nats.CLOSED) + defer c.RemoveStatusListener(closed) + if err := c.Conn.Drain(); err != nil { + return err + } + for status := range closed { + if status == nats.CLOSED { + return nil + } + } + return nil } type natsQueue struct { url string - nc *nats.Conn - mu sync.Mutex - unique map[string]time.Time + mu sync.Mutex + nc natsConnection + + unique uniqueness.MemoryStore } func (q *natsQueue) Driver() queue.Driver { @@ -43,41 +77,60 @@ func (q *natsQueue) Preflight(ctx context.Context) error { if err := ctx.Err(); err != nil { return err } - if err := q.ensureConn(); err != nil { + nc, err := q.connection() + if err != nil { return err } - return q.nc.FlushWithContext(ctx) + flushCtx, cancel := natsRoundTripContext(ctx) + defer cancel() + return nc.FlushWithContext(flushCtx) } func newNATSQueue(url string) *natsQueue { - return &natsQueue{ - url: url, - unique: make(map[string]time.Time), - } + return &natsQueue{url: url} } +// ensureConn establishes at most one shared Core NATS connection. func (q *natsQueue) ensureConn() error { + _, err := q.connection() + return err +} + +// connection returns the connection established while holding the same lock used by Shutdown. +func (q *natsQueue) connection() (natsConnection, error) { + q.mu.Lock() + defer q.mu.Unlock() if q.nc != nil { - return nil + return q.nc, nil } nc, err := nats.Connect(q.url) if err != nil { - return err + return nil, err } - q.nc = nc - return nil + q.nc = &synchronousNATSConnection{Conn: nc} + return q.nc, nil } func (q *natsQueue) Shutdown(_ context.Context) error { - if q.nc != nil { - q.nc.Drain() - q.nc.Close() - q.nc = nil + q.mu.Lock() + nc := q.nc + q.nc = nil + q.mu.Unlock() + if nc != nil { + // Every accepted publish already completed a flush, so producer shutdown only needs to prevent reuse and close the socket. + nc.Close() } return nil } -func (q *natsQueue) Dispatch(_ context.Context, job queue.Job) error { +// Dispatch flushes initial publication so acceptance includes a Core NATS server roundtrip. +func (q *natsQueue) Dispatch(ctx context.Context, job queue.Job) error { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } if err := queuecore.ValidateDriverJob(job); err != nil { return err } @@ -85,59 +138,92 @@ func (q *natsQueue) Dispatch(_ context.Context, job queue.Job) error { if parsed.QueueName == "" { return fmt.Errorf("job queue is required") } - if q.nc == nil { - if err := q.ensureConn(); err != nil { - return err + nc, err := q.connection() + if err != nil { + return err + } + var ( + uniqueKey string + uniqueToken uint64 + ) + if parsed.UniqueTTL > 0 { + var acquired bool + uniqueKey, uniqueToken, acquired = q.claimUnique(job, parsed.QueueName, parsed.UniqueTTL) + if !acquired { + return queuecore.ErrDuplicate } } - if parsed.UniqueTTL > 0 && !q.claimUnique(job, parsed.QueueName, parsed.UniqueTTL) { - return queuecore.ErrDuplicate + + msg, err := natsMessageForJob(job, parsed) + if err != nil { + q.unique.Release(uniqueKey, uniqueToken) + return err + } + + payload, err := json.Marshal(msg) + if err != nil { + q.unique.Release(uniqueKey, uniqueToken) + return err + } + err = nc.Publish(natsSubject(parsed.QueueName), payload) + if err != nil { + q.unique.Release(uniqueKey, uniqueToken) + return err } + flushCtx, cancel := natsRoundTripContext(ctx) + defer cancel() + // A flush proves only that the Core NATS server observed this ephemeral publish, not durable storage. + return nc.FlushWithContext(flushCtx) +} - msg := natsMessage{ +// natsMessageForJob converts one validated queue job into the stable NATS wire +// representation while keeping direct-delivery metadata optional. +func natsMessageForJob(job queue.Job, options queue.DriverJobOptions) (natsMessage, error) { + message := natsMessage{ Type: job.Type, Payload: job.PayloadBytes(), - Queue: parsed.QueueName, + Queue: options.QueueName, PublishedAtMS: time.Now().UnixMilli(), } - if parsed.MaxRetry != nil { - msg.MaxRetry = *parsed.MaxRetry + metadata := queue.DriverMetadata(job) + if metadata.SchemaVersion != 0 { + encoded, err := json.Marshal(metadata) + if err != nil { + return natsMessage{}, fmt.Errorf("encode NATS driver job metadata: %w", err) + } + message.Metadata = encoded } - if parsed.Backoff != nil && *parsed.Backoff > 0 { - msg.BackoffMillis = parsed.Backoff.Milliseconds() + if options.MaxRetry != nil { + message.MaxRetry = *options.MaxRetry } - if parsed.Timeout != nil && *parsed.Timeout > 0 { - msg.TimeoutMillis = parsed.Timeout.Milliseconds() + if options.Backoff != nil && *options.Backoff > 0 { + message.BackoffMillis = options.Backoff.Milliseconds() } - if parsed.Delay > 0 { - msg.AvailableAtMS = time.Now().Add(parsed.Delay).UnixMilli() + if options.Timeout != nil && *options.Timeout > 0 { + message.TimeoutMillis = options.Timeout.Milliseconds() } - - payload, err := json.Marshal(msg) - if err != nil { - return err + if options.Delay > 0 { + message.AvailableAtMS = time.Now().Add(options.Delay).UnixMilli() } - return q.nc.Publish(natsSubject(parsed.QueueName), payload) + return message, nil } -func (q *natsQueue) claimUnique(job queue.Job, queueName string, ttl time.Duration) bool { - now := time.Now() - key := queueName + ":" + job.Type + ":" + string(job.PayloadBytes()) - - q.mu.Lock() - defer q.mu.Unlock() - for candidate, expiresAt := range q.unique { - if expiresAt.Before(now) { - delete(q.unique, candidate) - } - } - if expiresAt, ok := q.unique[key]; ok && expiresAt.After(now) { - return false - } - q.unique[key] = now.Add(ttl) - return true +// claimUnique returns the ownership token needed to compensate a rejected publish. +func (q *natsQueue) claimUnique(job queue.Job, queueName string, ttl time.Duration) (string, uint64, bool) { + key := queuecore.UniqueKey(job, queueName) + token, ok := q.unique.Acquire(key, ttl) + return key, token, ok } +// natsSubject maps one physical queue onto its Core NATS subject. func natsSubject(queueName string) string { return "queue." + queueName } + +// natsRoundTripContext supplies the deadline required by NATS while retaining a shorter caller deadline. +func natsRoundTripContext(ctx context.Context) (context.Context, context.CancelFunc) { + if ctx == nil { + ctx = context.Background() + } + return context.WithTimeout(ctx, natsRoundTripTimeout) +} diff --git a/driver/natsqueue/queue_nats_impl_test.go b/driver/natsqueue/queue_nats_impl_test.go index f13493e..535b3ec 100644 --- a/driver/natsqueue/queue_nats_impl_test.go +++ b/driver/natsqueue/queue_nats_impl_test.go @@ -1,7 +1,11 @@ package natsqueue import ( + "bytes" "context" + "encoding/json" + "errors" + "net" "testing" "time" @@ -9,6 +13,171 @@ import ( "github.com/nats-io/nats.go" ) +// TestNATSDirectDeliveryMetadataRoundTrip verifies producer framing, worker +// reconstruction, retry preservation, and legacy-envelope observation. +func TestNATSDirectDeliveryMetadataRoundTrip(t *testing.T) { + wantMetadata := queue.DriverJobMetadata{ + SchemaVersion: queue.DriverJobMetadataVersion, + DispatchID: "dsp_nats_direct", + JobID: "job_nats_direct", + Queue: "critical", + } + wantPayload := []byte(`{"report_id":42}`) + job := queue.DriverWithMetadata( + queue.NewJob("reports:build").Payload(wantPayload).OnQueue("critical").Retry(3), + wantMetadata, + ) + message, err := natsMessageForJob(job, queue.DriverOptions(job)) + if err != nil { + t.Fatalf("build direct message: %v", err) + } + var wireMetadata queue.DriverJobMetadata + if err := json.Unmarshal(message.Metadata, &wireMetadata); err != nil || wireMetadata != wantMetadata { + t.Fatalf("wire metadata = %+v, want %+v (err=%v)", wireMetadata, wantMetadata, err) + } + + wire, err := json.Marshal(message) + if err != nil { + t.Fatalf("marshal direct message: %v", err) + } + var decoded natsMessage + if err := json.Unmarshal(wire, &decoded); err != nil { + t.Fatalf("unmarshal direct message: %v", err) + } + delivery := natsDeliveryJob(decoded) + if delivery.Type != "reports:build" || !bytes.Equal(delivery.PayloadBytes(), wantPayload) { + t.Fatalf("delivery = type:%q payload:%q", delivery.Type, delivery.PayloadBytes()) + } + if got := queue.DriverMetadata(delivery); got != wantMetadata { + t.Fatalf("reconstructed metadata = %+v, want %+v", got, wantMetadata) + } + observed := queue.ResolveObservedJobMetadataFromJob(delivery) + if observed.DispatchID != wantMetadata.DispatchID || observed.JobID != wantMetadata.JobID || observed.JobType != job.Type { + t.Fatalf("direct observation = %+v", observed) + } + var events []queue.Event + worker := &natsWorker{observer: queue.ObserverFunc(func(_ context.Context, event queue.Event) { + events = append(events, event) + })} + worker.observeRepublishFailure(context.Background(), decoded, errors.New("republish failed")) + if len(events) != 1 || events[0].DispatchID != wantMetadata.DispatchID || events[0].JobID != wantMetadata.JobID { + t.Fatalf("direct republish observation = %+v", events) + } + + decoded.Attempt++ + connection := &natsConnectionStub{} + worker.conn = connection + if err := worker.republish(decoded); err != nil { + t.Fatalf("republish direct message: %v", err) + } + if len(connection.published) != 1 { + t.Fatalf("republished messages = %d, want 1", len(connection.published)) + } + var retry natsMessage + if err := json.Unmarshal(connection.published[0], &retry); err != nil { + t.Fatalf("unmarshal retry message: %v", err) + } + retryJob := natsDeliveryJob(retry) + if got := queue.DriverMetadata(retryJob); got != wantMetadata { + t.Fatalf("retry metadata = %+v, want %+v", got, wantMetadata) + } + if got := queue.DriverOptions(retryJob).Attempt; got != 1 { + t.Fatalf("retry attempt = %d, want 1", got) + } + + legacyPayload := []byte(`{"schema_version":1,"dispatch_id":"dsp_nats_legacy","job_id":"job_nats_legacy","job":{"type":"reports:legacy","payload":"e30="}}`) + legacy := queue.ResolveObservedJobMetadataFromJob(natsDeliveryJob(natsMessage{Type: "bus:job", Payload: legacyPayload})) + if legacy.JobType != "reports:legacy" || legacy.DispatchID != "dsp_nats_legacy" || legacy.JobID != "job_nats_legacy" { + t.Fatalf("legacy observation = %+v", legacy) + } + + plainJob := queue.NewJob("reports:plain").OnQueue("default") + plain, err := natsMessageForJob(plainJob, queue.DriverOptions(plainJob)) + if err != nil { + t.Fatalf("build metadata-absent message: %v", err) + } + plainWire, err := json.Marshal(plain) + if err != nil { + t.Fatalf("marshal metadata-absent message: %v", err) + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(plainWire, &fields); err != nil { + t.Fatalf("inspect metadata-absent message: %v", err) + } + if _, ok := fields["metadata"]; ok { + t.Fatalf("metadata-absent wire unexpectedly contains metadata: %s", plainWire) + } +} + +// TestNATSUntrustedMetadataRemainsAnOpaqueRetrySidecar verifies valid +// application bytes survive malformed metadata and future fields survive republish. +func TestNATSUntrustedMetadataRemainsAnOpaqueRetrySidecar(t *testing.T) { + for _, raw := range []string{`"malformed"`, `{"schema_version":"bad","dispatch_id":"spoofed"}`} { + wire := []byte(`{"type":"reports:build","payload":"AQI=","queue":"critical","metadata":` + raw + `}`) + var message natsMessage + if err := json.Unmarshal(wire, &message); err != nil { + t.Fatalf("decode message with metadata %s: %v", raw, err) + } + delivery := natsDeliveryJob(message) + if delivery.Type != "reports:build" || !bytes.Equal(delivery.PayloadBytes(), []byte{1, 2}) { + t.Fatalf("delivery with metadata %s = type:%q payload:%v", raw, delivery.Type, delivery.PayloadBytes()) + } + if metadata := queue.DriverMetadata(delivery); metadata != (queue.DriverJobMetadata{}) { + t.Fatalf("untrusted metadata %s became trusted: %+v", raw, metadata) + } + } + + future := json.RawMessage(`{"schema_version":99,"dispatch_id":"future","future_field":{"id":7}}`) + connection := &natsConnectionStub{} + worker := &natsWorker{conn: connection} + if err := worker.republish(natsMessage{Type: "reports:build", Queue: "critical", Metadata: future}); err != nil { + t.Fatalf("republish future metadata: %v", err) + } + var retry natsMessage + if len(connection.published) != 1 { + t.Fatalf("republished messages = %d, want 1", len(connection.published)) + } + if err := json.Unmarshal(connection.published[0], &retry); err != nil { + t.Fatalf("decode future retry: %v", err) + } + if !bytes.Equal(retry.Metadata, future) { + t.Fatalf("future retry metadata = %s, want %s", retry.Metadata, future) + } + if metadata := queue.DriverMetadata(natsDeliveryJob(retry)); metadata != (queue.DriverJobMetadata{}) { + t.Fatalf("future metadata became trusted: %+v", metadata) + } +} + +type natsConnectionStub struct { + publishErr error + flushErr error + publishN int + flushN int + closeN int + flushCtx context.Context + published [][]byte +} + +// Publish returns the configured acceptance result. +func (s *natsConnectionStub) Publish(_ string, payload []byte) error { + s.publishN++ + s.published = append(s.published, append([]byte(nil), payload...)) + return s.publishErr +} + +// FlushWithContext records the bounded server-roundtrip request. +func (s *natsConnectionStub) FlushWithContext(ctx context.Context) error { + s.flushN++ + s.flushCtx = ctx + return s.flushErr +} + +// Drain reports successful shutdown for the stub connection. +func (s *natsConnectionStub) Drain() error { return nil } + +// Close records producer resource cleanup. +func (s *natsConnectionStub) Close() { s.closeN++ } + func TestNATSQueue_EnsureConnShortCircuitsWhenPresent(t *testing.T) { q := newNATSQueue("nats://127.0.0.1:1") q.nc = &nats.Conn{} @@ -42,10 +211,10 @@ func TestNATSQueue_ShutdownNilConnAndHelpers(t *testing.T) { } job := queue.NewJob("job:nats").Payload(map[string]any{"id": 1}).OnQueue("default") - if !q.claimUnique(job, "default", time.Minute) { + if _, _, ok := q.claimUnique(job, "default", time.Minute); !ok { t.Fatal("expected first unique claim to succeed") } - if q.claimUnique(job, "default", time.Minute) { + if _, _, ok := q.claimUnique(job, "default", time.Minute); ok { t.Fatal("expected duplicate unique claim to fail") } @@ -54,6 +223,21 @@ func TestNATSQueue_ShutdownNilConnAndHelpers(t *testing.T) { } } +// TestNATSQueueShutdownClosesWithoutAsynchronousDrain verifies producer cleanup cannot outlive the public shutdown deadline. +func TestNATSQueueShutdownClosesWithoutAsynchronousDrain(t *testing.T) { + connection := &natsConnectionStub{} + q := newNATSQueue("nats://example") + q.nc = connection + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := q.Shutdown(ctx); err != nil { + t.Fatalf("shutdown: %v", err) + } + if connection.closeN != 1 || q.nc != nil { + t.Fatalf("producer cleanup = closes:%d retained:%T, want 1/nil", connection.closeN, q.nc) + } +} + func TestNATSQueue_EnsureConnFailure(t *testing.T) { q := newNATSQueue("://bad-url") if err := q.ensureConn(); err == nil { @@ -64,9 +248,227 @@ func TestNATSQueue_EnsureConnFailure(t *testing.T) { } } +// TestNATSQueuePreflightBoundaries verifies readiness reports both connection +// and server-roundtrip failures without requiring a live NATS server. +func TestNATSQueuePreflightBoundaries(t *testing.T) { + t.Run("canceled context", func(t *testing.T) { + q := newNATSQueue("nats://example") + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := q.Preflight(ctx); !errors.Is(err, context.Canceled) { + t.Fatalf("preflight error = %v, want context.Canceled", err) + } + if q.nc != nil { + t.Fatalf("canceled preflight established connection %T", q.nc) + } + }) + + t.Run("connection failure", func(t *testing.T) { + q := newNATSQueue("://bad-url") + if err := q.Preflight(context.Background()); err == nil { + t.Fatal("expected preflight connection failure") + } + }) + + t.Run("flush result", func(t *testing.T) { + flushErr := errors.New("readiness flush failed") + connection := &natsConnectionStub{flushErr: flushErr} + q := newNATSQueue("nats://example") + q.nc = connection + if err := q.Preflight(nil); !errors.Is(err, flushErr) { + t.Fatalf("preflight error = %v, want %v", err, flushErr) + } + if connection.flushN != 1 { + t.Fatalf("preflight flush calls = %d, want 1", connection.flushN) + } + deadline, ok := connection.flushCtx.Deadline() + if !ok { + t.Fatal("preflight flush context has no deadline") + } + remaining := time.Until(deadline) + if remaining <= 0 || remaining > natsRoundTripTimeout { + t.Fatalf("preflight deadline remaining = %v, want within (0, %v]", remaining, natsRoundTripTimeout) + } + }) + + t.Run("shorter caller deadline", func(t *testing.T) { + connection := &natsConnectionStub{} + q := newNATSQueue("nats://example") + q.nc = connection + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + callerDeadline, _ := ctx.Deadline() + if err := q.Preflight(ctx); err != nil { + t.Fatalf("preflight with caller deadline: %v", err) + } + flushDeadline, ok := connection.flushCtx.Deadline() + if !ok || !flushDeadline.Equal(callerDeadline) { + t.Fatalf("preflight flush deadline = %v/%t, want caller deadline %v", flushDeadline, ok, callerDeadline) + } + }) +} + +// TestNATSQueuePublicReadyReportsUnavailableBackend verifies the public queue +// path does not mistake a reachable non-NATS socket for backend readiness. +func TestNATSQueuePublicReadyReportsUnavailableBackend(t *testing.T) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen for non-NATS endpoint: %v", err) + } + done := make(chan struct{}) + go func() { + defer close(done) + for { + conn, acceptErr := listener.Accept() + if acceptErr != nil { + return + } + _ = conn.Close() + } + }() + t.Cleanup(func() { + _ = listener.Close() + <-done + }) + + runtime, err := New("nats://" + listener.Addr().String()) + if err != nil { + t.Fatalf("new public NATS queue: %v", err) + } + t.Cleanup(func() { _ = runtime.Shutdown(context.Background()) }) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + if err := runtime.Ready(ctx); err == nil { + t.Fatal("public NATS readiness accepted a non-NATS endpoint") + } +} + +// TestNATSQueueNilDispatchContext verifies a nil caller context still reaches +// the bounded server acceptance roundtrip. +func TestNATSQueueNilDispatchContext(t *testing.T) { + connection := &natsConnectionStub{} + q := newNATSQueue("nats://example") + q.nc = connection + if err := q.Dispatch(nil, queue.NewJob("reports:nil-context").OnQueue("default")); err != nil { + t.Fatalf("dispatch with nil context: %v", err) + } + if connection.publishN != 1 || connection.flushN != 1 { + t.Fatalf("publish/flush calls = %d/%d, want 1/1", connection.publishN, connection.flushN) + } +} + +// TestNATSQueueDispatchPreservesTemporalOptions verifies the producer writes +// retry timing and availability controls into the direct-delivery envelope. +func TestNATSQueueDispatchPreservesTemporalOptions(t *testing.T) { + connection := &natsConnectionStub{} + q := newNATSQueue("nats://example") + q.nc = connection + delay := 2 * time.Second + beforeDispatch := time.Now() + job := queue.NewJob("reports:scheduled"). + OnQueue("critical"). + Retry(4). + Backoff(250 * time.Millisecond). + Timeout(3 * time.Second). + Delay(delay) + + if err := q.Dispatch(context.Background(), job); err != nil { + t.Fatalf("dispatch temporal job: %v", err) + } + if len(connection.published) != 1 { + t.Fatalf("published messages = %d, want 1", len(connection.published)) + } + var message natsMessage + if err := json.Unmarshal(connection.published[0], &message); err != nil { + t.Fatalf("decode temporal message: %v", err) + } + if message.MaxRetry != 4 || message.BackoffMillis != 250 || message.TimeoutMillis != 3_000 { + t.Fatalf("temporal options = retry:%d backoff:%d timeout:%d", message.MaxRetry, message.BackoffMillis, message.TimeoutMillis) + } + minimumAvailableAt := beforeDispatch.Add(delay).Add(-time.Millisecond).UnixMilli() + maximumAvailableAt := time.Now().Add(delay).Add(time.Millisecond).UnixMilli() + if message.AvailableAtMS < minimumAvailableAt || message.AvailableAtMS > maximumAvailableAt { + t.Fatalf("available_at_ms = %d, want between %d and %d", message.AvailableAtMS, minimumAvailableAt, maximumAvailableAt) + } +} + +// TestNATSRoundTripContextNormalizesNil verifies server roundtrips remain +// bounded even when no caller context is available. +func TestNATSRoundTripContextNormalizesNil(t *testing.T) { + ctx, cancel := natsRoundTripContext(nil) + defer cancel() + deadline, ok := ctx.Deadline() + if !ok { + t.Fatal("nil roundtrip context did not receive a deadline") + } + remaining := time.Until(deadline) + if remaining <= 0 || remaining > natsRoundTripTimeout { + t.Fatalf("roundtrip deadline remaining = %v, want within (0, %v]", remaining, natsRoundTripTimeout) + } +} + func TestNATSQueue_Driver(t *testing.T) { q := newNATSQueue("nats://127.0.0.1:1") if q.Driver() != queue.DriverNATS { t.Fatalf("expected driver %q, got %q", queue.DriverNATS, q.Driver()) } } + +// TestNATSQueueRejectedPublishReleasesUniqueClaim verifies a failed publish cannot poison the TTL window. +func TestNATSQueueRejectedPublishReleasesUniqueClaim(t *testing.T) { + publishErr := errors.New("publish rejected") + connection := &natsConnectionStub{publishErr: publishErr} + q := newNATSQueue("nats://example") + q.nc = connection + job := queue.NewJob("reports:build").Payload([]byte(`{"id":1}`)).OnQueue("default").UniqueFor(time.Minute) + if err := q.Dispatch(context.Background(), job); !errors.Is(err, publishErr) { + t.Fatalf("first dispatch error = %v, want publish rejection", err) + } + connection.publishErr = nil + if err := q.Dispatch(context.Background(), job); err != nil { + t.Fatalf("second dispatch remained poisoned: %v", err) + } + if err := q.Dispatch(context.Background(), job); !errors.Is(err, queue.ErrDuplicate) { + t.Fatalf("accepted publish did not retain claim: %v", err) + } +} + +// TestNATSQueueFlushFailureRetainsUniqueClaim verifies an ambiguous server-roundtrip error fails closed against duplicates. +func TestNATSQueueFlushFailureRetainsUniqueClaim(t *testing.T) { + flushErr := errors.New("flush response lost") + connection := &natsConnectionStub{flushErr: flushErr} + q := newNATSQueue("nats://example") + q.nc = connection + job := queue.NewJob("reports:build").OnQueue("default").UniqueFor(time.Minute) + if err := q.Dispatch(context.Background(), job); !errors.Is(err, flushErr) { + t.Fatalf("flush failure = %v, want %v", err, flushErr) + } + connection.flushErr = nil + if err := q.Dispatch(context.Background(), job); !errors.Is(err, queue.ErrDuplicate) { + t.Fatalf("ambiguous publish did not retain uniqueness claim: %v", err) + } + if connection.publishN != 1 || connection.flushN != 1 { + t.Fatalf("publish/flush calls = %d/%d, want 1/1", connection.publishN, connection.flushN) + } +} + +// TestNATSQueueCanceledDispatchStopsBeforeClaim verifies cancellation cannot publish or consume instance uniqueness state. +func TestNATSQueueCanceledDispatchStopsBeforeClaim(t *testing.T) { + connection := &natsConnectionStub{} + q := newNATSQueue("nats://example") + q.nc = connection + job := queue.NewJob("reports:canceled").OnQueue("default").UniqueFor(time.Minute) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := q.Dispatch(ctx, job); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled dispatch = %v, want context.Canceled", err) + } + key, token, ok := q.claimUnique(job, "default", time.Minute) + if !ok { + t.Fatal("canceled dispatch consumed uniqueness state") + } + q.unique.Release(key, token) + if connection.publishN != 0 || connection.flushN != 0 { + t.Fatalf("canceled dispatch touched NATS: publish=%d flush=%d", connection.publishN, connection.flushN) + } +} diff --git a/driver/natsqueue/worker_nats_impl.go b/driver/natsqueue/worker_nats_impl.go index 0b0a58c..fa10b26 100644 --- a/driver/natsqueue/worker_nats_impl.go +++ b/driver/natsqueue/worker_nats_impl.go @@ -4,9 +4,11 @@ import ( "context" "encoding/json" "sync" + "sync/atomic" "time" "github.com/goforj/queue" + "github.com/goforj/queue/busruntime" "github.com/goforj/queue/queuecore" "github.com/nats-io/nats.go" ) @@ -19,14 +21,48 @@ type natsWorker struct { mu sync.RWMutex handlers map[string]queue.Handler - conn *nats.Conn - sub *nats.Subscription - start sync.Once - sem chan struct{} + startStop sync.Mutex + started bool + stopDone chan struct{} + stopErr error + connect natsWorkerConnector + + conn natsConnection + sub natsWorkerSubscription + sem chan struct{} + running sync.WaitGroup + delayed sync.WaitGroup observer queue.Observer } +type natsWorkerSubscription interface { + Drain() error +} + +type synchronousNATSSubscription struct { + *nats.Subscription +} + +// Drain waits until Core NATS has stopped intake and completed every queued callback. +func (s *synchronousNATSSubscription) Drain() error { + if s == nil || s.Subscription == nil || !s.IsValid() { + return nil + } + closed := s.StatusChanged(nats.SubscriptionClosed) + if err := s.Subscription.Drain(); err != nil { + return err + } + for status := range closed { + if status == nats.SubscriptionClosed { + return nil + } + } + return nil +} + +type natsWorkerConnector func(url, subject string, callback nats.MsgHandler) (natsConnection, natsWorkerSubscription, error) + type natsWorkerConfig struct { URL string DefaultQueue string @@ -62,49 +98,114 @@ func (w *natsWorker) Register(jobType string, handler queue.Handler) { } func (w *natsWorker) StartWorkers(ctx context.Context) error { - if ctx != nil && ctx.Err() != nil { - return ctx.Err() + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } + w.startStop.Lock() + defer w.startStop.Unlock() + if w.stopDone != nil { + return queue.ErrQueuerShuttingDown } - var startErr error - w.start.Do(func() { - nc, err := nats.Connect(w.url) - if err != nil { - startErr = err + if w.started { + return nil + } + connect := w.connect + if connect == nil { + connect = connectNATSWorker + } + w.sem = make(chan struct{}, w.workers) + ready := make(chan struct{}) + var acceptCallbacks atomic.Bool + nc, sub, err := connect(w.url, natsSubject(w.defaultQueue), func(message *nats.Msg) { + <-ready + if !acceptCallbacks.Load() { return } - w.sem = make(chan struct{}, w.workers) - sub, err := nc.Subscribe(natsSubject(w.defaultQueue), func(message *nats.Msg) { - w.sem <- struct{}{} - w.running.Add(1) - go func() { - defer func() { - <-w.sem - w.running.Done() - }() - w.processMessage(message) + w.running.Add(1) + w.sem <- struct{}{} + go func() { + defer func() { + <-w.sem + w.running.Done() }() - }) - if err != nil { - nc.Close() - startErr = err - return - } - w.conn = nc - w.sub = sub + w.processMessage(message) + }() }) - return startErr + if err != nil { + return err + } + w.conn = nc + w.sub = sub + flushCtx, cancel := natsRoundTripContext(ctx) + flushErr := nc.FlushWithContext(flushCtx) + cancel() + if flushErr != nil { + close(ready) + _ = sub.Drain() + nc.Close() + w.conn = nil + w.sub = nil + return flushErr + } + acceptCallbacks.Store(true) + w.started = true + close(ready) + return nil } -func (w *natsWorker) Shutdown(_ context.Context) error { - if w.sub != nil { - _ = w.sub.Drain() +// Shutdown stops intake before waiting for handlers and delayed republishes, then closes their shared connection. +func (w *natsWorker) Shutdown(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() } - if w.conn != nil { - _ = w.conn.Drain() - w.conn.Close() + w.startStop.Lock() + if !w.started && w.stopDone == nil { + w.startStop.Unlock() + return nil + } + if w.stopDone == nil { + w.stopDone = make(chan struct{}) + done := w.stopDone + sub := w.sub + conn := w.conn + go func() { + var stopErr error + if sub != nil { + stopErr = sub.Drain() + } + w.running.Wait() + w.delayed.Wait() + if conn != nil { + if drainErr := conn.Drain(); stopErr == nil { + stopErr = drainErr + } + conn.Close() + } + w.startStop.Lock() + w.started = false + w.conn = nil + w.sub = nil + w.stopErr = stopErr + w.startStop.Unlock() + close(done) + }() + } + done := w.stopDone + w.startStop.Unlock() + select { + case <-done: + w.startStop.Lock() + err := w.stopErr + // Drain diagnostics describe a cleanup that has already completed; report them once so a later root shutdown can finish producer cleanup. + w.stopErr = nil + w.startStop.Unlock() + return err + case <-ctx.Done(): + return ctx.Err() } - w.running.Wait() - return nil } func (w *natsWorker) processMessage(message *nats.Msg) { @@ -115,7 +216,9 @@ func (w *natsWorker) processMessage(message *nats.Msg) { if incoming.AvailableAtMS > 0 { remaining := time.Until(time.UnixMilli(incoming.AvailableAtMS)) if remaining > 0 { + w.delayed.Add(1) time.AfterFunc(remaining, func() { + defer w.delayed.Done() if err := w.republish(incoming); err != nil { w.observeRepublishFailure(context.Background(), incoming, err) } @@ -131,7 +234,8 @@ func (w *natsWorker) processMessage(message *nats.Msg) { return } - ctx := context.Background() + attempt := busruntime.DeliveryAttempt{Number: incoming.Attempt, MaxRetry: incoming.MaxRetry} + ctx := busruntime.WithDeliveryAttempt(context.Background(), attempt) if incoming.TimeoutMillis > 0 { var cancel context.CancelFunc ctx, cancel = context.WithTimeout(ctx, time.Duration(incoming.TimeoutMillis)*time.Millisecond) @@ -139,19 +243,19 @@ func (w *natsWorker) processMessage(message *nats.Msg) { } err := handler( ctx, - queuecore.DriverWithAttempt( - queue.NewJob(incoming.Type). - Payload(incoming.Payload). - OnQueue(incoming.Queue). - Retry(incoming.MaxRetry), - incoming.Attempt, - ), + natsDeliveryJob(incoming), ) - if err == nil { + switch busruntime.ClassifyAttempt(attempt, err) { + case busruntime.AttemptSucceeded, busruntime.AttemptFailed: return - } - if incoming.Attempt >= incoming.MaxRetry { + case busruntime.AttemptRedeliver: + // Core NATS has no broker-managed negative acknowledgement, so uncommitted work must be republished explicitly. + incoming.AvailableAtMS = 0 + if err := w.republish(incoming); err != nil { + w.observeRepublishFailure(ctx, incoming, err) + } return + case busruntime.AttemptRetry: } incoming.Attempt++ if incoming.BackoffMillis > 0 { @@ -164,6 +268,20 @@ func (w *natsWorker) processMessage(message *nats.Msg) { } } +// connectNATSWorker creates the Core NATS subscription owned by one worker lifecycle. +func connectNATSWorker(url, subject string, callback nats.MsgHandler) (natsConnection, natsWorkerSubscription, error) { + nc, err := nats.Connect(url) + if err != nil { + return nil, nil, err + } + sub, err := nc.Subscribe(subject, callback) + if err != nil { + nc.Close() + return nil, nil, err + } + return &synchronousNATSConnection{Conn: nc}, &synchronousNATSSubscription{Subscription: sub}, nil +} + func (w *natsWorker) republish(message natsMessage) error { if w.conn == nil { return nats.ErrConnectionClosed @@ -172,22 +290,52 @@ func (w *natsWorker) republish(message natsMessage) error { if err != nil { return err } - return w.conn.Publish(natsSubject(message.Queue), payload) + if err := w.conn.Publish(natsSubject(message.Queue), payload); err != nil { + return err + } + ctx, cancel := natsRoundTripContext(context.Background()) + defer cancel() + return w.conn.FlushWithContext(ctx) } func (w *natsWorker) observeRepublishFailure(ctx context.Context, message natsMessage, err error) { + metadata := queue.ResolveObservedJobMetadataFromJob(natsDeliveryJob(message)) queuecore.SafeObserve(ctx, w.observer, queue.Event{ - Kind: queue.EventRepublishFailed, - Driver: queue.DriverNATS, - Queue: queuecore.NormalizeQueueName(message.Queue), - JobType: queue.ResolveObservedJobType(message.Type, message.Payload), - Attempt: message.Attempt, - MaxRetry: message.MaxRetry, - Err: err, - Time: time.Now(), + Kind: queue.EventRepublishFailed, + Driver: queue.DriverNATS, + Queue: queuecore.NormalizeQueueName(message.Queue), + JobType: metadata.JobType, + JobKey: metadata.JobKey, + DispatchID: metadata.DispatchID, + JobID: metadata.JobID, + ChainID: metadata.ChainID, + BatchID: metadata.BatchID, + Attempt: message.Attempt, + MaxRetry: message.MaxRetry, + Err: err, + Time: time.Now(), }) } +// natsDeliveryJob reconstructs one NATS delivery without coupling application +// payload bytes to the optional direct-delivery metadata channel. +func natsDeliveryJob(message natsMessage) queue.Job { + job := queuecore.DriverWithAttempt( + queue.NewJob(message.Type). + Payload(message.Payload). + OnQueue(message.Queue). + Retry(message.MaxRetry), + message.Attempt, + ) + if len(message.Metadata) > 0 { + var metadata queue.DriverJobMetadata + if err := json.Unmarshal(message.Metadata, &metadata); err == nil { + job = queue.DriverWithMetadata(job, metadata) + } + } + return job +} + func defaultWorkerCount(n int) int { if n <= 0 { return 1 diff --git a/driver/natsqueue/worker_nats_impl_test.go b/driver/natsqueue/worker_nats_impl_test.go index d9d35c7..5720bf3 100644 --- a/driver/natsqueue/worker_nats_impl_test.go +++ b/driver/natsqueue/worker_nats_impl_test.go @@ -4,14 +4,86 @@ import ( "context" "encoding/json" "errors" + "sync" "testing" "time" "github.com/goforj/queue" + "github.com/goforj/queue/busruntime" "github.com/goforj/queue/queuecore" "github.com/nats-io/nats.go" ) +type natsWorkerSubscriptionStub struct { + drained chan struct{} + once sync.Once + drainErr error + release chan struct{} +} + +// Drain records that intake stopped before worker settlement resources closed. +func (s *natsWorkerSubscriptionStub) Drain() error { + s.once.Do(func() { close(s.drained) }) + if s.release != nil { + <-s.release + } + return s.drainErr +} + +type natsWorkerConnectionLifecycleStub struct { + mu sync.Mutex + published chan struct{} + drained chan struct{} + pubOnce sync.Once + drainOnce sync.Once + closed bool + flushErr error + drainErr error +} + +// Publish records replacement work and rejects publication after Close. +func (s *natsWorkerConnectionLifecycleStub) Publish(string, []byte) error { + s.mu.Lock() + closed := s.closed + s.mu.Unlock() + if closed { + return nats.ErrConnectionClosed + } + s.pubOnce.Do(func() { close(s.published) }) + return nil +} + +// FlushWithContext completes the fake server roundtrip unless the connection already closed. +func (s *natsWorkerConnectionLifecycleStub) FlushWithContext(context.Context) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + return nats.ErrConnectionClosed + } + return s.flushErr +} + +// Drain records graceful connection drain after every expected replacement publish. +func (s *natsWorkerConnectionLifecycleStub) Drain() error { + s.drainOnce.Do(func() { close(s.drained) }) + return s.drainErr +} + +// Close marks the fake connection unavailable for later publication. +func (s *natsWorkerConnectionLifecycleStub) Close() { + s.mu.Lock() + s.closed = true + s.mu.Unlock() +} + +// newNATSWorkerLifecycleStubs creates observable subscription and connection boundaries for shutdown tests. +func newNATSWorkerLifecycleStubs() (*natsWorkerConnectionLifecycleStub, *natsWorkerSubscriptionStub) { + return &natsWorkerConnectionLifecycleStub{ + published: make(chan struct{}), + drained: make(chan struct{}), + }, &natsWorkerSubscriptionStub{drained: make(chan struct{})} +} + func TestNATSWorker_NewRegisterAndShutdown(t *testing.T) { w := newNATSWorker("nats://example:4222") if w.url != "nats://example:4222" { @@ -48,6 +120,385 @@ func TestNATSWorker_StartWorkersCanceledContext(t *testing.T) { } } +// TestNATSWorkerNilContextAndIdempotentStart verifies a nil startup context is +// normalized and an already-started worker does not reconnect. +func TestNATSWorkerNilContextAndIdempotentStart(t *testing.T) { + w := newNATSWorker("nats://example:4222") + connection, subscription := newNATSWorkerLifecycleStubs() + var calls int + w.connect = func(string, string, nats.MsgHandler) (natsConnection, natsWorkerSubscription, error) { + calls++ + return connection, subscription, nil + } + if err := w.StartWorkers(nil); err != nil { + t.Fatalf("start workers with nil context: %v", err) + } + if err := w.StartWorkers(context.Background()); err != nil { + t.Fatalf("idempotent start workers: %v", err) + } + if calls != 1 { + t.Fatalf("connector calls = %d, want 1", calls) + } + if err := w.Shutdown(nil); err != nil { + t.Fatalf("shutdown with nil context: %v", err) + } +} + +// TestNATSWorkerAcceptedCallbackRunsThroughWorkerLifecycle verifies successful +// readiness admits subscribed messages and shutdown waits for their handlers. +func TestNATSWorkerAcceptedCallbackRunsThroughWorkerLifecycle(t *testing.T) { + w := newNATSWorkerWithConfig(natsWorkerConfig{ + URL: "nats://example:4222", + DefaultQueue: "critical", + Workers: 1, + }) + connection, subscription := newNATSWorkerLifecycleStubs() + var callback nats.MsgHandler + var connectedSubject string + w.connect = func(_ string, subject string, handler nats.MsgHandler) (natsConnection, natsWorkerSubscription, error) { + connectedSubject = subject + callback = handler + return connection, subscription, nil + } + handled := make(chan queue.Job, 1) + handlerStarted := make(chan struct{}) + releaseHandler := make(chan struct{}) + handlerReleased := false + defer func() { + if !handlerReleased { + close(releaseHandler) + } + }() + w.Register("reports:accepted", func(_ context.Context, job queue.Job) error { + handled <- job + close(handlerStarted) + <-releaseHandler + return nil + }) + if err := w.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + if connectedSubject != "queue.critical" || callback == nil { + t.Fatalf("subscription = subject:%q callback:%t", connectedSubject, callback != nil) + } + payload, err := json.Marshal(natsMessage{Type: "reports:accepted", Queue: "critical", Payload: []byte("report")}) + if err != nil { + t.Fatalf("marshal accepted message: %v", err) + } + callback(&nats.Msg{Data: payload}) + select { + case <-handlerStarted: + case <-time.After(time.Second): + t.Fatal("accepted callback did not reach its handler") + } + job := <-handled + if job.Type != "reports:accepted" || string(job.PayloadBytes()) != "report" { + t.Fatalf("handled job = type:%q payload:%q", job.Type, job.PayloadBytes()) + } + + shutdownDone := make(chan error, 1) + go func() { shutdownDone <- w.Shutdown(context.Background()) }() + select { + case <-subscription.drained: + case <-time.After(time.Second): + t.Fatal("shutdown did not drain the subscription") + } + select { + case err := <-shutdownDone: + t.Fatalf("shutdown returned before the handler completed: %v", err) + default: + } + select { + case <-connection.drained: + t.Fatal("connection drained before the handler completed") + default: + } + + handlerReleased = true + close(releaseHandler) + select { + case err := <-shutdownDone: + if err != nil { + t.Fatalf("shutdown: %v", err) + } + case <-time.After(time.Second): + t.Fatal("shutdown did not finish after the handler completed") + } +} + +// TestNATSWorkerRejectsStartDuringShutdown verifies callers can retry a timed +// out drain while worker restart remains blocked. +func TestNATSWorkerRejectsStartDuringShutdown(t *testing.T) { + w := newNATSWorker("nats://example:4222") + connection, subscription := newNATSWorkerLifecycleStubs() + subscription.release = make(chan struct{}) + w.conn = connection + w.sub = subscription + w.started = true + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := w.Shutdown(ctx); !errors.Is(err, context.Canceled) { + t.Fatalf("shutdown error = %v, want context.Canceled", err) + } + if err := w.StartWorkers(context.Background()); !errors.Is(err, queue.ErrQueuerShuttingDown) { + t.Fatalf("start during shutdown error = %v, want ErrQueuerShuttingDown", err) + } + close(subscription.release) + if err := w.Shutdown(context.Background()); err != nil { + t.Fatalf("retry shutdown: %v", err) + } +} + +// TestNATSWorkerStartRetriesAfterConnectionFailure verifies one transient connect error cannot poison worker startup. +func TestNATSWorkerStartRetriesAfterConnectionFailure(t *testing.T) { + w := newNATSWorker("nats://example:4222") + connection, subscription := newNATSWorkerLifecycleStubs() + connectErr := errors.New("nats unavailable") + var calls int + w.connect = func(string, string, nats.MsgHandler) (natsConnection, natsWorkerSubscription, error) { + calls++ + if calls == 1 { + return nil, nil, connectErr + } + return connection, subscription, nil + } + if err := w.StartWorkers(context.Background()); !errors.Is(err, connectErr) { + t.Fatalf("first start error = %v, want %v", err, connectErr) + } + if err := w.StartWorkers(context.Background()); err != nil { + t.Fatalf("retry start: %v", err) + } + if calls != 2 || !w.started || w.conn == nil || w.sub == nil { + t.Fatalf("retry state = calls:%d started:%t conn:%T sub:%T", calls, w.started, w.conn, w.sub) + } + if err := w.Shutdown(context.Background()); err != nil { + t.Fatalf("shutdown: %v", err) + } +} + +// TestNATSWorkerStartRetriesAfterSubscriptionFlushFailure verifies startup is not accepted until the server observes the subscription. +func TestNATSWorkerStartRetriesAfterSubscriptionFlushFailure(t *testing.T) { + w := newNATSWorker("nats://example:4222") + firstConnection, firstSubscription := newNATSWorkerLifecycleStubs() + firstConnection.flushErr = errors.New("subscription flush failed") + secondConnection, secondSubscription := newNATSWorkerLifecycleStubs() + connections := []*natsWorkerConnectionLifecycleStub{firstConnection, secondConnection} + subscriptions := []*natsWorkerSubscriptionStub{firstSubscription, secondSubscription} + var calls int + w.connect = func(string, string, nats.MsgHandler) (natsConnection, natsWorkerSubscription, error) { + index := calls + calls++ + return connections[index], subscriptions[index], nil + } + if err := w.StartWorkers(context.Background()); !errors.Is(err, firstConnection.flushErr) { + t.Fatalf("first start error = %v, want %v", err, firstConnection.flushErr) + } + if !firstConnection.closed || w.started || w.conn != nil || w.sub != nil { + t.Fatalf("failed flush cleanup = closed:%t started:%t conn:%T sub:%T", firstConnection.closed, w.started, w.conn, w.sub) + } + if err := w.StartWorkers(context.Background()); err != nil { + t.Fatalf("retry start after flush failure: %v", err) + } + if calls != 2 || !w.started { + t.Fatalf("retry state = calls:%d started:%t", calls, w.started) + } + if err := w.Shutdown(context.Background()); err != nil { + t.Fatalf("shutdown: %v", err) + } +} + +// TestNATSWorkerFlushFailureRejectsPendingCallbacks verifies a callback that +// arrives during startup cannot escape after readiness fails. +func TestNATSWorkerFlushFailureRejectsPendingCallbacks(t *testing.T) { + w := newNATSWorker("nats://example:4222") + connection, subscription := newNATSWorkerLifecycleStubs() + connection.flushErr = errors.New("subscription flush failed") + var handled int + w.Register("job:pending", func(context.Context, queue.Job) error { + handled++ + return nil + }) + callbackStarted := make(chan struct{}) + callbackDone := make(chan struct{}) + w.connect = func(_ string, _ string, callback nats.MsgHandler) (natsConnection, natsWorkerSubscription, error) { + go func() { + close(callbackStarted) + callback(&nats.Msg{Data: []byte(`{"type":"job:pending","queue":"default"}`)}) + close(callbackDone) + }() + <-callbackStarted + return connection, subscription, nil + } + + if err := w.StartWorkers(context.Background()); !errors.Is(err, connection.flushErr) { + t.Fatalf("start error = %v, want %v", err, connection.flushErr) + } + <-callbackDone + w.running.Wait() + if handled != 0 { + t.Fatalf("failed startup accepted %d callbacks", handled) + } +} + +// TestConnectNATSWorkerRejectsInvalidURL verifies the production connector +// returns parse failures before attempting subscription setup. +func TestConnectNATSWorkerRejectsInvalidURL(t *testing.T) { + nc, sub, err := connectNATSWorker("://bad-url", "queue.default", func(*nats.Msg) {}) + if err == nil || nc != nil || sub != nil { + t.Fatalf("invalid connector result = conn:%T sub:%T err:%v", nc, sub, err) + } +} + +// TestNATSWorkerDefaultConnectorRejectsInvalidURL verifies StartWorkers uses +// the production connector when no custom lifecycle connector is configured. +func TestNATSWorkerDefaultConnectorRejectsInvalidURL(t *testing.T) { + w := newNATSWorker("://bad-url") + if err := w.StartWorkers(context.Background()); err == nil { + t.Fatal("expected default connector to reject invalid URL") + } + if w.started || w.conn != nil || w.sub != nil { + t.Fatalf("failed default start retained state: started:%t conn:%T sub:%T", w.started, w.conn, w.sub) + } +} + +// TestNATSWorkerRepublishRejected verifies a publish rejection is returned +// directly and never followed by a flush. +func TestNATSWorkerRepublishRejected(t *testing.T) { + publishErr := errors.New("retry publish rejected") + connection := &natsConnectionStub{publishErr: publishErr} + w := &natsWorker{conn: connection} + if err := w.republish(natsMessage{Type: "job:retry", Queue: "default"}); !errors.Is(err, publishErr) { + t.Fatalf("republish error = %v, want %v", err, publishErr) + } + if connection.publishN != 1 || connection.flushN != 0 { + t.Fatalf("publish/flush calls = %d/%d, want 1/0", connection.publishN, connection.flushN) + } +} + +// TestNATSWorkerRepublishFlushFailure verifies a lost flush response remains +// an ambiguous error after the retry publish reaches the connection. +func TestNATSWorkerRepublishFlushFailure(t *testing.T) { + flushErr := errors.New("retry flush response lost") + connection := &natsConnectionStub{flushErr: flushErr} + w := &natsWorker{conn: connection} + if err := w.republish(natsMessage{Type: "job:retry", Queue: "default"}); !errors.Is(err, flushErr) { + t.Fatalf("republish error = %v, want %v", err, flushErr) + } + if connection.publishN != 1 || connection.flushN != 1 { + t.Fatalf("publish/flush calls = %d/%d, want 1/1", connection.publishN, connection.flushN) + } +} + +// TestNATSWorkerShutdownDrainDiagnosticConverges verifies completed cleanup does not poison every later root shutdown attempt. +func TestNATSWorkerShutdownDrainDiagnosticConverges(t *testing.T) { + w := newNATSWorker("nats://example:4222") + connection, subscription := newNATSWorkerLifecycleStubs() + drainErr := errors.New("subscription drain diagnostic") + subscription.drainErr = drainErr + w.conn = connection + w.sub = subscription + w.started = true + if err := w.Shutdown(context.Background()); !errors.Is(err, drainErr) { + t.Fatalf("first shutdown error = %v, want %v", err, drainErr) + } + if err := w.Shutdown(context.Background()); err != nil { + t.Fatalf("completed cleanup remained poisoned: %v", err) + } +} + +// TestNATSWorkerShutdownConnectionDrainDiagnosticConverges verifies a completed +// connection drain is reported once without poisoning later root cleanup. +func TestNATSWorkerShutdownConnectionDrainDiagnosticConverges(t *testing.T) { + w := newNATSWorker("nats://example:4222") + connection, subscription := newNATSWorkerLifecycleStubs() + drainErr := errors.New("connection drain diagnostic") + connection.drainErr = drainErr + w.conn = connection + w.sub = subscription + w.started = true + + if err := w.Shutdown(context.Background()); !errors.Is(err, drainErr) { + t.Fatalf("first shutdown error = %v, want %v", err, drainErr) + } + if !connection.closed { + t.Fatal("connection remained open after its drain diagnostic") + } + if err := w.Shutdown(context.Background()); err != nil { + t.Fatalf("completed connection cleanup remained poisoned: %v", err) + } +} + +// TestNATSWorkerShutdownWaitsForInFlightRepublish verifies the connection remains open through a handler's best-effort Core NATS retry publication. +func TestNATSWorkerShutdownWaitsForInFlightRepublish(t *testing.T) { + w := newNATSWorker("nats://example:4222") + connection, subscription := newNATSWorkerLifecycleStubs() + w.conn = connection + w.sub = subscription + w.started = true + handlerStarted := make(chan struct{}) + releaseHandler := make(chan struct{}) + w.Register("job:retry-on-shutdown", func(context.Context, queue.Job) error { + close(handlerStarted) + <-releaseHandler + return errors.New("retry me") + }) + payload, err := json.Marshal(natsMessage{Type: "job:retry-on-shutdown", Queue: "default", MaxRetry: 1}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + w.running.Add(1) + go func() { + defer w.running.Done() + w.processMessage(&nats.Msg{Data: payload}) + }() + <-handlerStarted + shutdownResult := make(chan error, 1) + go func() { shutdownResult <- w.Shutdown(context.Background()) }() + <-subscription.drained + select { + case <-connection.drained: + t.Fatal("connection drained before the in-flight handler finished") + default: + } + close(releaseHandler) + <-connection.published + if err := <-shutdownResult; err != nil { + t.Fatalf("shutdown: %v", err) + } + select { + case <-connection.drained: + default: + t.Fatal("connection did not drain after replacement publication") + } +} + +// TestNATSWorkerShutdownTracksDelayedRepublish verifies timer-backed accepted work finishes before connection drain. +func TestNATSWorkerShutdownTracksDelayedRepublish(t *testing.T) { + w := newNATSWorker("nats://example:4222") + connection, subscription := newNATSWorkerLifecycleStubs() + w.conn = connection + w.sub = subscription + w.started = true + payload, err := json.Marshal(natsMessage{ + Type: "job:delayed-shutdown", + Queue: "default", + AvailableAtMS: time.Now().Add(25 * time.Millisecond).UnixMilli(), + }) + if err != nil { + t.Fatalf("marshal: %v", err) + } + w.processMessage(&nats.Msg{Data: payload}) + if err := w.Shutdown(context.Background()); err != nil { + t.Fatalf("shutdown: %v", err) + } + select { + case <-connection.published: + default: + t.Fatal("shutdown returned before delayed replacement publication") + } +} + func TestNATSWorker_ProcessMessageBranches(t *testing.T) { t.Run("invalid json ignored", func(t *testing.T) { w := newNATSWorker("nats://example:4222") @@ -63,6 +514,29 @@ func TestNATSWorker_ProcessMessageBranches(t *testing.T) { w.processMessage(&nats.Msg{Data: body}) }) + t.Run("expired availability processes immediately", func(t *testing.T) { + w := newNATSWorker("nats://example:4222") + var calls int + w.Register("job:ready", func(context.Context, queue.Job) error { + calls++ + return nil + }) + body, err := json.Marshal(natsMessage{ + Type: "job:ready", + Queue: "default", + AvailableAtMS: time.Now().Add(-time.Second).UnixMilli(), + }) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + w.processMessage(&nats.Msg{Data: body}) + + if calls != 1 { + t.Fatalf("immediate handler calls = %d, want 1", calls) + } + }) + t.Run("success uses timeout and job options", func(t *testing.T) { called := 0 w := newNATSWorker("nats://example:4222") @@ -105,7 +579,7 @@ func TestNATSWorker_ProcessMessageBranches(t *testing.T) { w := newNATSWorkerWithConfig(natsWorkerConfig{ URL: "nats://example:4222", Workers: 1, - Observer: queue.ObserverFunc(func(e queue.Event) { events = append(events, e) }), + Observer: queue.ObserverFunc(func(_ context.Context, e queue.Event) { events = append(events, e) }), }) w.Register("job:fail", func(context.Context, queue.Job) error { return errors.New("boom") }) body, err := json.Marshal(natsMessage{Type: "job:fail", Queue: "default", Attempt: 0, MaxRetry: 2, BackoffMillis: 5}) @@ -116,6 +590,9 @@ func TestNATSWorker_ProcessMessageBranches(t *testing.T) { if len(events) == 0 || events[0].Kind != queue.EventRepublishFailed || events[0].Driver != queue.DriverNATS { t.Fatalf("expected republish_failed nats event, got %+v", events) } + if events[0].Layer != queue.EventLayerWorker { + t.Fatalf("republish_failed layer = %q, want worker", events[0].Layer) + } }) t.Run("republish failure unwraps bus envelope job type", func(t *testing.T) { @@ -123,7 +600,7 @@ func TestNATSWorker_ProcessMessageBranches(t *testing.T) { w := newNATSWorkerWithConfig(natsWorkerConfig{ URL: "nats://example:4222", Workers: 1, - Observer: queue.ObserverFunc(func(e queue.Event) { events = append(events, e) }), + Observer: queue.ObserverFunc(func(_ context.Context, e queue.Event) { events = append(events, e) }), }) w.Register("bus:job", func(context.Context, queue.Job) error { return errors.New("boom") }) body, err := json.Marshal(natsMessage{ @@ -132,7 +609,7 @@ func TestNATSWorker_ProcessMessageBranches(t *testing.T) { Attempt: 0, MaxRetry: 2, BackoffMillis: 5, - Payload: []byte(`{"job":{"type":"monitoring:check"}}`), + Payload: []byte(`{"schema_version":1,"dispatch_id":"dsp_nats","job_id":"job_nats","chain_id":"chn_nats","job":{"type":"monitoring:check"}}`), }) if err != nil { t.Fatalf("marshal: %v", err) @@ -144,6 +621,9 @@ func TestNATSWorker_ProcessMessageBranches(t *testing.T) { if events[0].JobType != "monitoring:check" { t.Fatalf("expected unwrapped observed job type, got %q", events[0].JobType) } + if events[0].DispatchID != "dsp_nats" || events[0].JobID != "job_nats" || events[0].ChainID != "chn_nats" { + t.Fatalf("expected correlated nats event, got %+v", events[0]) + } }) t.Run("failed handler at max retries stops", func(t *testing.T) { @@ -156,3 +636,97 @@ func TestNATSWorker_ProcessMessageBranches(t *testing.T) { w.processMessage(&nats.Msg{Data: body}) }) } + +// TestNATSWorker_AttemptDecisionSettlement verifies terminal and uncommitted outcomes choose distinct Core NATS settlement paths. +func TestNATSWorker_AttemptDecisionSettlement(t *testing.T) { + t.Run("permanent failure does not republish", func(t *testing.T) { + var events []queue.Event + w := newNATSWorkerWithConfig(natsWorkerConfig{ + URL: "nats://example:4222", + Observer: queue.ObserverFunc(func(_ context.Context, event queue.Event) { events = append(events, event) }), + }) + w.Register("job:permanent", func(ctx context.Context, _ queue.Job) error { + attempt, ok := busruntime.DeliveryAttemptFromContext(ctx) + if !ok || attempt.Number != 0 || attempt.MaxRetry != 3 { + t.Fatalf("unexpected delivery attempt: %+v, present=%t", attempt, ok) + } + return busruntime.Permanent(errors.New("invalid job")) + }) + body, err := json.Marshal(natsMessage{Type: "job:permanent", Queue: "default", MaxRetry: 3}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + w.processMessage(&nats.Msg{Data: body}) + + if len(events) != 0 { + t.Fatalf("permanent failure must not reach the republish path, got %+v", events) + } + }) + + t.Run("uncommitted failure republishes the same attempt", func(t *testing.T) { + var events []queue.Event + w := newNATSWorkerWithConfig(natsWorkerConfig{ + URL: "nats://example:4222", + Observer: queue.ObserverFunc(func(_ context.Context, event queue.Event) { events = append(events, event) }), + }) + w.Register("job:uncommitted", func(ctx context.Context, _ queue.Job) error { + attempt, ok := busruntime.DeliveryAttemptFromContext(ctx) + if !ok || attempt.Number != 1 || attempt.MaxRetry != 4 { + t.Fatalf("unexpected delivery attempt: %+v, present=%t", attempt, ok) + } + return busruntime.Uncommitted(errors.New("store unavailable")) + }) + body, err := json.Marshal(natsMessage{ + Type: "job:uncommitted", + Queue: "default", + Attempt: 1, + MaxRetry: 4, + BackoffMillis: 1_000, + }) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + w.processMessage(&nats.Msg{Data: body}) + + if len(events) != 1 || events[0].Kind != queue.EventRepublishFailed { + t.Fatalf("core NATS must attempt to republish uncommitted work, got %+v", events) + } + if events[0].Attempt != 1 || events[0].MaxRetry != 4 { + t.Fatalf("uncommitted republish consumed an application attempt: %+v", events[0]) + } + }) + + t.Run("successful uncommitted republish preserves attempt and clears delay", func(t *testing.T) { + connection := &natsConnectionStub{} + w := newNATSWorker("nats://example:4222") + w.conn = connection + w.Register("job:uncommitted", func(context.Context, queue.Job) error { + return busruntime.Uncommitted(errors.New("store unavailable")) + }) + body, err := json.Marshal(natsMessage{ + Type: "job:uncommitted", + Queue: "default", + Attempt: 1, + MaxRetry: 4, + AvailableAtMS: time.Now().Add(-time.Minute).UnixMilli(), + }) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + w.processMessage(&nats.Msg{Data: body}) + + if connection.publishN != 1 || connection.flushN != 1 { + t.Fatalf("republish calls = publish:%d flush:%d, want 1/1", connection.publishN, connection.flushN) + } + var republished natsMessage + if err := json.Unmarshal(connection.published[0], &republished); err != nil { + t.Fatalf("decode republished message: %v", err) + } + if republished.Attempt != 1 || republished.AvailableAtMS != 0 { + t.Fatalf("republished attempt/delay = %d/%d, want 1/0", republished.Attempt, republished.AvailableAtMS) + } + }) +} diff --git a/driver/postgresqueue/go.mod b/driver/postgresqueue/go.mod index d56faac..0272dc3 100644 --- a/driver/postgresqueue/go.mod +++ b/driver/postgresqueue/go.mod @@ -1,11 +1,11 @@ module github.com/goforj/queue/driver/postgresqueue -go 1.24.4 +go 1.25.0 require ( github.com/goforj/queue v0.0.0 github.com/goforj/queue/driver/sqlqueuecore v0.0.0 - github.com/jackc/pgx/v5 v5.8.0 + github.com/jackc/pgx/v5 v5.9.2 ) require ( diff --git a/driver/postgresqueue/go.sum b/driver/postgresqueue/go.sum index dec6e1f..16aacb3 100644 --- a/driver/postgresqueue/go.sum +++ b/driver/postgresqueue/go.sum @@ -9,8 +9,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.8.0 h1:TYPDoleBBme0xGSAX3/+NujXXtpZn9HBONkQC7IEZSo= -github.com/jackc/pgx/v5 v5.8.0/go.mod h1:QVeDInX2m9VyzvNeiCJVjCkNFqzsNb43204HshNSZKw= +github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw= +github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= diff --git a/driver/postgresqueue/postgresqueue.go b/driver/postgresqueue/postgresqueue.go index 30d41c4..0d51440 100644 --- a/driver/postgresqueue/postgresqueue.go +++ b/driver/postgresqueue/postgresqueue.go @@ -1,3 +1,7 @@ +// Package postgresqueue provides the PostgreSQL-backed queue driver. +// +// This module requires Go 1.25 or newer because its pgx dependency uses that +// baseline beginning with the first release that fixes GO-2026-5004. package postgresqueue import ( @@ -10,10 +14,12 @@ import ( _ "github.com/jackc/pgx/v5/stdlib" ) +// Config configures the PostgreSQL queue driver. type Config struct { queueconfig.DriverBaseConfig DB *sql.DB DSN string + DisableAutoMigrate bool ProcessingRecoveryGrace time.Duration ProcessingLeaseNoTimeout time.Duration } @@ -48,6 +54,7 @@ func New(dsn string, opts ...queue.Option) (*queue.Queue, error) { // }, // DB: nil, // optional; provide *sql.DB instead of DSN // DSN: "postgres://user:pass@127.0.0.1:5432/queue?sslmode=disable", // optional if DB is set +// DisableAutoMigrate: false, // set true when schema migrations are managed externally // ProcessingRecoveryGrace: 2 * time.Second, // default if <=0: 2s // ProcessingLeaseNoTimeout: 5 * time.Minute, // default if <=0: 5m // }, @@ -62,6 +69,7 @@ func NewWithConfig(cfg Config, opts ...queue.Option) (*queue.Queue, error) { DriverBaseConfig: cfg.DriverBaseConfig, DB: cfg.DB, DSN: cfg.DSN, + DisableAutoMigrate: cfg.DisableAutoMigrate, ProcessingRecoveryGrace: cfg.ProcessingRecoveryGrace, ProcessingLeaseNoTimeout: cfg.ProcessingLeaseNoTimeout, }, opts...) diff --git a/driver/rabbitmqqueue/queue_rabbitmq_impl.go b/driver/rabbitmqqueue/queue_rabbitmq_impl.go index 4767e50..3cc0e60 100644 --- a/driver/rabbitmqqueue/queue_rabbitmq_impl.go +++ b/driver/rabbitmqqueue/queue_rabbitmq_impl.go @@ -10,20 +10,24 @@ import ( "time" "github.com/goforj/queue" + "github.com/goforj/queue/internal/uniqueness" "github.com/goforj/queue/queuecore" amqp "github.com/rabbitmq/amqp091-go" ) +const rabbitPublishConfirmationTimeout = 15 * time.Second + type rabbitMQMessage struct { - Type string `json:"type"` - Payload []byte `json:"payload,omitempty"` - Queue string `json:"queue"` - Attempt int `json:"attempt,omitempty"` - MaxRetry int `json:"max_retry,omitempty"` - BackoffMillis int64 `json:"backoff_millis,omitempty"` - TimeoutMillis int64 `json:"timeout_millis,omitempty"` - AvailableAtMS int64 `json:"available_at_ms,omitempty"` - PublishedAtMS int64 `json:"published_at_ms,omitempty"` + Type string `json:"type"` + Payload []byte `json:"payload,omitempty"` + Queue string `json:"queue"` + Metadata json.RawMessage `json:"metadata,omitempty"` + Attempt int `json:"attempt,omitempty"` + MaxRetry int `json:"max_retry,omitempty"` + BackoffMillis int64 `json:"backoff_millis,omitempty"` + TimeoutMillis int64 `json:"timeout_millis,omitempty"` + AvailableAtMS int64 `json:"available_at_ms,omitempty"` + PublishedAtMS int64 `json:"published_at_ms,omitempty"` } type rabbitMQQueue struct { @@ -34,7 +38,7 @@ type rabbitMQQueue struct { mu sync.Mutex conn *amqp.Connection ch *amqp.Channel - unique map[string]time.Time + unique uniqueness.MemoryStore } func newRabbitMQQueue(url string, defaultQueue string) *rabbitMQQueue { @@ -44,7 +48,6 @@ func newRabbitMQQueue(url string, defaultQueue string) *rabbitMQQueue { return &rabbitMQQueue{ url: url, defaultQueue: defaultQueue, - unique: make(map[string]time.Time), } } @@ -71,10 +74,14 @@ func (q *rabbitMQQueue) Shutdown(_ context.Context) error { return nil } +// Dispatch requires a positive publisher confirmation before reporting acceptance. func (q *rabbitMQQueue) Dispatch(ctx context.Context, job queue.Job) error { if ctx == nil { ctx = context.Background() } + if err := ctx.Err(); err != nil { + return err + } if err := queuecore.ValidateDriverJob(job); err != nil { return err } @@ -82,70 +89,109 @@ func (q *rabbitMQQueue) Dispatch(ctx context.Context, job queue.Job) error { if parsed.QueueName == "" { return fmt.Errorf("job queue is required") } - if parsed.UniqueTTL > 0 && !q.claimUnique(job, parsed.QueueName, parsed.UniqueTTL) { - return queuecore.ErrDuplicate + var ( + uniqueKey string + uniqueToken uint64 + ) + if parsed.UniqueTTL > 0 { + var acquired bool + uniqueKey, uniqueToken, acquired = q.claimUnique(job, parsed.QueueName, parsed.UniqueTTL) + if !acquired { + return queuecore.ErrDuplicate + } } + message, err := rabbitMQMessageForJob(job, parsed) + if err != nil { + q.unique.Release(uniqueKey, uniqueToken) + return err + } + body, err := json.Marshal(message) + if err != nil { + q.unique.Release(uniqueKey, uniqueToken) + return err + } + + q.mu.Lock() + targetQueue := rabbitPhysicalQueueName(q.defaultQueue, parsed.QueueName) + err = q.enqueueWithReconnectLocked(ctx, targetQueue, body) + q.mu.Unlock() + if err != nil && !isRabbitPublishAmbiguous(err) { + q.unique.Release(uniqueKey, uniqueToken) + } + return err +} + +// rabbitMQMessageForJob converts one validated queue job into the stable +// RabbitMQ wire representation while keeping direct-delivery metadata optional. +func rabbitMQMessageForJob(job queue.Job, options queue.DriverJobOptions) (rabbitMQMessage, error) { message := rabbitMQMessage{ Type: job.Type, Payload: job.PayloadBytes(), - Queue: parsed.QueueName, + Queue: options.QueueName, PublishedAtMS: time.Now().UnixMilli(), } - if parsed.MaxRetry != nil { - message.MaxRetry = *parsed.MaxRetry + metadata := queue.DriverMetadata(job) + if metadata.SchemaVersion != 0 { + encoded, err := json.Marshal(metadata) + if err != nil { + return rabbitMQMessage{}, fmt.Errorf("encode RabbitMQ driver job metadata: %w", err) + } + message.Metadata = encoded } - if parsed.Backoff != nil && *parsed.Backoff > 0 { - message.BackoffMillis = parsed.Backoff.Milliseconds() + if options.MaxRetry != nil { + message.MaxRetry = *options.MaxRetry } - if parsed.Timeout != nil && *parsed.Timeout > 0 { - message.TimeoutMillis = parsed.Timeout.Milliseconds() + if options.Backoff != nil && *options.Backoff > 0 { + message.BackoffMillis = options.Backoff.Milliseconds() } - if parsed.Delay > 0 { - message.AvailableAtMS = time.Now().Add(parsed.Delay).UnixMilli() + if options.Timeout != nil && *options.Timeout > 0 { + message.TimeoutMillis = options.Timeout.Milliseconds() } - body, err := json.Marshal(message) - if err != nil { - return err + if options.Delay > 0 { + message.AvailableAtMS = time.Now().Add(options.Delay).UnixMilli() } + return message, nil +} - q.mu.Lock() - defer q.mu.Unlock() - if err := q.ensureConnectedLocked(); err != nil { +// claimUnique returns the ownership token needed to compensate a rejected publish. +func (q *rabbitMQQueue) claimUnique(job queue.Job, queueName string, ttl time.Duration) (string, uint64, bool) { + key := queuecore.UniqueKey(job, queueName) + token, ok := q.unique.Acquire(key, ttl) + return key, token, ok +} + +// enqueueWithReconnectLocked retries one publish after replacing a closed connection. +func (q *rabbitMQQueue) enqueueWithReconnectLocked(ctx context.Context, queueName string, body []byte) error { + return retryRabbitPublish( + q.ensureConnectedLocked, + func() error { return q.enqueueLocked(ctx, queueName, body) }, + q.closeLocked, + ) +} + +// retryRabbitPublish reconnects only when the first publish is known to have +// failed because the transport was already closed. +func retryRabbitPublish(ensureConnected func() error, publish func() error, closeConnection func()) error { + if err := ensureConnected(); err != nil { return err } - targetQueue := rabbitPhysicalQueueName(q.defaultQueue, parsed.QueueName) - if err := q.enqueueLocked(ctx, targetQueue, body); err != nil { + if err := publish(); err != nil { + if isRabbitPublishAmbiguous(err) { + return err + } if !isRabbitConnectionClosed(err) { return err } - q.closeLocked() - if reconnectErr := q.ensureConnectedLocked(); reconnectErr != nil { + closeConnection() + if reconnectErr := ensureConnected(); reconnectErr != nil { return reconnectErr } - return q.enqueueLocked(ctx, targetQueue, body) + return publish() } return nil } -func (q *rabbitMQQueue) claimUnique(job queue.Job, queueName string, ttl time.Duration) bool { - now := time.Now() - key := queueName + ":" + job.Type + ":" + string(job.PayloadBytes()) - - q.mu.Lock() - defer q.mu.Unlock() - for candidate, expiresAt := range q.unique { - if expiresAt.Before(now) { - delete(q.unique, candidate) - } - } - if expiresAt, ok := q.unique[key]; ok && expiresAt.After(now) { - return false - } - q.unique[key] = now.Add(ttl) - return true -} - func (q *rabbitMQQueue) ensureConnectedLocked() error { if q.conn != nil && !q.conn.IsClosed() && q.ch != nil && !q.ch.IsClosed() { return nil @@ -164,6 +210,11 @@ func (q *rabbitMQQueue) ensureConnectedLocked() error { _ = conn.Close() return err } + if err := ch.Confirm(false); err != nil { + _ = ch.Close() + _ = conn.Close() + return err + } q.conn = conn q.ch = ch return nil @@ -187,13 +238,89 @@ func (q *rabbitMQQueue) enqueueLocked(ctx context.Context, queueName string, bod if _, err := q.ch.QueueDeclare(queueName, true, false, false, false, nil); err != nil { return err } - return q.ch.PublishWithContext(ctx, "", queueName, false, false, amqp.Publishing{ + return publishRabbitConfirmed(ctx, q.ch, "", queueName, amqp.Publishing{ ContentType: "application/json", Body: body, DeliveryMode: amqp.Persistent, }) } +type rabbitPublishConfirmation interface { + WaitContext(ctx context.Context) (bool, error) +} + +type rabbitPublishAmbiguousError struct { + cause error +} + +// Error describes a publish whose broker outcome could not be determined. +func (e rabbitPublishAmbiguousError) Error() string { return e.cause.Error() } + +// Unwrap preserves the network or context cause for diagnostics. +func (e rabbitPublishAmbiguousError) Unwrap() error { return e.cause } + +// publishRabbitConfirmed waits for the broker to accept a persistent publish before its caller commits acceptance. +func publishRabbitConfirmed(ctx context.Context, ch *amqp.Channel, exchange, queueName string, message amqp.Publishing) error { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } + if ch == nil { + return amqp.ErrClosed + } + confirmation, err := ch.PublishWithDeferredConfirmWithContext(ctx, exchange, queueName, false, false, message) + return completeRabbitPublish(ctx, confirmation, err) +} + +// completeRabbitPublish preserves send ambiguity before waiting for the +// broker's positive confirmation. +func completeRabbitPublish(ctx context.Context, confirmation rabbitPublishConfirmation, publishErr error) error { + if publishErr != nil { + return rabbitPublishAmbiguousError{cause: publishErr} + } + return awaitRabbitConfirmation(ctx, confirmation) +} + +// rabbitPublishContext caps broker-confirmation latency while preserving any shorter caller deadline. +func rabbitPublishContext(ctx context.Context) (context.Context, context.CancelFunc, error) { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return nil, nil, err + } + bounded, cancel := context.WithTimeout(ctx, rabbitPublishConfirmationTimeout) + return bounded, cancel, nil +} + +// awaitRabbitConfirmation rejects negative and ambiguous broker acknowledgements. +func awaitRabbitConfirmation(ctx context.Context, confirmation rabbitPublishConfirmation) error { + bounded, cancel, err := rabbitPublishContext(ctx) + if err != nil { + return err + } + defer cancel() + if confirmation == nil { + return rabbitPublishAmbiguousError{cause: fmt.Errorf("rabbitmq publish returned no confirmation")} + } + acked, err := confirmation.WaitContext(bounded) + if err != nil { + return rabbitPublishAmbiguousError{cause: fmt.Errorf("wait for rabbitmq publish confirmation: %w", err)} + } + if !acked { + return fmt.Errorf("rabbitmq broker rejected publish") + } + return nil +} + +// isRabbitPublishAmbiguous identifies failures that may have occurred after the broker accepted a publish. +func isRabbitPublishAmbiguous(err error) bool { + var ambiguous rabbitPublishAmbiguousError + return errors.As(err, &ambiguous) +} + func isRabbitConnectionClosed(err error) bool { if err == nil { return false diff --git a/driver/rabbitmqqueue/queue_rabbitmq_impl_test.go b/driver/rabbitmqqueue/queue_rabbitmq_impl_test.go index 688c198..d07bed2 100644 --- a/driver/rabbitmqqueue/queue_rabbitmq_impl_test.go +++ b/driver/rabbitmqqueue/queue_rabbitmq_impl_test.go @@ -1,7 +1,9 @@ package rabbitmqqueue import ( + "bytes" "context" + "encoding/json" "errors" "testing" "time" @@ -10,6 +12,134 @@ import ( amqp "github.com/rabbitmq/amqp091-go" ) +// TestRabbitMQDirectDeliveryMetadataRoundTrip verifies producer framing, worker +// reconstruction, retry preservation, and legacy-envelope observation. +func TestRabbitMQDirectDeliveryMetadataRoundTrip(t *testing.T) { + wantMetadata := queue.DriverJobMetadata{ + SchemaVersion: queue.DriverJobMetadataVersion, + DispatchID: "dsp_rabbit_direct", + JobID: "job_rabbit_direct", + Queue: "critical", + } + wantPayload := []byte(`{"report_id":42}`) + job := queue.DriverWithMetadata( + queue.NewJob("reports:build").Payload(wantPayload).OnQueue("critical").Retry(3), + wantMetadata, + ) + message, err := rabbitMQMessageForJob(job, queue.DriverOptions(job)) + if err != nil { + t.Fatalf("build direct message: %v", err) + } + var wireMetadata queue.DriverJobMetadata + if err := json.Unmarshal(message.Metadata, &wireMetadata); err != nil || wireMetadata != wantMetadata { + t.Fatalf("wire metadata = %+v, want %+v (err=%v)", wireMetadata, wantMetadata, err) + } + + wire, err := json.Marshal(message) + if err != nil { + t.Fatalf("marshal direct message: %v", err) + } + var decoded rabbitMQMessage + if err := json.Unmarshal(wire, &decoded); err != nil { + t.Fatalf("unmarshal direct message: %v", err) + } + delivery := rabbitMQDeliveryJob(decoded) + if delivery.Type != "reports:build" || !bytes.Equal(delivery.PayloadBytes(), wantPayload) { + t.Fatalf("delivery = type:%q payload:%q", delivery.Type, delivery.PayloadBytes()) + } + if got := queue.DriverMetadata(delivery); got != wantMetadata { + t.Fatalf("reconstructed metadata = %+v, want %+v", got, wantMetadata) + } + observed := queue.ResolveObservedJobMetadataFromJob(delivery) + if observed.DispatchID != wantMetadata.DispatchID || observed.JobID != wantMetadata.JobID || observed.JobType != job.Type { + t.Fatalf("direct observation = %+v", observed) + } + var events []queue.Event + worker := &rabbitMQWorker{observer: queue.ObserverFunc(func(_ context.Context, event queue.Event) { + events = append(events, event) + })} + worker.observeRepublishFailure(context.Background(), decoded, errors.New("republish failed")) + if len(events) != 1 || events[0].DispatchID != wantMetadata.DispatchID || events[0].JobID != wantMetadata.JobID { + t.Fatalf("direct republish observation = %+v", events) + } + + decoded.Attempt++ + var retry rabbitMQMessage + worker.cfg.DefaultQueue = "default" + worker.publishOverride = func(_ context.Context, message rabbitMQMessage) error { + retry = message + return nil + } + if err := worker.publish(context.Background(), decoded); err != nil { + t.Fatalf("republish direct message: %v", err) + } + retryJob := rabbitMQDeliveryJob(retry) + if got := queue.DriverMetadata(retryJob); got != wantMetadata { + t.Fatalf("retry metadata = %+v, want %+v", got, wantMetadata) + } + if got := queue.DriverOptions(retryJob).Attempt; got != 1 { + t.Fatalf("retry attempt = %d, want 1", got) + } + + legacyPayload := []byte(`{"schema_version":1,"dispatch_id":"dsp_rabbit_legacy","job_id":"job_rabbit_legacy","job":{"type":"reports:legacy","payload":"e30="}}`) + legacy := queue.ResolveObservedJobMetadataFromJob(rabbitMQDeliveryJob(rabbitMQMessage{Type: "bus:job", Payload: legacyPayload})) + if legacy.JobType != "reports:legacy" || legacy.DispatchID != "dsp_rabbit_legacy" || legacy.JobID != "job_rabbit_legacy" { + t.Fatalf("legacy observation = %+v", legacy) + } + + plainJob := queue.NewJob("reports:plain").OnQueue("default") + plain, err := rabbitMQMessageForJob(plainJob, queue.DriverOptions(plainJob)) + if err != nil { + t.Fatalf("build metadata-absent message: %v", err) + } + plainWire, err := json.Marshal(plain) + if err != nil { + t.Fatalf("marshal metadata-absent message: %v", err) + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(plainWire, &fields); err != nil { + t.Fatalf("inspect metadata-absent message: %v", err) + } + if _, ok := fields["metadata"]; ok { + t.Fatalf("metadata-absent wire unexpectedly contains metadata: %s", plainWire) + } +} + +// TestRabbitMQUntrustedMetadataRemainsAnOpaqueRetrySidecar verifies valid +// application bytes survive malformed metadata and future fields survive republish. +func TestRabbitMQUntrustedMetadataRemainsAnOpaqueRetrySidecar(t *testing.T) { + for _, raw := range []string{`"malformed"`, `{"schema_version":"bad","dispatch_id":"spoofed"}`} { + wire := []byte(`{"type":"reports:build","payload":"AQI=","queue":"critical","metadata":` + raw + `}`) + var message rabbitMQMessage + if err := json.Unmarshal(wire, &message); err != nil { + t.Fatalf("decode message with metadata %s: %v", raw, err) + } + delivery := rabbitMQDeliveryJob(message) + if delivery.Type != "reports:build" || !bytes.Equal(delivery.PayloadBytes(), []byte{1, 2}) { + t.Fatalf("delivery with metadata %s = type:%q payload:%v", raw, delivery.Type, delivery.PayloadBytes()) + } + if metadata := queue.DriverMetadata(delivery); metadata != (queue.DriverJobMetadata{}) { + t.Fatalf("untrusted metadata %s became trusted: %+v", raw, metadata) + } + } + + future := json.RawMessage(`{"schema_version":99,"dispatch_id":"future","future_field":{"id":7}}`) + var retry rabbitMQMessage + worker := &rabbitMQWorker{publishOverride: func(_ context.Context, message rabbitMQMessage) error { + retry = message + return nil + }} + if err := worker.publish(context.Background(), rabbitMQMessage{Type: "reports:build", Queue: "critical", Metadata: future}); err != nil { + t.Fatalf("republish future metadata: %v", err) + } + if !bytes.Equal(retry.Metadata, future) { + t.Fatalf("future retry metadata = %s, want %s", retry.Metadata, future) + } + if metadata := queue.DriverMetadata(rabbitMQDeliveryJob(retry)); metadata != (queue.DriverJobMetadata{}) { + t.Fatalf("future metadata became trusted: %+v", metadata) + } +} + func TestRabbitMQQueue_HelperBranches(t *testing.T) { qDefault := newRabbitMQQueue("amqp://example", "") if qDefault.defaultQueue != "default" { @@ -33,6 +163,25 @@ func TestRabbitMQQueue_HelperBranches(t *testing.T) { } } +// TestRabbitMQQueueDriverAndPreflight verifies driver identity and preflight +// cancellation before exercising the deterministic connection rejection path. +func TestRabbitMQQueueDriverAndPreflight(t *testing.T) { + q := newRabbitMQQueue("://bad-url", "default") + q.dialTimeout = 5 * time.Millisecond + if got := q.Driver(); got != queue.DriverRabbitMQ { + t.Fatalf("driver = %q, want %q", got, queue.DriverRabbitMQ) + } + + canceled, cancel := context.WithCancel(context.Background()) + cancel() + if err := q.Preflight(canceled); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled preflight error = %v, want context.Canceled", err) + } + if err := q.Preflight(nil); err == nil { + t.Fatal("invalid RabbitMQ URL unexpectedly passed preflight") + } +} + func TestRabbitMQQueue_DispatchValidationAndDuplicate(t *testing.T) { q := newRabbitMQQueue("amqp://example", "default") @@ -44,23 +193,163 @@ func TestRabbitMQQueue_DispatchValidationAndDuplicate(t *testing.T) { } job := queue.NewJob("job:dup").Payload([]byte(`{"k":"v"}`)).OnQueue("default").UniqueFor(10 * time.Second) - _ = q.claimUnique(job, "default", 10*time.Second) + _, _, _ = q.claimUnique(job, "default", 10*time.Second) if err := q.Dispatch(context.Background(), job); !errors.Is(err, queue.ErrDuplicate) { t.Fatalf("expected ErrDuplicate before dial path, got %v", err) } } +// TestRabbitMQQueueCanceledDispatchStopsBeforeClaim verifies cancellation cannot publish or consume uniqueness state. +func TestRabbitMQQueueCanceledDispatchStopsBeforeClaim(t *testing.T) { + q := newRabbitMQQueue("://bad-url", "default") + job := queue.NewJob("job:canceled").OnQueue("default").UniqueFor(time.Minute) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + if err := q.Dispatch(ctx, job); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled dispatch error = %v, want context.Canceled", err) + } + key, token, acquired := q.claimUnique(job, "default", time.Minute) + if !acquired { + t.Fatal("canceled dispatch consumed the uniqueness claim") + } + q.unique.Release(key, token) +} + func TestRabbitMQQueue_ClaimUniquePrunesExpired(t *testing.T) { q := newRabbitMQQueue("amqp://example", "default") job := queue.NewJob("job:unique").Payload([]byte(`{"id":1}`)).OnQueue("default") - key := "default:" + job.Type + ":" + string(job.PayloadBytes()) - q.unique[key] = time.Now().Add(-time.Second) - - if ok := q.claimUnique(job, "default", 5*time.Second); !ok { + if _, _, ok := q.claimUnique(job, "default", time.Millisecond); !ok { + t.Fatal("expected initial claim to succeed") + } + time.Sleep(2 * time.Millisecond) + if _, _, ok := q.claimUnique(job, "default", 5*time.Second); !ok { t.Fatal("expected expired key to be pruned and claim to succeed") } } +// TestRabbitMQQueueRejectedDispatchReleasesUniqueClaim verifies connection rejection cannot retain a false acceptance. +func TestRabbitMQQueueRejectedDispatchReleasesUniqueClaim(t *testing.T) { + q := newRabbitMQQueue("://bad-url", "default") + q.dialTimeout = 5 * time.Millisecond + job := queue.NewJob("job:unique:rejected").OnQueue("default").UniqueFor(time.Minute) + first := q.Dispatch(context.Background(), job) + if first == nil || errors.Is(first, queue.ErrDuplicate) { + t.Fatalf("first dispatch error = %v, want connection rejection", first) + } + second := q.Dispatch(context.Background(), job) + if second == nil || errors.Is(second, queue.ErrDuplicate) { + t.Fatalf("second dispatch error = %v, uniqueness claim was not compensated", second) + } +} + +// TestRetryRabbitPublish verifies reconnect decisions without weakening the +// broker ambiguity rule that prevents duplicate publishes. +func TestRetryRabbitPublish(t *testing.T) { + connectErr := errors.New("connect failed") + publishErr := errors.New("publish rejected") + reconnectErr := errors.New("reconnect failed") + retryErr := errors.New("retry failed") + ambiguousCause := errors.New("publish outcome unknown") + ambiguousErr := rabbitPublishAmbiguousError{cause: ambiguousCause} + tests := []struct { + name string + ensureErrors []error + publishErrors []error + wantCalls []string + wantErr error + }{ + { + name: "initial connection rejection", + ensureErrors: []error{connectErr}, + wantCalls: []string{"ensure"}, + wantErr: connectErr, + }, + { + name: "accepted first publish", + ensureErrors: []error{nil}, + publishErrors: []error{nil}, + wantCalls: []string{"ensure", "publish"}, + }, + { + name: "ambiguous publish is not repeated", + ensureErrors: []error{nil}, + publishErrors: []error{ambiguousErr}, + wantCalls: []string{"ensure", "publish"}, + wantErr: ambiguousCause, + }, + { + name: "non-connection rejection is not repeated", + ensureErrors: []error{nil}, + publishErrors: []error{publishErr}, + wantCalls: []string{"ensure", "publish"}, + wantErr: publishErr, + }, + { + name: "reconnect rejection", + ensureErrors: []error{nil, reconnectErr}, + publishErrors: []error{amqp.ErrClosed}, + wantCalls: []string{"ensure", "publish", "close", "ensure"}, + wantErr: reconnectErr, + }, + { + name: "accepted retry", + ensureErrors: []error{nil, nil}, + publishErrors: []error{amqp.ErrClosed, nil}, + wantCalls: []string{"ensure", "publish", "close", "ensure", "publish"}, + }, + { + name: "retry rejection", + ensureErrors: []error{nil, nil}, + publishErrors: []error{amqp.ErrClosed, retryErr}, + wantCalls: []string{"ensure", "publish", "close", "ensure", "publish"}, + wantErr: retryErr, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var calls []string + ensureIndex := 0 + publishIndex := 0 + ensureConnected := func() error { + calls = append(calls, "ensure") + if ensureIndex >= len(test.ensureErrors) { + t.Fatal("unexpected ensure call") + return nil + } + err := test.ensureErrors[ensureIndex] + ensureIndex++ + return err + } + publish := func() error { + calls = append(calls, "publish") + if publishIndex >= len(test.publishErrors) { + t.Fatal("unexpected publish call") + return nil + } + err := test.publishErrors[publishIndex] + publishIndex++ + return err + } + closeConnection := func() { calls = append(calls, "close") } + + err := retryRabbitPublish(ensureConnected, publish, closeConnection) + if !errors.Is(err, test.wantErr) { + t.Fatalf("retry error = %v, want %v", err, test.wantErr) + } + if len(calls) != len(test.wantCalls) { + t.Fatalf("calls = %v, want %v", calls, test.wantCalls) + } + for i := range calls { + if calls[i] != test.wantCalls[i] { + t.Fatalf("calls = %v, want %v", calls, test.wantCalls) + } + } + }) + } +} + func TestRabbitMQQueue_EnsureConnectedLockedAndErrorClassifier(t *testing.T) { q := newRabbitMQQueue("://bad-url", "default") q.dialTimeout = 5 * time.Millisecond @@ -93,3 +382,26 @@ func TestRabbitPhysicalQueueName(t *testing.T) { t.Fatalf("expected hard default fallback, got %q", got) } } + +// TestRabbitPublishHelpersNormalizeNilContexts verifies optional caller contexts +// still reach deterministic pre-publish validation. +func TestRabbitPublishHelpersNormalizeNilContexts(t *testing.T) { + if err := publishRabbitConfirmed(nil, nil, "", "default", amqp.Publishing{}); !errors.Is(err, amqp.ErrClosed) { + t.Fatalf("nil-channel publish error = %v, want amqp.ErrClosed", err) + } + + ctx, cancel, err := rabbitPublishContext(nil) + if err != nil { + t.Fatalf("nil-context publish boundary: %v", err) + } + cancel() + if ctx == nil { + t.Fatal("nil caller context produced a nil bounded context") + } + + cause := errors.New("confirmation lost") + ambiguous := rabbitPublishAmbiguousError{cause: cause} + if got := ambiguous.Error(); got != cause.Error() { + t.Fatalf("ambiguous error text = %q, want %q", got, cause.Error()) + } +} diff --git a/driver/rabbitmqqueue/rabbitmqqueue.go b/driver/rabbitmqqueue/rabbitmqqueue.go index 128f899..78860e2 100644 --- a/driver/rabbitmqqueue/rabbitmqqueue.go +++ b/driver/rabbitmqqueue/rabbitmqqueue.go @@ -54,10 +54,11 @@ func NewWithConfig(cfg Config, opts ...queue.Option) (*queue.Queue, error) { if cfg.URL == "" { return nil, fmt.Errorf("rabbitmq url is required") } + observer := driverbridge.NewObserverSink(cfg.Observer) rootCfg := queue.Config{ Driver: queue.DriverRabbitMQ, DefaultQueue: cfg.DefaultQueue, - Observer: cfg.Observer, + Observer: observer, } defaultQueue := queue.PhysicalQueueName(cfg.DefaultQueue, cfg.DefaultQueue) return driverbridge.NewQueueFromDriver(rootCfg, newRabbitMQQueue(cfg.URL, defaultQueue), func(workers int) (any, error) { @@ -65,7 +66,7 @@ func NewWithConfig(cfg Config, opts ...queue.Option) (*queue.Queue, error) { DefaultQueue: defaultQueue, RabbitMQURL: cfg.URL, Workers: workers, - Observer: cfg.Observer, + Observer: observer, }), nil }, opts...) } diff --git a/driver/rabbitmqqueue/worker_rabbitmq_impl.go b/driver/rabbitmqqueue/worker_rabbitmq_impl.go index f484982..9ffe0ac 100644 --- a/driver/rabbitmqqueue/worker_rabbitmq_impl.go +++ b/driver/rabbitmqqueue/worker_rabbitmq_impl.go @@ -3,11 +3,13 @@ package rabbitmqqueue import ( "context" "encoding/json" + "fmt" "strconv" "sync" "time" "github.com/goforj/queue" + "github.com/goforj/queue/busruntime" "github.com/goforj/queue/queuecore" amqp "github.com/rabbitmq/amqp091-go" ) @@ -22,12 +24,14 @@ type rabbitMQWorker struct { started bool cancel context.CancelFunc wg sync.WaitGroup + stopDone chan struct{} conn *amqp.Connection ch *amqp.Channel - pubMu sync.Mutex - observer queue.Observer + pubMu sync.Mutex + observer queue.Observer + publishOverride func(context.Context, rabbitMQMessage) error } type rabbitMQWorkerConfig struct { @@ -84,6 +88,11 @@ func (w *rabbitMQWorker) StartWorkers(ctx context.Context) error { _ = conn.Close() return err } + if err := ch.Confirm(false); err != nil { + _ = ch.Close() + _ = conn.Close() + return err + } if _, err := ch.QueueDeclare(w.cfg.DefaultQueue, true, false, false, false, nil); err != nil { _ = ch.Close() _ = conn.Close() @@ -109,29 +118,52 @@ func (w *rabbitMQWorker) StartWorkers(ctx context.Context) error { return nil } -func (w *rabbitMQWorker) Shutdown(_ context.Context) error { +// Shutdown stops intake and keeps settlement resources open until in-flight deliveries drain or the caller deadline expires. +func (w *rabbitMQWorker) Shutdown(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } w.startStop.Lock() if !w.started { w.startStop.Unlock() return nil } - cancel := w.cancel - w.started = false + if w.stopDone == nil { + w.stopDone = make(chan struct{}) + cancel := w.cancel + ch := w.ch + conn := w.conn + done := w.stopDone + if cancel != nil { + cancel() + } + go func() { + w.wg.Wait() + closeRabbitResources(ch, conn) + w.startStop.Lock() + if w.ch == ch { + w.ch = nil + } + if w.conn == conn { + w.conn = nil + } + w.started = false + w.stopDone = nil + w.startStop.Unlock() + close(done) + }() + } + done := w.stopDone ch := w.ch conn := w.conn w.startStop.Unlock() - - if cancel != nil { - cancel() - } - if ch != nil { - _ = ch.Close() - } - if conn != nil { - _ = conn.Close() + select { + case <-done: + return nil + case <-ctx.Done(): + closeRabbitResources(ch, conn) + return ctx.Err() } - w.wg.Wait() - return nil } func (w *rabbitMQWorker) loop(ctx context.Context, deliveries <-chan amqp.Delivery) { @@ -149,22 +181,23 @@ func (w *rabbitMQWorker) loop(ctx context.Context, deliveries <-chan amqp.Delive } } +// processDelivery commits positive facts only after the original RabbitMQ delivery is acknowledged. func (w *rabbitMQWorker) processDelivery(ctx context.Context, delivery amqp.Delivery) { var incoming rabbitMQMessage if err := json.Unmarshal(delivery.Body, &incoming); err != nil { - _ = delivery.Ack(false) + w.ack(ctx, delivery, incoming) return } if incoming.AvailableAtMS > 0 { remaining := time.Until(time.UnixMilli(incoming.AvailableAtMS)) if remaining > 0 { - if err := w.publish(incoming); err != nil { + if err := w.publish(context.Background(), incoming); err != nil { w.observeRepublishFailure(ctx, incoming, err) - _ = delivery.Nack(false, true) + w.nack(ctx, delivery, incoming, true) return } - _ = delivery.Ack(false) + w.ack(ctx, delivery, incoming) return } incoming.AvailableAtMS = 0 @@ -174,11 +207,13 @@ func (w *rabbitMQWorker) processDelivery(ctx context.Context, delivery amqp.Deli handler, ok := w.handlers[incoming.Type] w.mu.RUnlock() if !ok { - _ = delivery.Ack(false) + w.ack(ctx, delivery, incoming) return } - runCtx := context.Background() + attempt := busruntime.DeliveryAttempt{Number: incoming.Attempt, MaxRetry: incoming.MaxRetry} + runCtx := busruntime.WithDeliveryAttempt(context.Background(), attempt) + runCtx, settlement := busruntime.WithDeliverySettlement(runCtx) if incoming.TimeoutMillis > 0 { var cancel context.CancelFunc runCtx, cancel = context.WithTimeout(runCtx, time.Duration(incoming.TimeoutMillis)*time.Millisecond) @@ -186,50 +221,120 @@ func (w *rabbitMQWorker) processDelivery(ctx context.Context, delivery amqp.Deli } err := handler( runCtx, - queuecore.DriverWithAttempt( - queue.NewJob(incoming.Type). - Payload(incoming.Payload). - OnQueue(incoming.Queue). - Retry(incoming.MaxRetry), - incoming.Attempt, - ), + rabbitMQDeliveryJob(incoming), ) - if err == nil { - _ = delivery.Ack(false) + switch busruntime.ClassifyAttempt(attempt, err) { + case busruntime.AttemptSucceeded, busruntime.AttemptFailed: + if w.ack(runCtx, delivery, incoming) { + settlement.Commit() + } return - } - if incoming.Attempt >= incoming.MaxRetry { - _ = delivery.Ack(false) + case busruntime.AttemptRedeliver: + w.nack(runCtx, delivery, incoming, true) return + case busruntime.AttemptRetry: } + settledMessage := incoming incoming.Attempt++ if incoming.BackoffMillis > 0 { incoming.AvailableAtMS = time.Now().Add(time.Duration(incoming.BackoffMillis) * time.Millisecond).UnixMilli() } else { incoming.AvailableAtMS = 0 } - if err := w.publish(incoming); err != nil { + if err := w.publish(context.Background(), incoming); err != nil { w.observeRepublishFailure(runCtx, incoming, err) - _ = delivery.Nack(false, true) + w.nack(runCtx, delivery, settledMessage, true) return } - _ = delivery.Ack(false) + if w.ack(runCtx, delivery, settledMessage) { + settlement.Commit() + } +} + +// ack reports a failed positive settlement and returns whether the broker accepted the acknowledgement. +func (w *rabbitMQWorker) ack(ctx context.Context, delivery amqp.Delivery, message rabbitMQMessage) bool { + if err := delivery.Ack(false); err != nil { + w.observeSettlementFailure(ctx, message, fmt.Errorf("ack rabbitmq delivery: %w", err)) + return false + } + return true +} + +// nack reports a failed negative settlement because redelivery intent did not reach the broker. +func (w *rabbitMQWorker) nack(ctx context.Context, delivery amqp.Delivery, message rabbitMQMessage, requeue bool) { + if err := delivery.Nack(false, requeue); err != nil { + w.observeSettlementFailure(ctx, message, fmt.Errorf("nack rabbitmq delivery: %w", err)) + } } func (w *rabbitMQWorker) observeRepublishFailure(ctx context.Context, message rabbitMQMessage, err error) { + metadata := queue.ResolveObservedJobMetadataFromJob(rabbitMQDeliveryJob(message)) queuecore.SafeObserve(ctx, w.observer, queue.Event{ - Kind: queue.EventRepublishFailed, - Driver: queue.DriverRabbitMQ, - Queue: queuecore.NormalizeQueueName(message.Queue), - JobType: queue.ResolveObservedJobType(message.Type, message.Payload), - Attempt: message.Attempt, - MaxRetry: message.MaxRetry, - Err: err, - Time: time.Now(), + Kind: queue.EventRepublishFailed, + Driver: queue.DriverRabbitMQ, + Queue: queuecore.NormalizeQueueName(message.Queue), + JobType: metadata.JobType, + JobKey: metadata.JobKey, + DispatchID: metadata.DispatchID, + JobID: metadata.JobID, + ChainID: metadata.ChainID, + BatchID: metadata.BatchID, + Attempt: message.Attempt, + MaxRetry: message.MaxRetry, + Err: err, + Time: time.Now(), }) } -func (w *rabbitMQWorker) publish(message rabbitMQMessage) error { +// observeSettlementFailure emits the canonical worker fact for an uncommitted RabbitMQ acknowledgement. +func (w *rabbitMQWorker) observeSettlementFailure(ctx context.Context, message rabbitMQMessage, err error) { + metadata := queue.ResolveObservedJobMetadataFromJob(rabbitMQDeliveryJob(message)) + queuecore.SafeObserve(ctx, w.observer, queue.Event{ + Kind: queue.EventSettlementFailed, + Driver: queue.DriverRabbitMQ, + Queue: queuecore.NormalizeQueueName(message.Queue), + JobType: metadata.JobType, + JobKey: metadata.JobKey, + DispatchID: metadata.DispatchID, + JobID: metadata.JobID, + ChainID: metadata.ChainID, + BatchID: metadata.BatchID, + Attempt: message.Attempt, + MaxRetry: message.MaxRetry, + Err: err, + Time: time.Now(), + }) +} + +// rabbitMQDeliveryJob reconstructs one RabbitMQ delivery while retaining +// supported direct-delivery metadata outside the application payload. +func rabbitMQDeliveryJob(message rabbitMQMessage) queue.Job { + job := queuecore.DriverWithAttempt( + queue.NewJob(message.Type). + Payload(message.Payload). + OnQueue(message.Queue). + Retry(message.MaxRetry), + message.Attempt, + ) + if len(message.Metadata) > 0 { + var metadata queue.DriverJobMetadata + if err := json.Unmarshal(message.Metadata, &metadata); err == nil { + job = queue.DriverWithMetadata(job, metadata) + } + } + return job +} + +// publish declares the destination and waits for broker confirmation before reporting success. +func (w *rabbitMQWorker) publish(ctx context.Context, message rabbitMQMessage) error { + settlementCtx, cancel, err := rabbitPublishContext(ctx) + if err != nil { + return err + } + defer cancel() + if w.publishOverride != nil { + return w.publishOverride(settlementCtx, message) + } w.startStop.Lock() ch := w.ch w.startStop.Unlock() @@ -255,7 +360,7 @@ func (w *rabbitMQWorker) publish(message rabbitMQMessage) error { return err } if delay <= 0 { - return ch.PublishWithContext(context.Background(), "", queueName, false, false, amqp.Publishing{ + return publishRabbitConfirmed(settlementCtx, ch, "", queueName, amqp.Publishing{ ContentType: "application/json", Body: body, DeliveryMode: amqp.Persistent, @@ -274,7 +379,7 @@ func (w *rabbitMQWorker) publish(message rabbitMQMessage) error { if _, err := ch.QueueDeclare(delayQueue, true, false, false, false, args); err != nil { return err } - return ch.PublishWithContext(context.Background(), "", delayQueue, false, false, amqp.Publishing{ + return publishRabbitConfirmed(settlementCtx, ch, "", delayQueue, amqp.Publishing{ ContentType: "application/json", Body: body, Expiration: strconv.FormatInt(delayMS, 10), @@ -288,3 +393,13 @@ func defaultWorkerCount(n int) int { } return n } + +// closeRabbitResources closes settlement resources only after a graceful drain or an expired shutdown deadline. +func closeRabbitResources(ch *amqp.Channel, conn *amqp.Connection) { + if ch != nil { + _ = ch.Close() + } + if conn != nil { + _ = conn.Close() + } +} diff --git a/driver/rabbitmqqueue/worker_rabbitmq_impl_test.go b/driver/rabbitmqqueue/worker_rabbitmq_impl_test.go index fa54d7d..f89e231 100644 --- a/driver/rabbitmqqueue/worker_rabbitmq_impl_test.go +++ b/driver/rabbitmqqueue/worker_rabbitmq_impl_test.go @@ -8,23 +8,136 @@ import ( "time" "github.com/goforj/queue" + "github.com/goforj/queue/busruntime" "github.com/goforj/queue/queuecore" amqp "github.com/rabbitmq/amqp091-go" ) type ackRecorder struct { - acks int - nacks int + acks int + nacks int + nackRequeue bool + ackErr error + nackErr error +} + +type rabbitConfirmationStub struct { + acked bool + err error +} + +type rabbitContextConfirmationStub struct{} + +// WaitContext returns the configured broker confirmation. +func (s rabbitConfirmationStub) WaitContext(context.Context) (bool, error) { + return s.acked, s.err +} + +// WaitContext exposes cancellation from the caller's publish boundary. +func (rabbitContextConfirmationStub) WaitContext(ctx context.Context) (bool, error) { + <-ctx.Done() + return false, ctx.Err() } func (a *ackRecorder) Ack(_ uint64, _ bool) error { a.acks++ - return nil + return a.ackErr } -func (a *ackRecorder) Nack(_ uint64, _ bool, _ bool) error { +// Nack records whether the worker requested broker redelivery. +func (a *ackRecorder) Nack(_ uint64, _ bool, requeue bool) error { a.nacks++ - return nil + a.nackRequeue = requeue + return a.nackErr +} + +// TestRabbitMQWorkerSettlementFailuresAreObserved verifies Ack and Nack errors become correlated worker facts. +func TestRabbitMQWorkerSettlementFailuresAreObserved(t *testing.T) { + tests := []struct { + name string + handlerErr error + maxRetry int + acks *ackRecorder + }{ + {name: "ack", maxRetry: 0, acks: &ackRecorder{ackErr: errors.New("ack failed")}}, + {name: "nack", handlerErr: busruntime.Uncommitted(errors.New("store failed")), maxRetry: 2, acks: &ackRecorder{nackErr: errors.New("nack failed")}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var events []queue.Event + committed := false + var handlerSettlement busruntime.DeliverySettlementIdentity + var handlerSettlementOK bool + var observedSettlement busruntime.DeliverySettlementIdentity + var observedSettlementOK bool + w := &rabbitMQWorker{ + handlers: map[string]queue.Handler{"bus:job": func(ctx context.Context, _ queue.Job) error { + handlerSettlement, handlerSettlementOK = busruntime.DeliverySettlementIdentityFromContext(ctx) + if !busruntime.DeferUntilDeliveryCommitted(ctx, func() { committed = true }) { + t.Fatal("handler context did not carry a settlement boundary") + } + return test.handlerErr + }}, + observer: queue.ObserverFunc(func(ctx context.Context, event queue.Event) { + observedSettlement, observedSettlementOK = busruntime.DeliverySettlementIdentityFromContext(ctx) + events = append(events, event) + }), + } + payload := []byte(`{"schema_version":1,"dispatch_id":"dsp_rabbit_settle","job_id":"job_rabbit_settle","job":{"type":"reports:build","payload":"eyJpZCI6MX0="}}`) + body, err := json.Marshal(rabbitMQMessage{Type: "bus:job", Queue: "critical", Payload: payload, MaxRetry: test.maxRetry}) + if err != nil { + t.Fatalf("marshal body: %v", err) + } + w.processDelivery(context.Background(), amqp.Delivery{Body: body, Acknowledger: test.acks, DeliveryTag: 70}) + if len(events) != 1 || events[0].Kind != queue.EventSettlementFailed { + t.Fatalf("settlement events = %+v, want one failure", events) + } + if events[0].Layer != queue.EventLayerWorker || events[0].JobType != "reports:build" || events[0].DispatchID != "dsp_rabbit_settle" { + t.Fatalf("settlement correlation = %+v", events[0]) + } + if committed { + t.Fatal("failed acknowledgement committed deferred handler outcome") + } + if !handlerSettlementOK || !observedSettlementOK || observedSettlement != handlerSettlement { + t.Fatal("settlement observer did not retain the handler's delivery identity") + } + }) + } +} + +// TestRabbitMQWorkerRetrySettlementFailureUsesDeliveredAttempt verifies replacement metadata does not overwrite the unsettled delivery's correlation. +func TestRabbitMQWorkerRetrySettlementFailureUsesDeliveredAttempt(t *testing.T) { + acks := &ackRecorder{ackErr: errors.New("ack failed")} + var events []queue.Event + var handlerSettlement busruntime.DeliverySettlementIdentity + var handlerSettlementOK bool + var observedSettlement busruntime.DeliverySettlementIdentity + var observedSettlementOK bool + w := &rabbitMQWorker{ + handlers: map[string]queue.Handler{"job:retry:settlement": func(ctx context.Context, _ queue.Job) error { + handlerSettlement, handlerSettlementOK = busruntime.DeliverySettlementIdentityFromContext(ctx) + return errors.New("retry me") + }}, + cfg: rabbitMQWorkerConfig{DefaultQueue: "default"}, + observer: queue.ObserverFunc(func(ctx context.Context, event queue.Event) { + observedSettlement, observedSettlementOK = busruntime.DeliverySettlementIdentityFromContext(ctx) + events = append(events, event) + }), + publishOverride: func(context.Context, rabbitMQMessage) error { + return nil + }, + } + body, err := json.Marshal(rabbitMQMessage{Type: "job:retry:settlement", Queue: "critical", Attempt: 1, MaxRetry: 3}) + if err != nil { + t.Fatalf("marshal body: %v", err) + } + w.processDelivery(context.Background(), amqp.Delivery{Body: body, Acknowledger: acks, DeliveryTag: 71}) + if len(events) != 1 || events[0].Kind != queue.EventSettlementFailed || events[0].Attempt != 1 { + t.Fatalf("settlement events = %+v, want original attempt 1", events) + } + if !handlerSettlementOK || !observedSettlementOK || observedSettlement != handlerSettlement { + t.Fatal("retry settlement observer did not retain the handler's delivery identity") + } } func (a *ackRecorder) Reject(_ uint64, _ bool) error { return nil } @@ -75,6 +188,34 @@ func TestRabbitMQWorker_StartWorkersNilContextDialFailure(t *testing.T) { } } +// TestRabbitMQWorkerShutdownHonorsDeadline verifies a stuck in-flight handler cannot block the caller forever. +func TestRabbitMQWorkerShutdownHonorsDeadline(t *testing.T) { + w := newRabbitMQWorker(rabbitMQWorkerConfig{}) + w.started = true + w.cancel = func() {} + release := make(chan struct{}) + w.wg.Add(1) + go func() { + defer w.wg.Done() + <-release + }() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Millisecond) + defer cancel() + if err := w.Shutdown(ctx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("shutdown error = %v, want deadline exceeded", err) + } + if !w.started { + t.Fatal("timed-out shutdown exposed the worker as restartable while work remained") + } + close(release) + if err := w.Shutdown(context.Background()); err != nil { + t.Fatalf("complete shutdown: %v", err) + } + if w.started { + t.Fatal("completed shutdown retained started state") + } +} + func TestRabbitMQWorker_ProcessDeliveryBranches(t *testing.T) { t.Run("invalid json ack", func(t *testing.T) { acks := &ackRecorder{} @@ -101,6 +242,7 @@ func TestRabbitMQWorker_ProcessDeliveryBranches(t *testing.T) { t.Run("success handler ack", func(t *testing.T) { acks := &ackRecorder{} called := 0 + committed := false w := &rabbitMQWorker{handlers: map[string]queue.Handler{ "job:ok": func(ctx context.Context, job queue.Job) error { called++ @@ -111,6 +253,9 @@ func TestRabbitMQWorker_ProcessDeliveryBranches(t *testing.T) { if job.Type != "job:ok" || opts.QueueName != "critical" || opts.Attempt != 1 { t.Fatalf("unexpected job fields: type=%q queue=%q attempt=%d", job.Type, opts.QueueName, opts.Attempt) } + if !busruntime.DeferUntilDeliveryCommitted(ctx, func() { committed = true }) { + t.Fatal("handler context did not carry a settlement boundary") + } return nil }, }} @@ -122,6 +267,9 @@ func TestRabbitMQWorker_ProcessDeliveryBranches(t *testing.T) { if called != 1 || acks.acks != 1 { t.Fatalf("expected handler once and ack once, got called=%d ack=%d", called, acks.acks) } + if !committed { + t.Fatal("successful acknowledgement did not commit deferred handler success") + } }) t.Run("future delivery publish path with nil channel nacks", func(t *testing.T) { @@ -130,7 +278,7 @@ func TestRabbitMQWorker_ProcessDeliveryBranches(t *testing.T) { w := &rabbitMQWorker{ handlers: map[string]queue.Handler{}, cfg: rabbitMQWorkerConfig{DefaultQueue: "default"}, - observer: queue.ObserverFunc(func(e queue.Event) { events = append(events, e) }), + observer: queue.ObserverFunc(func(_ context.Context, e queue.Event) { events = append(events, e) }), } body, err := json.Marshal(rabbitMQMessage{Type: "job:future", Queue: "default", AvailableAtMS: time.Now().Add(2 * time.Second).UnixMilli()}) if err != nil { @@ -143,6 +291,9 @@ func TestRabbitMQWorker_ProcessDeliveryBranches(t *testing.T) { if len(events) == 0 || events[0].Kind != queue.EventRepublishFailed || events[0].Driver != queue.DriverRabbitMQ { t.Fatalf("expected republish_failed rabbitmq event, got %+v", events) } + if events[0].Layer != queue.EventLayerWorker { + t.Fatalf("republish_failed layer = %q, want worker", events[0].Layer) + } }) t.Run("republish failure unwraps bus envelope job type", func(t *testing.T) { @@ -151,13 +302,13 @@ func TestRabbitMQWorker_ProcessDeliveryBranches(t *testing.T) { w := &rabbitMQWorker{ handlers: map[string]queue.Handler{}, cfg: rabbitMQWorkerConfig{DefaultQueue: "default"}, - observer: queue.ObserverFunc(func(e queue.Event) { events = append(events, e) }), + observer: queue.ObserverFunc(func(_ context.Context, e queue.Event) { events = append(events, e) }), } body, err := json.Marshal(rabbitMQMessage{ - Type: "job:future", + Type: "bus:job", Queue: "default", AvailableAtMS: time.Now().Add(2 * time.Second).UnixMilli(), - Payload: []byte(`{"job":{"type":"monitoring:check"}}`), + Payload: []byte(`{"schema_version":1,"dispatch_id":"dsp_rabbit","job_id":"job_rabbit","batch_id":"bat_rabbit","job":{"type":"monitoring:check"}}`), }) if err != nil { t.Fatalf("marshal: %v", err) @@ -166,8 +317,11 @@ func TestRabbitMQWorker_ProcessDeliveryBranches(t *testing.T) { if len(events) == 0 { t.Fatal("expected republish failure event") } - if events[0].JobType != "job:future" { - t.Fatalf("expected non-bus job type to pass through, got %q", events[0].JobType) + if events[0].JobType != "monitoring:check" { + t.Fatalf("expected unwrapped observed job type, got %q", events[0].JobType) + } + if events[0].DispatchID != "dsp_rabbit" || events[0].JobID != "job_rabbit" || events[0].BatchID != "bat_rabbit" { + t.Fatalf("expected correlated rabbitmq event, got %+v", events[0]) } }) @@ -189,6 +343,153 @@ func TestRabbitMQWorker_ProcessDeliveryBranches(t *testing.T) { } }) + t.Run("republish and nack failures retain their physical attempts", func(t *testing.T) { + acks := &ackRecorder{nackErr: errors.New("nack failed")} + var events []queue.Event + w := &rabbitMQWorker{ + handlers: map[string]queue.Handler{ + "job:retry:failed-settlement": func(context.Context, queue.Job) error { return errors.New("retry") }, + }, + cfg: rabbitMQWorkerConfig{DefaultQueue: "default"}, + observer: queue.ObserverFunc(func(_ context.Context, event queue.Event) { + events = append(events, event) + }), + publishOverride: func(context.Context, rabbitMQMessage) error { + return errors.New("publish failed") + }, + } + body, err := json.Marshal(rabbitMQMessage{ + Type: "job:retry:failed-settlement", + Queue: "critical", + Attempt: 2, + MaxRetry: 4, + }) + if err != nil { + t.Fatalf("marshal: %v", err) + } + w.processDelivery(context.Background(), amqp.Delivery{Body: body, Acknowledger: acks, DeliveryTag: 55}) + if acks.acks != 0 || acks.nacks != 1 { + t.Fatalf("publish failure ack/nack = %d/%d, want 0/1", acks.acks, acks.nacks) + } + if len(events) != 2 { + t.Fatalf("failure events = %+v, want republish and settlement failures", events) + } + if events[0].Kind != queue.EventRepublishFailed || events[0].Attempt != 3 { + t.Fatalf("republish failure = %+v, want replacement attempt 3", events[0]) + } + if events[1].Kind != queue.EventSettlementFailed || events[1].Attempt != 2 { + t.Fatalf("settlement failure = %+v, want original receipt attempt 2", events[1]) + } + }) + + t.Run("failed handler acks only after confirmed replacement", func(t *testing.T) { + acks := &ackRecorder{} + published := 0 + w := &rabbitMQWorker{ + handlers: map[string]queue.Handler{ + "job:retry:confirmed": func(context.Context, queue.Job) error { return errors.New("boom") }, + }, + cfg: rabbitMQWorkerConfig{DefaultQueue: "default"}, + publishOverride: func(ctx context.Context, message rabbitMQMessage) error { + published++ + if ctx.Err() != nil { + t.Fatalf("replacement publish context is already canceled: %v", ctx.Err()) + } + if _, ok := ctx.Deadline(); !ok { + t.Fatal("replacement publish context has no settlement deadline") + } + if acks.acks != 0 { + t.Fatal("original delivery was acknowledged before replacement confirmation") + } + if message.Attempt != 1 { + t.Fatalf("replacement attempt = %d, want 1", message.Attempt) + } + return nil + }, + } + body, err := json.Marshal(rabbitMQMessage{Type: "job:retry:confirmed", Queue: "default", MaxRetry: 2}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + w.processDelivery(context.Background(), amqp.Delivery{Body: body, Acknowledger: acks, DeliveryTag: 55}) + if published != 1 || acks.acks != 1 || acks.nacks != 0 { + t.Fatalf("publish/ack/nack = %d/%d/%d, want 1/1/0", published, acks.acks, acks.nacks) + } + }) + + t.Run("expired handler timeout does not cancel replacement settlement", func(t *testing.T) { + acks := &ackRecorder{} + published := 0 + w := &rabbitMQWorker{ + handlers: map[string]queue.Handler{ + "job:retry:timeout": func(ctx context.Context, _ queue.Job) error { + <-ctx.Done() + return ctx.Err() + }, + }, + cfg: rabbitMQWorkerConfig{DefaultQueue: "default"}, + publishOverride: func(ctx context.Context, message rabbitMQMessage) error { + published++ + if ctx.Err() != nil { + t.Fatalf("expired handler context leaked into settlement: %v", ctx.Err()) + } + if _, ok := ctx.Deadline(); !ok { + t.Fatal("replacement settlement context has no deadline") + } + if message.Attempt != 1 { + t.Fatalf("replacement attempt = %d, want 1", message.Attempt) + } + return nil + }, + } + body, err := json.Marshal(rabbitMQMessage{ + Type: "job:retry:timeout", + Queue: "default", + MaxRetry: 1, + TimeoutMillis: 1, + }) + if err != nil { + t.Fatalf("marshal: %v", err) + } + w.processDelivery(context.Background(), amqp.Delivery{Body: body, Acknowledger: acks, DeliveryTag: 56}) + if published != 1 || acks.acks != 1 || acks.nacks != 0 { + t.Fatalf("publish/ack/nack = %d/%d/%d, want 1/1/0", published, acks.acks, acks.nacks) + } + }) + + t.Run("canceled delivery context does not cancel delayed replacement settlement", func(t *testing.T) { + acks := &ackRecorder{} + published := 0 + w := &rabbitMQWorker{ + handlers: map[string]queue.Handler{}, + cfg: rabbitMQWorkerConfig{DefaultQueue: "default"}, + publishOverride: func(ctx context.Context, _ rabbitMQMessage) error { + published++ + if ctx.Err() != nil { + t.Fatalf("delivery cancellation leaked into delayed settlement: %v", ctx.Err()) + } + if _, ok := ctx.Deadline(); !ok { + t.Fatal("delayed settlement context has no deadline") + } + return nil + }, + } + body, err := json.Marshal(rabbitMQMessage{ + Type: "job:future:canceled", + Queue: "default", + AvailableAtMS: time.Now().Add(time.Second).UnixMilli(), + }) + if err != nil { + t.Fatalf("marshal: %v", err) + } + deliveryCtx, cancel := context.WithCancel(context.Background()) + cancel() + w.processDelivery(deliveryCtx, amqp.Delivery{Body: body, Acknowledger: acks, DeliveryTag: 57}) + if published != 1 || acks.acks != 1 || acks.nacks != 0 { + t.Fatalf("publish/ack/nack = %d/%d/%d, want 1/1/0", published, acks.acks, acks.nacks) + } + }) + t.Run("failed handler terminal acks", func(t *testing.T) { acks := &ackRecorder{} w := &rabbitMQWorker{ @@ -208,12 +509,83 @@ func TestRabbitMQWorker_ProcessDeliveryBranches(t *testing.T) { }) } +// TestRabbitMQWorker_AttemptDecisionSettlement verifies terminal and uncommitted outcomes choose acknowledgement behavior without consuming retries. +func TestRabbitMQWorker_AttemptDecisionSettlement(t *testing.T) { + t.Run("permanent failure acks without republishing", func(t *testing.T) { + acks := &ackRecorder{} + var events []queue.Event + w := &rabbitMQWorker{ + handlers: map[string]queue.Handler{ + "job:permanent": func(ctx context.Context, _ queue.Job) error { + attempt, ok := busruntime.DeliveryAttemptFromContext(ctx) + if !ok || attempt.Number != 0 || attempt.MaxRetry != 3 { + t.Fatalf("unexpected delivery attempt: %+v, present=%t", attempt, ok) + } + return busruntime.Permanent(errors.New("invalid job")) + }, + }, + cfg: rabbitMQWorkerConfig{DefaultQueue: "default"}, + observer: queue.ObserverFunc(func(_ context.Context, event queue.Event) { events = append(events, event) }), + } + body, err := json.Marshal(rabbitMQMessage{Type: "job:permanent", Queue: "default", MaxRetry: 3}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + w.processDelivery(context.Background(), amqp.Delivery{Body: body, Acknowledger: acks, DeliveryTag: 7}) + + if acks.acks != 1 || acks.nacks != 0 { + t.Fatalf("permanent failure must ack once, got ack=%d nack=%d", acks.acks, acks.nacks) + } + if len(events) != 0 { + t.Fatalf("permanent failure must not reach the republish path, got %+v", events) + } + }) + + t.Run("uncommitted failure nacks with requeue", func(t *testing.T) { + acks := &ackRecorder{} + var events []queue.Event + w := &rabbitMQWorker{ + handlers: map[string]queue.Handler{ + "job:uncommitted": func(ctx context.Context, _ queue.Job) error { + attempt, ok := busruntime.DeliveryAttemptFromContext(ctx) + if !ok || attempt.Number != 1 || attempt.MaxRetry != 4 { + t.Fatalf("unexpected delivery attempt: %+v, present=%t", attempt, ok) + } + return busruntime.Uncommitted(errors.New("store unavailable")) + }, + }, + cfg: rabbitMQWorkerConfig{DefaultQueue: "default"}, + observer: queue.ObserverFunc(func(_ context.Context, event queue.Event) { events = append(events, event) }), + } + body, err := json.Marshal(rabbitMQMessage{ + Type: "job:uncommitted", + Queue: "default", + Attempt: 1, + MaxRetry: 4, + BackoffMillis: 1_000, + }) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + w.processDelivery(context.Background(), amqp.Delivery{Body: body, Acknowledger: acks, DeliveryTag: 8}) + + if acks.acks != 0 || acks.nacks != 1 || !acks.nackRequeue { + t.Fatalf("uncommitted failure must nack with requeue, got ack=%d nack=%d requeue=%t", acks.acks, acks.nacks, acks.nackRequeue) + } + if len(events) != 0 { + t.Fatalf("uncommitted failure must not publish a replacement, got %+v", events) + } + }) +} + func TestRabbitMQWorker_PublishNilChannelAndImmediateDelay(t *testing.T) { w := &rabbitMQWorker{cfg: rabbitMQWorkerConfig{DefaultQueue: ""}} - if err := w.publish(rabbitMQMessage{Type: "job:nilch", Queue: "default"}); !errors.Is(err, amqp.ErrClosed) { + if err := w.publish(context.Background(), rabbitMQMessage{Type: "job:nilch", Queue: "default"}); !errors.Is(err, amqp.ErrClosed) { t.Fatalf("publish with nil channel should return amqp.ErrClosed, got %v", err) } - if err := w.publish(rabbitMQMessage{ + if err := w.publish(context.Background(), rabbitMQMessage{ Type: "job:past", Queue: "default", AvailableAtMS: time.Now().Add(-10 * time.Millisecond).UnixMilli(), @@ -221,3 +593,96 @@ func TestRabbitMQWorker_PublishNilChannelAndImmediateDelay(t *testing.T) { t.Fatalf("publish past delay with nil channel should return amqp.ErrClosed, got %v", err) } } + +// TestAwaitRabbitConfirmation verifies only a positive confirmation commits a publish. +func TestAwaitRabbitConfirmation(t *testing.T) { + cause := errors.New("confirmation channel closed") + canceledCtx, cancel := context.WithCancel(context.Background()) + cancel() + tests := []struct { + name string + ctx context.Context + confirmation rabbitPublishConfirmation + wantErr bool + }{ + {name: "missing", wantErr: true}, + {name: "nack", confirmation: rabbitConfirmationStub{}, wantErr: true}, + {name: "wait error", confirmation: rabbitConfirmationStub{err: cause}, wantErr: true}, + {name: "context canceled", ctx: canceledCtx, confirmation: rabbitContextConfirmationStub{}, wantErr: true}, + {name: "ack", confirmation: rabbitConfirmationStub{acked: true}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx := test.ctx + if ctx == nil { + ctx = context.Background() + } + err := awaitRabbitConfirmation(ctx, test.confirmation) + if (err != nil) != test.wantErr { + t.Fatalf("awaitRabbitConfirmation() error = %v, wantErr %t", err, test.wantErr) + } + }) + } +} + +// TestRabbitPublishContextBoundsBackground verifies producer confirmation cannot wait forever without a caller deadline. +func TestRabbitPublishContextBoundsBackground(t *testing.T) { + ctx, cancel, err := rabbitPublishContext(context.Background()) + if err != nil { + t.Fatalf("rabbit publish context: %v", err) + } + defer cancel() + deadline, ok := ctx.Deadline() + if !ok { + t.Fatal("bounded publish context has no deadline") + } + remaining := time.Until(deadline) + if remaining <= 0 || remaining > rabbitPublishConfirmationTimeout { + t.Fatalf("publish confirmation deadline remaining = %v", remaining) + } + + canceled, cancelCanceled := context.WithCancel(context.Background()) + cancelCanceled() + if err := publishRabbitConfirmed(canceled, nil, "", "default", amqp.Publishing{}); !errors.Is(err, context.Canceled) { + t.Fatalf("pre-canceled publish error = %v, want context.Canceled", err) + } +} + +// TestRabbitPublishAmbiguityClassification verifies lost confirmations cannot be treated as safe pre-publish rejection. +func TestRabbitPublishAmbiguityClassification(t *testing.T) { + waitErr := errors.New("confirmation lost") + err := awaitRabbitConfirmation(context.Background(), rabbitConfirmationStub{err: waitErr}) + if !isRabbitPublishAmbiguous(err) || !errors.Is(err, waitErr) { + t.Fatalf("confirmation error = %v, want ambiguous wrapped cause", err) + } + publishErr := errors.New("publish response lost") + err = completeRabbitPublish(context.Background(), nil, publishErr) + if !isRabbitPublishAmbiguous(err) || !errors.Is(err, publishErr) { + t.Fatalf("publish error = %v, want ambiguous wrapped cause", err) + } + err = completeRabbitPublish(context.Background(), nil, nil) + if !isRabbitPublishAmbiguous(err) { + t.Fatalf("missing deferred confirmation error = %v, want ambiguous", err) + } + if err := completeRabbitPublish(context.Background(), rabbitConfirmationStub{acked: true}, nil); err != nil { + t.Fatalf("completed publish: %v", err) + } + if isRabbitPublishAmbiguous(errors.New("dial rejected")) { + t.Fatal("pre-publish failure classified as ambiguous") + } +} + +// TestRabbitMQWorkerNilShutdownAndCanceledPublish verifies lifecycle context +// normalization and pre-publish cancellation without opening broker resources. +func TestRabbitMQWorkerNilShutdownAndCanceledPublish(t *testing.T) { + w := &rabbitMQWorker{} + if err := w.Shutdown(nil); err != nil { + t.Fatalf("nil-context shutdown: %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := w.publish(ctx, rabbitMQMessage{Type: "job:canceled", Queue: "default"}); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled worker publish error = %v, want context.Canceled", err) + } +} diff --git a/driver/redisqueue/delivery_metadata.go b/driver/redisqueue/delivery_metadata.go new file mode 100644 index 0000000..dca83ae --- /dev/null +++ b/driver/redisqueue/delivery_metadata.go @@ -0,0 +1,22 @@ +package redisqueue + +import ( + "encoding/json" + + "github.com/goforj/queue" +) + +const redisDriverJobMetadataHeader = "goforj-queue-driver-job-metadata" + +// redisJobWithDriverMetadata restores supported correlation without making malformed or future metadata a delivery failure. +func redisJobWithDriverMetadata(job queue.Job, headers map[string]string) queue.Job { + raw, ok := headers[redisDriverJobMetadataHeader] + if !ok { + return job + } + var metadata queue.DriverJobMetadata + if err := json.Unmarshal([]byte(raw), &metadata); err != nil { + return job + } + return queue.DriverWithMetadata(job, metadata) +} diff --git a/driver/redisqueue/delivery_metadata_test.go b/driver/redisqueue/delivery_metadata_test.go new file mode 100644 index 0000000..1abc906 --- /dev/null +++ b/driver/redisqueue/delivery_metadata_test.go @@ -0,0 +1,235 @@ +package redisqueue + +import ( + "bytes" + "context" + "testing" + + "github.com/goforj/queue" + "github.com/goforj/queue/internal/driverbridge" + backend "github.com/hibiken/asynq" +) + +// TestRedisQueueDispatchCarriesDriverMetadataWithoutRetry pins the additive Asynq header wire independently of retry policy. +func TestRedisQueueDispatchCarriesDriverMetadataWithoutRetry(t *testing.T) { + client := &redisEnqueueClientStub{} + driver := &redisQueue{client: client} + payload := []byte{0x00, 0xff, 0x7f} + metadata := queue.DriverJobMetadata{ + SchemaVersion: queue.DriverJobMetadataVersion, + DispatchID: "dsp_redis_direct", + JobID: "job_redis_direct", + ChainID: "chn_redis_direct", + BatchID: "bat_redis_direct", + Queue: "critical", + } + job := queue.DriverWithMetadata( + queue.NewJob("reports:build").Payload(payload).OnQueue("critical"), + metadata, + ) + + if err := driver.Dispatch(context.Background(), job); err != nil { + t.Fatalf("dispatch direct job: %v", err) + } + if client.task == nil { + t.Fatal("dispatch did not enqueue an Asynq task") + } + if client.task.Type() != job.Type || !bytes.Equal(client.task.Payload(), payload) { + t.Fatalf("physical task = type:%q payload:%v, want type:%q payload:%v", client.task.Type(), client.task.Payload(), job.Type, payload) + } + headers := client.task.Headers() + if len(headers) != 1 { + t.Fatalf("headers = %#v, want only direct-delivery metadata", headers) + } + const wantMetadata = `{"schema_version":1,"dispatch_id":"dsp_redis_direct","job_id":"job_redis_direct","chain_id":"chn_redis_direct","batch_id":"bat_redis_direct","queue":"critical"}` + if got := headers[redisDriverJobMetadataHeader]; got != wantMetadata { + t.Fatalf("driver metadata header = %q, want %q", got, wantMetadata) + } + if _, ok := headers[redisApplicationMaxRetryHeader]; ok { + t.Fatalf("job without Retry gained application retry header: %#v", headers) + } +} + +// TestRedisQueueDispatchKeepsMetadataAlongsideRetryReserve verifies both private headers survive one task representation. +func TestRedisQueueDispatchKeepsMetadataAlongsideRetryReserve(t *testing.T) { + client := &redisEnqueueClientStub{} + driver := &redisQueue{client: client} + metadata := queue.DriverJobMetadata{ + SchemaVersion: queue.DriverJobMetadataVersion, + DispatchID: "dsp_retry", + JobID: "job_retry", + Queue: "critical", + } + job := queue.DriverWithMetadata( + queue.NewJob("reports:retry").Payload([]byte("payload")).OnQueue("critical").Retry(2), + metadata, + ) + + if err := driver.Dispatch(context.Background(), job); err != nil { + t.Fatalf("dispatch retrying direct job: %v", err) + } + headers := client.task.Headers() + if len(headers) != 2 { + t.Fatalf("headers = %#v, want metadata and retry reserve", headers) + } + if got := headers[redisApplicationMaxRetryHeader]; got != "2" { + t.Fatalf("application retry header = %q, want 2", got) + } + reconstructed := redisJobWithDriverMetadata(queue.NewJob(job.Type).Payload(job.PayloadBytes()), headers) + if got := queue.DriverMetadata(reconstructed); got != metadata { + t.Fatalf("reconstructed metadata = %+v, want %+v", got, metadata) + } + if got := headers[redisApplicationMaxRetryHeader]; got != "2" { + t.Fatalf("metadata reconstruction changed retry header to %q", got) + } +} + +// TestRedisWorkerReconstructsDriverMetadataForHandlerAndObserver proves the native worker restores correlation before either consumer path. +func TestRedisWorkerReconstructsDriverMetadataForHandlerAndObserver(t *testing.T) { + server := &serverStub{} + var events []queue.Event + observer := queue.ObserverFunc(func(_ context.Context, event queue.Event) { + events = append(events, event) + }) + worker := newRedisWorker(server, backend.NewServeMux(), observer) + wantMetadata := queue.DriverJobMetadata{ + SchemaVersion: queue.DriverJobMetadataVersion, + DispatchID: "dsp_handler", + JobID: "job_handler", + ChainID: "chn_handler", + BatchID: "bat_handler", + Queue: "critical", + } + payload := []byte(`{"report_id":17}`) + var handled bool + worker.Register("reports:direct", func(_ context.Context, job queue.Job) error { + handled = true + if job.Type != "reports:direct" || !bytes.Equal(job.PayloadBytes(), payload) { + t.Fatalf("handler job = type:%q payload:%q", job.Type, job.PayloadBytes()) + } + if got := queue.DriverMetadata(job); got != wantMetadata { + t.Fatalf("handler metadata = %+v, want %+v", got, wantMetadata) + } + return nil + }) + if err := worker.StartWorkers(context.Background()); err != nil { + t.Fatalf("start worker: %v", err) + } + task := backend.NewTaskWithHeaders("reports:direct", payload, map[string]string{ + redisDriverJobMetadataHeader: `{"schema_version":1,"dispatch_id":"dsp_handler","job_id":"job_handler","chain_id":"chn_handler","batch_id":"bat_handler","queue":"critical"}`, + }) + if err := server.lastStartHandler.ProcessTask(context.Background(), task); err != nil { + t.Fatalf("process direct task: %v", err) + } + if !handled { + t.Fatal("direct handler was not called") + } + if len(events) != 2 { + t.Fatalf("events = %+v, want start and success", events) + } + for _, event := range events { + if event.JobType != "reports:direct" || event.DispatchID != wantMetadata.DispatchID || event.JobID != wantMetadata.JobID || event.ChainID != wantMetadata.ChainID || event.BatchID != wantMetadata.BatchID { + t.Fatalf("observer did not prefer direct metadata: %+v", event) + } + } +} + +// TestRedisDriverMetadataReachesMessageHandler exercises the complete root Message path through an Asynq task reconstructed as a Job. +func TestRedisDriverMetadataReachesMessageHandler(t *testing.T) { + client := &redisEnqueueClientStub{} + producer := &redisQueue{client: client} + server := &serverStub{} + q, err := driverbridge.NewQueueFromDriver( + queue.Config{Driver: queue.DriverRedis, DefaultQueue: "default"}, + producer, + func(int) (any, error) { + return newRedisWorker(server, backend.NewServeMux(), nil), nil + }, + ) + if err != nil { + t.Fatalf("construct queue: %v", err) + } + var message queue.Message + q.Register("reports:message", func(_ context.Context, delivered queue.Message) error { + message = delivered + return nil + }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start queue workers: %v", err) + } + t.Cleanup(func() { + if err := q.Shutdown(context.Background()); err != nil { + t.Errorf("shutdown queue: %v", err) + } + }) + + result, err := q.Dispatch(queue.NewJob("reports:message").Payload([]byte(`{"report_id":23}`)).OnQueue("critical")) + if err != nil { + t.Fatalf("dispatch message: %v", err) + } + if client.task == nil { + t.Fatal("dispatch did not produce an Asynq task") + } + if err := server.lastStartHandler.ProcessTask(context.Background(), client.task); err != nil { + t.Fatalf("process reconstructed task: %v", err) + } + if message.SchemaVersion == 0 || message.DispatchID != result.DispatchID || message.JobID == "" || message.JobType != "reports:message" { + t.Fatalf("message correlation = %+v, dispatch = %+v", message, result) + } + var payload struct { + ReportID int `json:"report_id"` + } + if err := message.Bind(&payload); err != nil { + t.Fatalf("bind delivered message: %v", err) + } + if payload.ReportID != 23 { + t.Fatalf("message payload = %+v, want report_id 23", payload) + } +} + +// TestRedisWorkerIgnoresUnusableDriverMetadata verifies corrupt and future headers preserve ordinary delivery and raw observation. +func TestRedisWorkerIgnoresUnusableDriverMetadata(t *testing.T) { + tests := []struct { + name string + header string + }{ + {name: "malformed", header: `{"schema_version":`}, + {name: "unknown version", header: `{"schema_version":2,"dispatch_id":"spoofed"}`}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + server := &serverStub{} + var events []queue.Event + observer := queue.ObserverFunc(func(_ context.Context, event queue.Event) { + events = append(events, event) + }) + worker := newRedisWorker(server, backend.NewServeMux(), observer) + worker.Register("reports:fallback", func(_ context.Context, job queue.Job) error { + if metadata := queue.DriverMetadata(job); metadata.SchemaVersion != 0 { + t.Fatalf("handler trusted unusable metadata: %+v", metadata) + } + if got := string(job.PayloadBytes()); got != "ordinary-payload" { + t.Fatalf("handler payload = %q, want ordinary-payload", got) + } + return nil + }) + if err := worker.StartWorkers(context.Background()); err != nil { + t.Fatalf("start worker: %v", err) + } + task := backend.NewTaskWithHeaders("reports:fallback", []byte("ordinary-payload"), map[string]string{ + redisDriverJobMetadataHeader: test.header, + }) + if err := server.lastStartHandler.ProcessTask(context.Background(), task); err != nil { + t.Fatalf("process fallback task: %v", err) + } + if len(events) != 2 { + t.Fatalf("events = %+v, want start and success", events) + } + for _, event := range events { + if event.JobType != "reports:fallback" || event.DispatchID != "" || event.JobID != "" || event.ChainID != "" || event.BatchID != "" { + t.Fatalf("unusable metadata changed raw observation: %+v", event) + } + } + }) + } +} diff --git a/driver/redisqueue/go.mod b/driver/redisqueue/go.mod index 4714c0f..7591c77 100644 --- a/driver/redisqueue/go.mod +++ b/driver/redisqueue/go.mod @@ -5,13 +5,13 @@ go 1.24.4 require ( github.com/goforj/queue v0.0.0 github.com/hibiken/asynq v0.26.0 + github.com/redis/go-redis/v9 v9.14.1 ) require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/google/uuid v1.6.0 // indirect - github.com/redis/go-redis/v9 v9.14.1 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/spf13/cast v1.10.0 // indirect golang.org/x/sys v0.40.0 // indirect diff --git a/driver/redisqueue/queue_redis_impl.go b/driver/redisqueue/queue_redis_impl.go index 5bad3bd..12fd1cf 100644 --- a/driver/redisqueue/queue_redis_impl.go +++ b/driver/redisqueue/queue_redis_impl.go @@ -2,9 +2,13 @@ package redisqueue import ( "context" + "crypto/rand" + "encoding/hex" "encoding/json" "errors" "fmt" + "strconv" + "strings" "sync" "time" @@ -20,6 +24,7 @@ type redisEnqueueClient interface { } type redisInspector interface { + Close() error Queues() ([]string, error) GetQueueInfo(queue string) (*backend.QueueInfo, error) PauseQueue(queue string) error @@ -47,6 +52,8 @@ type redisQueue struct { client redisEnqueueClient inspector redisInspector timeline redisTimelineStore + unique redisUniqueStore + state redisStateStore ownsClient bool closeOnce sync.Once @@ -57,6 +64,17 @@ type redisTimelineStore interface { Set(ctx context.Context, key string, value any, expiration time.Duration) error } +type redisUniqueStore interface { + Acquire(ctx context.Context, key, token string, ttl time.Duration) (bool, error) + Release(ctx context.Context, key, token string) error +} + +type redisStateStore interface { + redisTimelineStore + redisUniqueStore + Close() error +} + type redisTimelineClient struct { client *redis.Client } @@ -75,10 +93,37 @@ func (c *redisTimelineClient) Set(ctx context.Context, key string, value any, ex return c.client.Set(ctx, key, value, expiration).Err() } +// Close releases the Redis client shared by timeline and logical uniqueness state. +func (c *redisTimelineClient) Close() error { + if c == nil || c.client == nil { + return nil + } + return c.client.Close() +} + +// Acquire atomically creates a logical uniqueness claim for its TTL window. +func (c *redisTimelineClient) Acquire(ctx context.Context, key, token string, ttl time.Duration) (bool, error) { + return c.client.SetNX(ctx, key, token, ttl).Result() +} + +// Release compensates a rejected enqueue without deleting a claim acquired after this token expired. +func (c *redisTimelineClient) Release(ctx context.Context, key, token string) error { + const compareAndDelete = ` +if redis.call("GET", KEYS[1]) == ARGV[1] then + return redis.call("DEL", KEYS[1]) +end +return 0` + return redis.NewScript(compareAndDelete).Run(ctx, c.client, []string{key}, token).Err() +} + const redisDefaultJobTimeout = 30 * time.Second +const redisUniqueCompensationTimeout = 5 * time.Second +const redisMinimumUniqueTTL = time.Second +const redisMaximumApplicationRetry = int(^uint32(0)>>1) - 1 -func newRedisQueue(client redisEnqueueClient, inspector redisInspector, timeline redisTimelineStore, ownsClient bool) *redisQueue { - return &redisQueue{client: client, inspector: inspector, timeline: timeline, ownsClient: ownsClient} +// newRedisQueue requires one state implementation so timeline and uniqueness behavior cannot silently diverge. +func newRedisQueue(client redisEnqueueClient, inspector redisInspector, state redisStateStore, ownsClient bool) *redisQueue { + return &redisQueue{client: client, inspector: inspector, timeline: state, unique: state, state: state, ownsClient: ownsClient} } func newRedisClient(cfg Config) redisEnqueueClient { @@ -97,7 +142,8 @@ func newRedisInspector(cfg Config) redisInspector { }) } -func newRedisTimelineStore(cfg Config) redisTimelineStore { +// newRedisTimelineStore creates the shared Redis state client used by history and logical claims. +func newRedisTimelineStore(cfg Config) redisStateStore { return &redisTimelineClient{ client: redis.NewClient(&redis.Options{ Addr: cfg.Addr, @@ -125,16 +171,36 @@ func (d *redisQueue) Preflight(ctx context.Context) error { return err } +// Shutdown closes every owned Redis resource once and reports cleanup failures only to the caller that performed the close. func (d *redisQueue) Shutdown(_ context.Context) error { - if d.ownsClient && d.client != nil { - d.closeOnce.Do(func() { - _ = d.client.Close() - }) + if !d.ownsClient { + return nil } - return nil + var closeErr error + d.closeOnce.Do(func() { + var closeErrs []error + if d.client != nil { + closeErrs = append(closeErrs, d.client.Close()) + } + if d.inspector != nil { + closeErrs = append(closeErrs, d.inspector.Close()) + } + if d.state != nil { + closeErrs = append(closeErrs, d.state.Close()) + } + closeErr = errors.Join(closeErrs...) + }) + return closeErr } -func (d *redisQueue) Dispatch(_ context.Context, job queue.Job) error { +// Dispatch validates Asynq options before acquiring the canonical logical claim. +func (d *redisQueue) Dispatch(ctx context.Context, job queue.Job) error { + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } if d.client == nil { return fmt.Errorf("queue client unavailable for redis driver") } @@ -142,9 +208,15 @@ func (d *redisQueue) Dispatch(_ context.Context, job queue.Job) error { return err } parsed := queuecore.DriverOptions(job) - if parsed.QueueName == "" { + if strings.TrimSpace(job.Type) == "" { + return fmt.Errorf("redis job type must contain one or more characters") + } + if strings.TrimSpace(parsed.QueueName) == "" { return fmt.Errorf("job queue is required") } + if parsed.UniqueTTL > 0 && parsed.UniqueTTL < redisMinimumUniqueTTL { + return fmt.Errorf("redis unique ttl must be >= %s", redisMinimumUniqueTTL) + } backendOpts := make([]backend.Option, 0, 5) backendOpts = append(backendOpts, backend.Queue(parsed.QueueName)) if parsed.Timeout != nil { @@ -152,8 +224,26 @@ func (d *redisQueue) Dispatch(_ context.Context, job queue.Job) error { } else { backendOpts = append(backendOpts, backend.Timeout(redisDefaultJobTimeout)) } + headers := make(map[string]string, 2) + metadata := queue.DriverMetadata(job) + if metadata.SchemaVersion != 0 { + encoded, err := json.Marshal(metadata) + if err != nil { + return fmt.Errorf("encode redis driver job metadata: %w", err) + } + headers[redisDriverJobMetadataHeader] = string(encoded) + } if parsed.MaxRetry != nil { - backendOpts = append(backendOpts, backend.MaxRetry(*parsed.MaxRetry)) + if *parsed.MaxRetry > redisMaximumApplicationRetry { + return fmt.Errorf("redis retry must be <= %d so its transport reserve fits the Asynq wire format", redisMaximumApplicationRetry) + } + backendOpts = append(backendOpts, backend.MaxRetry(*parsed.MaxRetry+1)) + headers[redisApplicationMaxRetryHeader] = strconv.Itoa(*parsed.MaxRetry) + } + payload := job.PayloadBytes() + task := backend.NewTask(job.Type, payload) + if len(headers) > 0 { + task = backend.NewTaskWithHeaders(job.Type, payload, headers) } if parsed.Backoff != nil && *parsed.Backoff > 0 { return queuecore.ErrBackoffUnsupported @@ -161,16 +251,60 @@ func (d *redisQueue) Dispatch(_ context.Context, job queue.Job) error { if parsed.Delay > 0 { backendOpts = append(backendOpts, backend.ProcessIn(parsed.Delay)) } + var ( + uniqueKey string + uniqueToken string + ) + logicalUnique := parsed.UniqueTTL > 0 && d.unique != nil + if logicalUnique { + uniqueKey = redisLogicalUniqueKey(job, parsed.QueueName) + var tokenErr error + uniqueToken, tokenErr = newRedisUniqueToken() + if tokenErr != nil { + return tokenErr + } + acquired, acquireErr := d.unique.Acquire(ctx, uniqueKey, uniqueToken, parsed.UniqueTTL) + if acquireErr != nil { + return acquireErr + } + if !acquired { + return queuecore.ErrDuplicate + } + } if parsed.UniqueTTL > 0 { + // Retaining Asynq's physical claim keeps direct jobs visible to older producers during one TTL rollout window. backendOpts = append(backendOpts, backend.Unique(parsed.UniqueTTL)) } - _, err := d.client.Enqueue(backend.NewTask(job.Type, job.PayloadBytes()), backendOpts...) + _, err := d.client.Enqueue(task, backendOpts...) if errors.Is(err, backend.ErrDuplicateTask) { + if !logicalUnique { + return queuecore.ErrDuplicate + } + compensationCtx, cancel := context.WithTimeout(context.Background(), redisUniqueCompensationTimeout) + defer cancel() + if releaseErr := d.unique.Release(compensationCtx, uniqueKey, uniqueToken); releaseErr != nil { + return errors.Join(queuecore.ErrDuplicate, fmt.Errorf("release redis uniqueness claim: %w", releaseErr)) + } return queuecore.ErrDuplicate } + // Other enqueue errors may arrive after Redis committed the task. Retaining the claim fails closed until its TTL instead of admitting a duplicate retry. return err } +// redisLogicalUniqueKey isolates goforj claims from Asynq's private key namespace. +func redisLogicalUniqueKey(job queue.Job, queueName string) string { + return "goforj:queue:unique:" + queuecore.UniqueKey(job, queueName) +} + +// newRedisUniqueToken prevents late compensation from deleting a newer TTL claim. +func newRedisUniqueToken() (string, error) { + var token [16]byte + if _, err := rand.Read(token[:]); err != nil { + return "", fmt.Errorf("create redis uniqueness token: %w", err) + } + return hex.EncodeToString(token[:]), nil +} + func (d *redisQueue) Pause(_ context.Context, queueName string) error { if d.inspector == nil { return queue.ErrPauseUnsupported @@ -283,7 +417,7 @@ func (d *redisQueue) ListJobs(ctx context.Context, opts queue.ListJobsOptions) ( Type: task.Type, Payload: string(task.Payload), Attempt: task.Retried, - MaxRetry: task.MaxRetry, + MaxRetry: redisApplicationMaxRetryFromHeaders(task.Headers, task.MaxRetry), LastError: task.LastErr, NextProcessAt: nextProcessAt, CompletedAt: completedAt, diff --git a/driver/redisqueue/queue_redis_impl_test.go b/driver/redisqueue/queue_redis_impl_test.go index 4d093de..792be92 100644 --- a/driver/redisqueue/queue_redis_impl_test.go +++ b/driver/redisqueue/queue_redis_impl_test.go @@ -3,10 +3,12 @@ package redisqueue import ( "context" "errors" + "sync" "testing" "time" "github.com/goforj/queue" + "github.com/goforj/queue/internal/driverbridge" backend "github.com/hibiken/asynq" ) @@ -27,22 +29,83 @@ type redisInspectorStub struct { runTaskID string archivedTaskID string deleteAllErr error + closeN int + closeErr error +} + +// Close records inspector ownership cleanup. +func (s *redisInspectorStub) Close() error { + s.closeN++ + return s.closeErr } type redisEnqueueClientStub struct { enqueueErr error enqueueN int closeN int + closeErr error + task *backend.Task + opts []backend.Option +} + +type redisUniqueStoreStub struct { + acquired bool + acquireErr error + releaseErr error + acquireKey string + acquireToken string + releaseKey string + releaseToken string +} + +type redisStateStoreStub struct { + redisUniqueStoreStub + closeN int + closeErr error +} + +// Acquire records the logical claim requested by the queue. +func (s *redisUniqueStoreStub) Acquire(_ context.Context, key, token string, _ time.Duration) (bool, error) { + s.acquireKey = key + s.acquireToken = token + return s.acquired, s.acquireErr } -func (s *redisEnqueueClientStub) Enqueue(*backend.Task, ...backend.Option) (*backend.TaskInfo, error) { +// Release records the ownership token used for compensation. +func (s *redisUniqueStoreStub) Release(_ context.Context, key, token string) error { + s.releaseKey = key + s.releaseToken = token + return s.releaseErr +} + +// Get returns no timeline sample because lifecycle tests only require the shared state contract. +func (s *redisStateStoreStub) Get(context.Context, string) (string, error) { + return "", nil +} + +// Set accepts timeline samples because lifecycle tests only require the shared state contract. +func (s *redisStateStoreStub) Set(context.Context, string, any, time.Duration) error { + return nil +} + +// Close records shared Redis state cleanup. +func (s *redisStateStoreStub) Close() error { + s.closeN++ + return s.closeErr +} + +// Enqueue records the task and options passed through the Redis acceptance boundary. +func (s *redisEnqueueClientStub) Enqueue(task *backend.Task, opts ...backend.Option) (*backend.TaskInfo, error) { s.enqueueN++ + s.task = task + s.opts = append([]backend.Option(nil), opts...) return &backend.TaskInfo{}, s.enqueueErr } +// Close records enqueue-client ownership cleanup. func (s *redisEnqueueClientStub) Close() error { s.closeN++ - return nil + return s.closeErr } func (s *redisInspectorStub) Queues() ([]string, error) { @@ -172,6 +235,55 @@ func (s *redisInspectorStub) GetTaskInfo(queueName, id string) (*backend.TaskInf return nil, backend.ErrTaskNotFound } +// TestRedisQueueConstruction verifies the producer shares one state client across history, uniqueness, and cleanup. +func TestRedisQueueConstruction(t *testing.T) { + client := &redisEnqueueClientStub{} + inspector := &redisInspectorStub{} + state := &redisStateStoreStub{} + driver := newRedisQueue(client, inspector, state, true) + + if driver.client != client || driver.inspector != inspector || driver.timeline != state || driver.unique != state || driver.state != state || !driver.ownsClient { + t.Fatalf("new Redis queue did not preserve dependencies: %+v", driver) + } + if got := driver.Driver(); got != queue.DriverRedis { + t.Fatalf("driver = %q, want %q", got, queue.DriverRedis) + } + + cfg := Config{Addr: "redis.example:6380", Password: "secret", DB: 3} + configuredState, ok := newRedisTimelineStore(cfg).(*redisTimelineClient) + if !ok { + t.Fatal("new Redis timeline store returned an unexpected implementation") + } + opts := configuredState.client.Options() + if opts.Addr != cfg.Addr || opts.Password != cfg.Password || opts.DB != cfg.DB { + t.Fatalf("state client options = addr:%q password:%q db:%d", opts.Addr, opts.Password, opts.DB) + } + if err := configuredState.Close(); err != nil { + t.Fatalf("close configured state client: %v", err) + } +} + +// TestRedisQueuePreflightBoundaries verifies context, dependency, and backend failures are returned before dispatch starts. +func TestRedisQueuePreflightBoundaries(t *testing.T) { + if err := (&redisQueue{inspector: &redisInspectorStub{}}).Preflight(nil); err != nil { + t.Fatalf("preflight with nil context: %v", err) + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := (&redisQueue{inspector: &redisInspectorStub{}}).Preflight(ctx); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled preflight = %v, want context.Canceled", err) + } + if err := (&redisQueue{}).Preflight(context.Background()); err == nil { + t.Fatal("preflight without an inspector unexpectedly succeeded") + } + + preflightErr := errors.New("redis unavailable") + if err := (&redisQueue{inspector: &redisInspectorStub{queuesErr: preflightErr}}).Preflight(context.Background()); !errors.Is(err, preflightErr) { + t.Fatalf("backend preflight = %v, want %v", err, preflightErr) + } +} + func TestRedisQueue_PauseResumeNormalization(t *testing.T) { inspector := &redisInspectorStub{} r := &redisQueue{inspector: inspector} @@ -262,7 +374,15 @@ func TestRedisQueue_AdminBranches(t *testing.T) { tasksByQueue: map[string]map[string][]*backend.TaskInfo{ "default": { backend.TaskStatePending.String(): { - {ID: "job-pending", Queue: "default", Type: "job:pending", Payload: []byte("payload"), State: backend.TaskStatePending}, + { + ID: "job-pending", + Queue: "default", + Type: "job:pending", + Payload: []byte("payload"), + State: backend.TaskStatePending, + MaxRetry: 3, + Headers: map[string]string{redisApplicationMaxRetryHeader: "2"}, + }, }, }, }, @@ -277,6 +397,9 @@ func TestRedisQueue_AdminBranches(t *testing.T) { if list.Total != 1 || len(list.Jobs) != 1 { t.Fatalf("expected one job, got total=%d len=%d", list.Total, len(list.Jobs)) } + if list.Jobs[0].MaxRetry != 2 { + t.Fatalf("admin max retry = %d, want application budget 2", list.Jobs[0].MaxRetry) + } if err := r.CancelJob(context.Background(), "job-pending"); err != nil { t.Fatalf("cancel job failed: %v", err) @@ -359,19 +482,287 @@ func TestRedisQueue_DispatchBranches(t *testing.T) { if client.enqueueN != 1 { t.Fatalf("expected one enqueue call, got %d", client.enqueueN) } + if got := client.task.Headers()[redisApplicationMaxRetryHeader]; got != "2" { + t.Fatalf("application retry header = %q, want 2", got) + } + var transportMaxRetry int + for _, option := range client.opts { + if option.Type() == backend.MaxRetryOpt { + transportMaxRetry = option.Value().(int) + } + } + if transportMaxRetry != 3 { + t.Fatalf("transport max retry = %d, want one-slot reserve 3", transportMaxRetry) + } + }) + + t.Run("retry reserve wire boundary", func(t *testing.T) { + client := &redisEnqueueClientStub{} + r := &redisQueue{client: client} + if err := r.Dispatch(context.Background(), queue.NewJob("job:redis").OnQueue("default").Retry(redisMaximumApplicationRetry)); err != nil { + t.Fatalf("maximum application retry rejected: %v", err) + } + if client.enqueueN != 1 { + t.Fatalf("maximum application retry enqueues = %d, want 1", client.enqueueN) + } + err := r.Dispatch(context.Background(), queue.NewJob("job:redis").OnQueue("default").Retry(redisMaximumApplicationRetry+1)) + if err == nil || client.enqueueN != 1 { + t.Fatalf("retry reserve overflow = error:%v total enqueues:%d, want rejection after first boundary enqueue", err, client.enqueueN) + } + }) + + t.Run("unique ttl validates before canonical claim", func(t *testing.T) { + client := &redisEnqueueClientStub{} + claims := &redisUniqueStoreStub{acquired: true} + r := &redisQueue{client: client, unique: claims} + err := r.Dispatch(context.Background(), queue.NewJob("job:redis").OnQueue("default").UniqueFor(time.Millisecond)) + if err == nil { + t.Fatal("sub-second redis uniqueness unexpectedly passed validation") + } + if claims.acquireKey != "" || client.enqueueN != 0 { + t.Fatalf("invalid uniqueness reached claim/enqueue: key=%q enqueues=%d", claims.acquireKey, client.enqueueN) + } }) } +// TestRedisQueueDispatchInputAndClaimFailures verifies context normalization, +// Redis-specific type validation, and claim-store failures stop at the expected boundary. +func TestRedisQueueDispatchInputAndClaimFailures(t *testing.T) { + t.Run("nil context", func(t *testing.T) { + client := &redisEnqueueClientStub{} + r := &redisQueue{client: client} + if err := r.Dispatch(nil, queue.NewJob("job:nil-context").OnQueue("default")); err != nil { + t.Fatalf("dispatch with nil context: %v", err) + } + if client.enqueueN != 1 { + t.Fatalf("enqueue calls = %d, want 1", client.enqueueN) + } + }) + + t.Run("canceled context", func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + client := &redisEnqueueClientStub{} + r := &redisQueue{client: client} + if err := r.Dispatch(ctx, queue.NewJob("job:canceled").OnQueue("default")); !errors.Is(err, context.Canceled) { + t.Fatalf("dispatch error = %v, want context.Canceled", err) + } + if client.enqueueN != 0 { + t.Fatalf("canceled dispatch reached enqueue %d times", client.enqueueN) + } + }) + + t.Run("whitespace type", func(t *testing.T) { + client := &redisEnqueueClientStub{} + r := &redisQueue{client: client} + if err := r.Dispatch(context.Background(), queue.NewJob(" \t").OnQueue("default")); err == nil { + t.Fatal("expected whitespace-only Redis type rejection") + } + if client.enqueueN != 0 { + t.Fatalf("invalid type reached enqueue %d times", client.enqueueN) + } + }) + + t.Run("whitespace queue", func(t *testing.T) { + client := &redisEnqueueClientStub{} + r := &redisQueue{client: client} + if err := r.Dispatch(context.Background(), queue.NewJob("job:redis").OnQueue(" \t")); err == nil { + t.Fatal("expected whitespace-only Redis queue rejection") + } + if client.enqueueN != 0 { + t.Fatalf("invalid queue reached enqueue %d times", client.enqueueN) + } + }) + + t.Run("delay and default timeout", func(t *testing.T) { + client := &redisEnqueueClientStub{} + r := &redisQueue{client: client} + if err := r.Dispatch(context.Background(), queue.NewJob("job:delayed").OnQueue("default").Delay(2*time.Second)); err != nil { + t.Fatalf("dispatch delayed job: %v", err) + } + var ( + delay time.Duration + timeout time.Duration + ) + for _, option := range client.opts { + switch option.Type() { + case backend.ProcessInOpt: + delay = option.Value().(time.Duration) + case backend.TimeoutOpt: + timeout = option.Value().(time.Duration) + } + } + if delay != 2*time.Second || timeout != redisDefaultJobTimeout { + t.Fatalf("dispatch options = delay:%s timeout:%s", delay, timeout) + } + }) + + t.Run("claim store failure", func(t *testing.T) { + acquireErr := errors.New("claim store unavailable") + client := &redisEnqueueClientStub{} + claims := &redisUniqueStoreStub{acquireErr: acquireErr} + r := &redisQueue{client: client, unique: claims} + err := r.Dispatch(context.Background(), queue.NewJob("job:claim").OnQueue("default").UniqueFor(time.Minute)) + if !errors.Is(err, acquireErr) { + t.Fatalf("dispatch error = %v, want %v", err, acquireErr) + } + if claims.acquireKey == "" || claims.acquireToken == "" || client.enqueueN != 0 { + t.Fatalf("claim failure state = key:%q token:%q enqueues:%d", claims.acquireKey, claims.acquireToken, client.enqueueN) + } + }) +} + +// TestRedisQueueLogicalUniqueFailureBoundaries verifies ambiguous failures retain claims while definite physical duplicates compensate them. +func TestRedisQueueLogicalUniqueFailureBoundaries(t *testing.T) { + payload := []byte(`{"schema_version":1,"dispatch_id":"volatile","job_id":"job_1","job":{"type":"reports:build","payload":"eyJpZCI6MX0="}}`) + job := queue.NewJob("bus:job").Payload(payload).OnQueue("critical").UniqueFor(time.Minute) + enqueueErr := errors.New("redis response lost") + client := &redisEnqueueClientStub{enqueueErr: enqueueErr} + claims := &redisUniqueStoreStub{acquired: true} + r := &redisQueue{client: client, unique: claims} + + err := r.Dispatch(context.Background(), job) + if !errors.Is(err, enqueueErr) { + t.Fatalf("dispatch error = %v, want enqueue rejection", err) + } + if claims.acquireKey == "" || claims.acquireToken == "" { + t.Fatalf("logical claim was incomplete: %+v", claims) + } + if claims.releaseKey != "" || claims.releaseToken != "" { + t.Fatalf("ambiguous enqueue failure released its safety claim: %+v", claims) + } + + client = &redisEnqueueClientStub{enqueueErr: backend.ErrDuplicateTask} + claims = &redisUniqueStoreStub{acquired: true} + r = &redisQueue{client: client, unique: claims} + if err := r.Dispatch(context.Background(), job); !errors.Is(err, queue.ErrDuplicate) { + t.Fatalf("physical duplicate error = %v, want ErrDuplicate", err) + } + if claims.releaseKey != claims.acquireKey || claims.releaseToken != claims.acquireToken { + t.Fatalf("physical duplicate compensation released a different owner: %+v", claims) + } + + releaseErr := errors.New("redis release failed") + claims = &redisUniqueStoreStub{acquired: true, releaseErr: releaseErr} + r = &redisQueue{client: &redisEnqueueClientStub{enqueueErr: backend.ErrDuplicateTask}, unique: claims} + err = r.Dispatch(context.Background(), job) + if !errors.Is(err, queue.ErrDuplicate) || !errors.Is(err, releaseErr) { + t.Fatalf("release failure error = %v, want duplicate and release causes", err) + } + + claims = &redisUniqueStoreStub{acquired: false} + r = &redisQueue{client: &redisEnqueueClientStub{}, unique: claims} + if err := r.Dispatch(context.Background(), job); !errors.Is(err, queue.ErrDuplicate) { + t.Fatalf("duplicate logical claim error = %v, want ErrDuplicate", err) + } +} + +// TestRedisQueue_ShutdownOwnsClientCloseOnce verifies owned cleanup reports every failure once before later retries converge. func TestRedisQueue_ShutdownOwnsClientCloseOnce(t *testing.T) { - client := &redisEnqueueClientStub{} - r := &redisQueue{client: client, ownsClient: true} - if err := r.Shutdown(context.Background()); err != nil { - t.Fatalf("shutdown failed: %v", err) + clientErr := errors.New("close enqueue client") + inspectorErr := errors.New("close inspector") + stateErr := errors.New("close state") + client := &redisEnqueueClientStub{closeErr: clientErr} + inspector := &redisInspectorStub{closeErr: inspectorErr} + state := &redisStateStoreStub{closeErr: stateErr} + r := newRedisQueue(client, inspector, state, true) + + err := r.Shutdown(context.Background()) + if !errors.Is(err, clientErr) || !errors.Is(err, inspectorErr) || !errors.Is(err, stateErr) { + t.Fatalf("first shutdown = %v, want all close failures", err) } if err := r.Shutdown(context.Background()); err != nil { - t.Fatalf("second shutdown failed: %v", err) + t.Fatalf("retry shutdown = %v, want completed cleanup", err) + } + if client.closeN != 1 || inspector.closeN != 1 || state.closeN != 1 { + t.Fatalf("close counts = client:%d inspector:%d state:%d, want one each", client.closeN, inspector.closeN, state.closeN) + } + + notOwnedClient := &redisEnqueueClientStub{} + notOwnedInspector := &redisInspectorStub{} + notOwnedState := &redisStateStoreStub{} + notOwned := newRedisQueue(notOwnedClient, notOwnedInspector, notOwnedState, false) + if err := notOwned.Shutdown(context.Background()); err != nil { + t.Fatalf("shutdown unowned resources: %v", err) + } + if notOwnedClient.closeN != 0 || notOwnedInspector.closeN != 0 || notOwnedState.closeN != 0 { + t.Fatalf("unowned resources closed = client:%d inspector:%d state:%d", notOwnedClient.closeN, notOwnedInspector.closeN, notOwnedState.closeN) + } +} + +// TestRedisQueue_ShutdownConcurrentCallersCloseResourcesOnce verifies one caller owns diagnostics while concurrent followers observe completed cleanup. +func TestRedisQueue_ShutdownConcurrentCallersCloseResourcesOnce(t *testing.T) { + clientErr := errors.New("close enqueue client") + inspectorErr := errors.New("close inspector") + stateErr := errors.New("close state") + client := &redisEnqueueClientStub{closeErr: clientErr} + inspector := &redisInspectorStub{closeErr: inspectorErr} + state := &redisStateStoreStub{closeErr: stateErr} + r := newRedisQueue(client, inspector, state, true) + + const callers = 32 + start := make(chan struct{}) + results := make(chan error, callers) + var callersDone sync.WaitGroup + callersDone.Add(callers) + for range callers { + go func() { + defer callersDone.Done() + <-start + results <- r.Shutdown(context.Background()) + }() + } + close(start) + callersDone.Wait() + close(results) + + diagnosticCalls := 0 + for err := range results { + if err == nil { + continue + } + diagnosticCalls++ + if !errors.Is(err, clientErr) || !errors.Is(err, inspectorErr) || !errors.Is(err, stateErr) { + t.Fatalf("concurrent shutdown = %v, want all close failures", err) + } + } + if diagnosticCalls != 1 { + t.Fatalf("shutdown diagnostic calls = %d, want 1", diagnosticCalls) + } + if client.closeN != 1 || inspector.closeN != 1 || state.closeN != 1 { + t.Fatalf("close counts = client:%d inspector:%d state:%d, want one each", client.closeN, inspector.closeN, state.closeN) + } +} + +// TestRedisQueue_ShutdownRetryClosesRootRuntime verifies a producer close diagnostic does not leave the public queue permanently draining. +func TestRedisQueue_ShutdownRetryClosesRootRuntime(t *testing.T) { + clientErr := errors.New("close enqueue client") + inspectorErr := errors.New("close inspector") + stateErr := errors.New("close state") + client := &redisEnqueueClientStub{closeErr: clientErr} + inspector := &redisInspectorStub{closeErr: inspectorErr} + state := &redisStateStoreStub{closeErr: stateErr} + producer := newRedisQueue(client, inspector, state, true) + q, err := driverbridge.NewQueueFromDriver( + queue.Config{Driver: queue.DriverRedis, DefaultQueue: "default"}, + producer, + nil, + ) + if err != nil { + t.Fatalf("construct queue: %v", err) + } + + err = q.Shutdown(context.Background()) + if !errors.Is(err, clientErr) || !errors.Is(err, inspectorErr) || !errors.Is(err, stateErr) { + t.Fatalf("first root shutdown = %v, want all close failures", err) + } + if err := q.Shutdown(context.Background()); err != nil { + t.Fatalf("retry root shutdown = %v, want terminal cleanup", err) + } + if err := q.Shutdown(context.Background()); err != nil { + t.Fatalf("closed root shutdown = %v, want idempotent success", err) } - if client.closeN != 1 { - t.Fatalf("expected close once, got %d", client.closeN) + if client.closeN != 1 || inspector.closeN != 1 || state.closeN != 1 { + t.Fatalf("close counts = client:%d inspector:%d state:%d, want one each", client.closeN, inspector.closeN, state.closeN) } } diff --git a/driver/redisqueue/redisqueue.go b/driver/redisqueue/redisqueue.go index c79d508..ea09f03 100644 --- a/driver/redisqueue/redisqueue.go +++ b/driver/redisqueue/redisqueue.go @@ -1,11 +1,13 @@ package redisqueue import ( + "errors" "fmt" "strings" "time" "github.com/goforj/queue" + "github.com/goforj/queue/busruntime" "github.com/goforj/queue/internal/driverbridge" "github.com/goforj/queue/queueconfig" "github.com/goforj/queue/queuecore" @@ -85,10 +87,11 @@ func NewWithConfig(cfg Config, opts ...queue.Option) (*queue.Queue, error) { if cfg.Addr == "" { return nil, fmt.Errorf("redis addr is required") } + observer := driverbridge.NewObserverSink(cfg.Observer) rootCfg := queue.Config{ Driver: queue.DriverRedis, DefaultQueue: cfg.DefaultQueue, - Observer: cfg.Observer, + Observer: observer, Logger: cfg.Logger, } driverBackend := newRedisQueue(newRedisClient(cfg), newRedisInspector(cfg), newRedisTimelineStore(cfg), true) @@ -100,7 +103,7 @@ func NewWithConfig(cfg Config, opts ...queue.Option) (*queue.Queue, error) { DB: cfg.DB, }, serverConfig(cfg, workers)), backend.NewServeMux(), - cfg.Observer, + observer, ), nil }, opts...) if err != nil { @@ -110,7 +113,11 @@ func NewWithConfig(cfg Config, opts ...queue.Option) (*queue.Queue, error) { } func serverConfig(cfg Config, workers int) backend.Config { - serverCfg := backend.Config{Concurrency: workers} + serverCfg := backend.Config{ + Concurrency: workers, + IsFailure: redisAttemptIsFailure, + RetryDelayFunc: redisRetryDelay, + } if queues := normalizeQueues(cfg.Queues, cfg.DefaultQueue); len(queues) > 0 { serverCfg.Queues = queues } @@ -128,6 +135,19 @@ func serverConfig(cfg Config, workers int) backend.Config { return serverCfg } +// redisAttemptIsFailure keeps workflow mutation and lease-recovery redelivery from consuming the application retry counter. +func redisAttemptIsFailure(err error) bool { + return err != nil && !busruntime.IsUncommitted(err) && !errors.Is(err, backend.ErrLeaseExpired) +} + +// redisRetryDelay keeps infrastructure redelivery responsive without changing application retry backoff. +func redisRetryDelay(attempt int, err error, task *backend.Task) time.Duration { + if busruntime.IsUncommitted(err) || errors.Is(err, backend.ErrLeaseExpired) { + return time.Second + } + return backend.DefaultRetryDelayFunc(attempt, err, task) +} + func normalizeQueues(raw map[string]int, fallbackDefault string) map[string]int { if len(raw) == 0 { return map[string]int{queuecore.NormalizeQueueName(queue.PhysicalQueueName(fallbackDefault, fallbackDefault)): 1} diff --git a/driver/redisqueue/redisqueue_test.go b/driver/redisqueue/redisqueue_test.go index f30bf1f..8495f3d 100644 --- a/driver/redisqueue/redisqueue_test.go +++ b/driver/redisqueue/redisqueue_test.go @@ -1,10 +1,13 @@ package redisqueue import ( + "errors" + "fmt" "reflect" "testing" "time" + "github.com/goforj/queue/busruntime" "github.com/goforj/queue/queueconfig" backend "github.com/hibiken/asynq" ) @@ -83,6 +86,47 @@ func TestServerConfig_ShutdownTimeoutPassthrough(t *testing.T) { } } +// TestServerConfig_UncommittedErrorsDoNotCountAsFailures verifies the configured predicate preserves retry count while Asynq still has transport capacity. +func TestServerConfig_UncommittedErrorsDoNotCountAsFailures(t *testing.T) { + isFailure := serverConfig(Config{}, 1).IsFailure + if isFailure == nil { + t.Fatal("expected failure classifier") + } + cause := errors.New("outcome store unavailable") + if isFailure(nil) { + t.Fatal("nil result must not count as a failure") + } + if !isFailure(cause) { + t.Fatal("application error must count as a failure") + } + if isFailure(busruntime.Uncommitted(cause)) { + t.Fatal("uncommitted error must not count as a failure") + } + if isFailure(fmt.Errorf("commit callback: %w", busruntime.Uncommitted(cause))) { + t.Fatal("wrapped uncommitted error must not count as a failure") + } + if isFailure(backend.ErrLeaseExpired) { + t.Fatal("lease recovery must not consume the application retry counter") + } + if !isFailure(busruntime.Permanent(cause)) { + t.Fatal("permanent application error must count as a failure") + } +} + +// TestRedisRetryDelaySeparatesInfrastructureFromApplicationBackoff verifies recovery does not inherit randomized application delays. +func TestRedisRetryDelaySeparatesInfrastructureFromApplicationBackoff(t *testing.T) { + cause := errors.New("failed") + if got := redisRetryDelay(0, busruntime.Uncommitted(cause), backend.NewTask("job", nil)); got != time.Second { + t.Fatalf("uncommitted retry delay = %v, want 1s", got) + } + if got := redisRetryDelay(0, backend.ErrLeaseExpired, backend.NewTask("job", nil)); got != time.Second { + t.Fatalf("lease recovery delay = %v, want 1s", got) + } + if got := redisRetryDelay(0, cause, backend.NewTask("job", nil)); got < 15*time.Second { + t.Fatalf("application retry delay = %v, want Asynq default", got) + } +} + func TestNormalizeQueues(t *testing.T) { got := normalizeQueues(map[string]int{"": 2, " critical ": 3, "zero": 0, "neg": -1}, "") want := map[string]int{"default": 2, "critical": 3} diff --git a/driver/redisqueue/retry_reserve.go b/driver/redisqueue/retry_reserve.go new file mode 100644 index 0000000..c2bf212 --- /dev/null +++ b/driver/redisqueue/retry_reserve.go @@ -0,0 +1,33 @@ +package redisqueue + +import ( + "strconv" + + backend "github.com/hibiken/asynq" +) + +const redisApplicationMaxRetryHeader = "goforj-queue-application-max-retry" + +// redisApplicationMaxRetry restores the public retry budget when a new task carries one reserved Asynq transport slot. +func redisApplicationMaxRetry(task *backend.Task, transportMaxRetry int) int { + if task == nil { + return transportMaxRetry + } + return redisApplicationMaxRetryFromHeaders(task.Headers(), transportMaxRetry) +} + +// redisApplicationMaxRetryFromHeaders keeps worker delivery and administrative snapshots on the same public budget. +func redisApplicationMaxRetryFromHeaders(headers map[string]string, transportMaxRetry int) int { + raw, ok := headers[redisApplicationMaxRetryHeader] + if !ok { + return transportMaxRetry + } + applicationMaxRetry, err := strconv.Atoi(raw) + if err != nil || applicationMaxRetry < 0 || applicationMaxRetry == int(^uint(0)>>1) { + return transportMaxRetry + } + if applicationMaxRetry+1 != transportMaxRetry { + return transportMaxRetry + } + return applicationMaxRetry +} diff --git a/driver/redisqueue/state_client_test.go b/driver/redisqueue/state_client_test.go new file mode 100644 index 0000000..c4b93fd --- /dev/null +++ b/driver/redisqueue/state_client_test.go @@ -0,0 +1,155 @@ +package redisqueue + +import ( + "context" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/redis/go-redis/v9" +) + +type redisProtocolError string + +// Error returns the Redis protocol error text. +func (e redisProtocolError) Error() string { return string(e) } + +// RedisError marks the value as a Redis server response for go-redis fallback handling. +func (redisProtocolError) RedisError() {} + +type redisCommandHook struct { + values map[string]string + setArgs []any + acquireArgs [][]any + evalScript string +} + +// DialHook preserves the default dial path, which command interception prevents these tests from reaching. +func (h *redisCommandHook) DialHook(next redis.DialHook) redis.DialHook { return next } + +// ProcessHook provides deterministic Redis replies while retaining go-redis command construction and script fallback behavior. +func (h *redisCommandHook) ProcessHook(redis.ProcessHook) redis.ProcessHook { + return func(_ context.Context, cmd redis.Cmder) error { + args := cmd.Args() + switch typed := cmd.(type) { + case *redis.StatusCmd: + if strings.ToLower(cmd.Name()) != "set" || len(args) != 5 { + return fmt.Errorf("unexpected Redis status command: %#v", args) + } + h.setArgs = append([]any(nil), args...) + key, _ := args[1].(string) + value, _ := args[2].(string) + h.values[key] = value + typed.SetVal("OK") + case *redis.BoolCmd: + if strings.ToLower(cmd.Name()) != "set" || len(args) != 6 { + return fmt.Errorf("unexpected Redis boolean command: %#v", args) + } + h.acquireArgs = append(h.acquireArgs, append([]any(nil), args...)) + key, _ := args[1].(string) + value, _ := args[2].(string) + if _, exists := h.values[key]; exists { + typed.SetVal(false) + } else { + h.values[key] = value + typed.SetVal(true) + } + case *redis.StringCmd: + if strings.ToLower(cmd.Name()) != "get" || len(args) != 2 { + return fmt.Errorf("unexpected Redis string command: %#v", args) + } + key, _ := args[1].(string) + value, exists := h.values[key] + if !exists { + return redis.Nil + } + typed.SetVal(value) + case *redis.Cmd: + switch strings.ToLower(cmd.Name()) { + case "evalsha": + return redisProtocolError("NOSCRIPT no matching script") + case "eval": + if len(args) != 5 { + return fmt.Errorf("unexpected Redis eval command: %#v", args) + } + h.evalScript, _ = args[1].(string) + key, _ := args[3].(string) + token, _ := args[4].(string) + if h.values[key] == token { + delete(h.values, key) + typed.SetVal(int64(1)) + } else { + typed.SetVal(int64(0)) + } + default: + return fmt.Errorf("unexpected Redis command: %#v", args) + } + default: + return fmt.Errorf("unexpected Redis command type %T", cmd) + } + return nil + } +} + +// ProcessPipelineHook preserves the default pipeline path because the state adapter issues standalone commands. +func (h *redisCommandHook) ProcessPipelineHook(next redis.ProcessPipelineHook) redis.ProcessPipelineHook { + return next +} + +// TestRedisTimelineClientStateSemantics verifies timeline and uniqueness commands share one go-redis command implementation. +func TestRedisTimelineClientStateSemantics(t *testing.T) { + hook := &redisCommandHook{values: make(map[string]string)} + client := redis.NewClient(&redis.Options{Addr: "redis.invalid:6379"}) + client.AddHook(hook) + state := &redisTimelineClient{client: client} + ctx := context.Background() + + if err := state.Set(ctx, "timeline", "sample", time.Minute); err != nil { + t.Fatalf("set timeline sample: %v", err) + } + if args := hook.setArgs; len(args) != 5 || args[0] != "set" || args[1] != "timeline" || args[2] != "sample" || args[3] != "ex" || args[4] != int64(60) { + t.Fatalf("timeline SET arguments = %#v, want expiring 60-second command", args) + } + if got, err := state.Get(ctx, "timeline"); err != nil || got != "sample" { + t.Fatalf("get timeline sample = %q, %v, want sample", got, err) + } + + acquired, err := state.Acquire(ctx, "claim", "owner-a", time.Minute) + if err != nil || !acquired { + t.Fatalf("acquire initial claim = %t, %v, want true", acquired, err) + } + acquired, err = state.Acquire(ctx, "claim", "owner-b", time.Minute) + if err != nil || acquired { + t.Fatalf("acquire competing claim = %t, %v, want false", acquired, err) + } + if len(hook.acquireArgs) != 2 { + t.Fatalf("claim SET commands = %d, want 2", len(hook.acquireArgs)) + } + for index, owner := range []string{"owner-a", "owner-b"} { + args := hook.acquireArgs[index] + if len(args) != 6 || args[0] != "set" || args[1] != "claim" || args[2] != owner || args[3] != "ex" || args[4] != int64(60) || args[5] != "nx" { + t.Fatalf("claim SET arguments %d = %#v, want owner-scoped 60-second NX command", index, args) + } + } + + if err := state.Release(ctx, "claim", "owner-b"); err != nil { + t.Fatalf("release non-owner claim: %v", err) + } + if got, err := client.Get(ctx, "claim").Result(); err != nil || got != "owner-a" { + t.Fatalf("claim after non-owner release = %q, %v, want owner-a", got, err) + } + if err := state.Release(ctx, "claim", "owner-a"); err != nil { + t.Fatalf("release owned claim: %v", err) + } + if _, err := state.Get(ctx, "claim"); !errors.Is(err, redis.Nil) { + t.Fatalf("released claim lookup = %v, want redis.Nil", err) + } + if !strings.Contains(hook.evalScript, `redis.call("GET", KEYS[1]) == ARGV[1]`) || !strings.Contains(hook.evalScript, `redis.call("DEL", KEYS[1])`) { + t.Fatalf("release script does not preserve token-checked deletion: %q", hook.evalScript) + } + if err := state.Close(); err != nil { + t.Fatalf("close state client: %v", err) + } +} diff --git a/driver/redisqueue/worker_redis_impl.go b/driver/redisqueue/worker_redis_impl.go index b6778ea..429a441 100644 --- a/driver/redisqueue/worker_redis_impl.go +++ b/driver/redisqueue/worker_redis_impl.go @@ -2,10 +2,13 @@ package redisqueue import ( "context" + "errors" + "fmt" "sync" "time" "github.com/goforj/queue" + "github.com/goforj/queue/busruntime" "github.com/goforj/queue/queuecore" backend "github.com/hibiken/asynq" ) @@ -17,13 +20,21 @@ type server interface { } type redisWorker struct { - server server - mux *backend.ServeMux - obs queue.Observer + server server + mux *backend.ServeMux + obs queue.Observer ctxDecorator func(context.Context) context.Context - mu sync.Mutex - started bool + mu sync.Mutex + started bool + draining bool + stopDone chan struct{} +} + +type redisTransportDelivery struct { + attempt int + maxRetry int + queue string } func newRedisWorker(server server, mux *backend.ServeMux, observer queue.Observer) *redisWorker { @@ -38,80 +49,127 @@ func (w *redisWorker) Register(jobType string, handler queue.Handler) { if jobType == "" || handler == nil { return } - if w.obs == nil { - w.mux.HandleFunc(jobType, func(ctx context.Context, job *backend.Task) error { - if w.ctxDecorator != nil { - if decorated := w.ctxDecorator(ctx); decorated != nil { - ctx = decorated - } - } - return handler(ctx, queue.NewJob(job.Type()).Payload(job.Payload())) - }) - return - } w.mux.HandleFunc(jobType, func(ctx context.Context, job *backend.Task) error { - if w.ctxDecorator != nil { - if decorated := w.ctxDecorator(ctx); decorated != nil { - ctx = decorated - } - } - attempt, _ := backend.GetRetryCount(ctx) - maxRetry, _ := backend.GetMaxRetry(ctx) - queueName, _ := backend.GetQueueName(ctx) - queueName = queuecore.NormalizeQueueName(queueName) - observedJobType := queue.ResolveObservedJobType(job.Type(), job.Payload()) - - start := time.Now() - base := queue.Event{ - Driver: queue.DriverRedis, - Queue: queueName, - JobType: observedJobType, - Attempt: attempt, - MaxRetry: maxRetry, - Time: start, + return w.processTask(ctx, job, handler, redisTransportDeliveryFromContext(ctx)) + }) +} + +// redisTransportDeliveryFromContext captures Asynq-owned values before an +// application decorator can replace the transport context. +func redisTransportDeliveryFromContext(ctx context.Context) redisTransportDelivery { + attempt, _ := backend.GetRetryCount(ctx) + maxRetry, _ := backend.GetMaxRetry(ctx) + queueName, _ := backend.GetQueueName(ctx) + return redisTransportDelivery{attempt: attempt, maxRetry: maxRetry, queue: queueName} +} + +// processTask decorates application context while deriving delivery and event +// facts exclusively from the transport snapshot captured by Register. +func (w *redisWorker) processTask(ctx context.Context, job *backend.Task, handler queue.Handler, transport redisTransportDelivery) error { + if w.ctxDecorator != nil { + if decorated := w.ctxDecorator(ctx); decorated != nil { + ctx = busruntime.PreserveDeliveryContext(ctx, decorated) } - base.Kind = queue.EventProcessStarted - queuecore.SafeObserve(ctx, w.obs, base) - - err := handler(ctx, queuecore.DriverWithAttempt( - queue.NewJob(job.Type()). - Payload(job.Payload()). - OnQueue(queueName). - Retry(maxRetry), - attempt, - )) - finish := base - finish.Time = time.Now() - finish.Duration = time.Since(start) - finish.Err = err - if err == nil { - finish.Kind = queue.EventProcessSucceeded + } + maxRetry := redisApplicationMaxRetry(job, transport.maxRetry) + physicalAttempt := busruntime.DeliveryAttempt{Number: transport.attempt, MaxRetry: maxRetry} + queueName := queuecore.NormalizeQueueName(transport.queue) + delivery := queuecore.DriverWithAttempt( + queue.NewJob(job.Type()). + Payload(job.Payload()). + OnQueue(queueName). + Retry(maxRetry), + transport.attempt, + ) + delivery = redisJobWithDriverMetadata(delivery, job.Headers()) + if w.obs == nil { + return redisSettlementError(physicalAttempt, handler(ctx, delivery)) + } + metadata := queue.ResolveObservedJobMetadataFromJob(delivery) + + start := time.Now() + base := queue.Event{ + Driver: queue.DriverRedis, + Queue: queueName, + JobType: metadata.JobType, + JobKey: metadata.JobKey, + DispatchID: metadata.DispatchID, + JobID: metadata.JobID, + ChainID: metadata.ChainID, + BatchID: metadata.BatchID, + Attempt: transport.attempt, + MaxRetry: maxRetry, + Time: start, + } + observeRedisAttemptStart(ctx, w.obs, base) + defer func() { + if recovered := recover(); recovered != nil { + finish := base + finish.Kind = queue.EventProcessFailed + finish.Time = time.Now() + finish.Duration = time.Since(start) + finish.Err = redisHandlerPanicError(recovered) queuecore.SafeObserve(ctx, w.obs, finish) - return nil + panic(recovered) } - finish.Kind = queue.EventProcessFailed + }() + + err := handler(ctx, delivery) + finish := base + finish.Time = time.Now() + finish.Duration = time.Since(start) + finish.Err = err + if err == nil { + finish.Kind = queue.EventProcessSucceeded queuecore.SafeObserve(ctx, w.obs, finish) - if finish.Attempt < finish.MaxRetry { - retry := finish - retry.Kind = queue.EventProcessRetried - retry.Err = nil - queuecore.SafeObserve(ctx, w.obs, retry) - } else { - archive := finish - archive.Kind = queue.EventProcessArchived - archive.Err = nil - queuecore.SafeObserve(ctx, w.obs, archive) - } + return nil + } + finish.Kind = queue.EventProcessFailed + queuecore.SafeObserve(ctx, w.obs, finish) + return redisSettlementError(physicalAttempt, err) +} + +// redisHandlerPanicError preserves error identity for telemetry while the +// worker re-panics so Asynq retains ownership of panic recovery and retry. +func redisHandlerPanicError(recovered any) error { + if err, ok := recovered.(error); ok { + return fmt.Errorf("redis handler panicked: %w", err) + } + return fmt.Errorf("redis handler panicked: %v", recovered) +} + +// observeRedisAttemptStart treats an Asynq retry delivery as evidence that its application retry was scheduled; infrastructure redelivery may repeat the fact. +func observeRedisAttemptStart(ctx context.Context, observer queue.Observer, event queue.Event) { + if event.Attempt > 0 { + retry := event + retry.Kind = queue.EventProcessRetried + queuecore.SafeObserve(ctx, observer, retry) + } + event.Kind = queue.EventProcessStarted + queuecore.SafeObserve(ctx, observer, event) +} + +// redisSettlementError explicitly archives terminal application outcomes before the reserved Asynq slot can become an extra application retry. +func redisSettlementError(attempt busruntime.DeliveryAttempt, err error) error { + if busruntime.ClassifyAttempt(attempt, err) != busruntime.AttemptFailed { return err - }) + } + if errors.Is(err, backend.SkipRetry) { + return err + } + return errors.Join(err, backend.SkipRetry) } +// StartWorkers rejects restart while an earlier server instance is still draining. func (w *redisWorker) StartWorkers(ctx context.Context) error { if ctx != nil && ctx.Err() != nil { return ctx.Err() } w.mu.Lock() defer w.mu.Unlock() + if w.draining { + return queue.ErrQueuerShuttingDown + } if w.started { return nil } @@ -122,25 +180,32 @@ func (w *redisWorker) StartWorkers(ctx context.Context) error { return nil } +// Shutdown retains the server drain until completion so a caller can retry after its context expires. func (w *redisWorker) Shutdown(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } w.mu.Lock() - started := w.started - w.started = false - w.mu.Unlock() - - if !started { + if !w.started && !w.draining { + w.mu.Unlock() return nil } - done := make(chan struct{}) - go func() { - defer close(done) - w.server.Shutdown() - }() - - if ctx == nil { - <-done - return nil + if !w.draining { + w.draining = true + w.stopDone = make(chan struct{}) + done := w.stopDone + go func() { + w.server.Shutdown() + w.mu.Lock() + w.started = false + w.draining = false + w.stopDone = nil + w.mu.Unlock() + close(done) + }() } + done := w.stopDone + w.mu.Unlock() select { case <-done: diff --git a/driver/redisqueue/worker_redis_impl_test.go b/driver/redisqueue/worker_redis_impl_test.go index ba482a2..e7444e2 100644 --- a/driver/redisqueue/worker_redis_impl_test.go +++ b/driver/redisqueue/worker_redis_impl_test.go @@ -3,9 +3,11 @@ package redisqueue import ( "context" "errors" + "strconv" "testing" "github.com/goforj/queue" + "github.com/goforj/queue/busruntime" backend "github.com/hibiken/asynq" ) @@ -58,7 +60,7 @@ func TestRedisWorker_RegisterStartShutdownBranches(t *testing.T) { t.Fatalf("expected one start call, got %d", server.startCalls) } - if err := w.Shutdown(context.Background()); err != nil { + if err := w.Shutdown(nil); err != nil { t.Fatalf("shutdown failed: %v", err) } if err := w.Shutdown(context.Background()); err != nil { @@ -94,8 +96,23 @@ func TestRedisWorker_ShutdownHonorsContext(t *testing.T) { if err := w.Shutdown(ctx); !errors.Is(err, context.Canceled) { t.Fatalf("expected context canceled, got %v", err) } + if !w.started || !w.draining { + t.Fatalf("worker lost retryable drain state: started=%t draining=%t", w.started, w.draining) + } + if err := w.StartWorkers(context.Background()); !errors.Is(err, queue.ErrQueuerShuttingDown) { + t.Fatalf("start during drain error = %v, want ErrQueuerShuttingDown", err) + } close(server.shutdownCh) + if err := w.Shutdown(context.Background()); err != nil { + t.Fatalf("retry shutdown failed: %v", err) + } + if server.shutdownCalls != 1 { + t.Fatalf("server shutdown calls = %d, want 1", server.shutdownCalls) + } + if w.started || w.draining { + t.Fatalf("worker remained active after drain: started=%t draining=%t", w.started, w.draining) + } } func TestRedisWorker_ProcessEventsWithObserver(t *testing.T) { @@ -119,8 +136,8 @@ func TestRedisWorker_ProcessEventsWithObserver(t *testing.T) { if err := server.lastStartHandler.ProcessTask(context.Background(), backend.NewTask("job:fail", []byte("fail"))); err == nil { t.Fatal("expected failing task error") } - if len(events) != 5 { - t.Fatalf("expected 5 process events, got %d", len(events)) + if len(events) != 4 { + t.Fatalf("expected 4 process events, got %d", len(events)) } if events[0].Kind != queue.EventProcessStarted || events[1].Kind != queue.EventProcessSucceeded { t.Fatalf("unexpected first pair kinds: %s, %s", events[0].Kind, events[1].Kind) @@ -128,9 +145,6 @@ func TestRedisWorker_ProcessEventsWithObserver(t *testing.T) { if events[2].Kind != queue.EventProcessStarted || events[3].Kind != queue.EventProcessFailed { t.Fatalf("unexpected second pair kinds: %s, %s", events[2].Kind, events[3].Kind) } - if events[4].Kind != queue.EventProcessArchived { - t.Fatalf("unexpected terminal event kind: %s", events[4].Kind) - } for _, event := range events { if event.Driver != queue.DriverRedis { t.Fatalf("expected redis driver, got %q", event.Driver) @@ -148,8 +162,167 @@ func TestRedisWorker_ProcessEventsWithObserver(t *testing.T) { if events[1].Time.IsZero() || events[3].Time.IsZero() { t.Fatal("expected event timestamps to be set") } - if events[4].Err != nil { - t.Fatal("expected archived event error to be nil") +} + +// TestRedisWorker_PanicClosesObserverActive verifies native Redis telemetry +// finalizes the failed attempt before preserving Asynq's panic semantics. +func TestRedisWorker_PanicClosesObserverActive(t *testing.T) { + server := &serverStub{} + collector := queue.NewStatsCollector() + var events []queue.Event + observer := queue.MultiObserver( + collector, + queue.ObserverFunc(func(_ context.Context, event queue.Event) { + events = append(events, event) + }), + ) + w := newRedisWorker(server, backend.NewServeMux(), observer) + w.Register("job:panic", func(context.Context, queue.Job) error { + panic("redis panic") + }) + if err := w.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + task := backend.NewTaskWithHeaders("job:panic", nil, map[string]string{ + redisDriverJobMetadataHeader: `{"schema_version":1,"dispatch_id":"dsp_redis_panic","job_id":"job_redis_panic"}`, + }) + var recovered any + func() { + defer func() { recovered = recover() }() + _ = server.lastStartHandler.ProcessTask(context.Background(), task) + }() + if recovered != "redis panic" { + t.Fatalf("recovered panic = %#v, want original value", recovered) + } + if len(events) != 2 || events[0].Kind != queue.EventProcessStarted || events[1].Kind != queue.EventProcessFailed { + t.Fatalf("panic events = %+v, want process_started then process_failed", events) + } + if events[1].Err == nil || events[1].Err.Error() != "redis handler panicked: redis panic" { + t.Fatalf("panic failure error = %v, want stable diagnostic", events[1].Err) + } + counters, ok := collector.Snapshot().Queue("default") + if !ok { + t.Fatal("expected default queue counters") + } + if counters.Active != 0 || counters.Failed != 1 || counters.Processed != 0 { + t.Fatalf("panic counters = %+v, want active=0 failed=1 processed=0", counters) + } +} + +// TestRedisHandlerPanicErrorPreservesErrorIdentity verifies Redis panic +// telemetry wraps error values and formats non-error values deterministically. +func TestRedisHandlerPanicErrorPreservesErrorIdentity(t *testing.T) { + sentinel := errors.New("redis panic sentinel") + if err := redisHandlerPanicError(sentinel); !errors.Is(err, sentinel) { + t.Fatalf("error panic = %v, want wrapped sentinel", err) + } + if err := redisHandlerPanicError("value"); err == nil || err.Error() != "redis handler panicked: value" { + t.Fatalf("value panic = %v, want stable diagnostic", err) + } +} + +// TestObserveRedisAttemptStartEmitsRetryDelivery verifies Redis reports a numbered retry when Asynq delivers that attempt. +func TestObserveRedisAttemptStartEmitsRetryDelivery(t *testing.T) { + var events []queue.Event + observer := queue.ObserverFunc(func(_ context.Context, event queue.Event) { + events = append(events, event) + }) + observeRedisAttemptStart(context.Background(), observer, queue.Event{ + Driver: queue.DriverRedis, + JobType: "job:retry", + Attempt: 1, + MaxRetry: 3, + }) + if len(events) != 2 || events[0].Kind != queue.EventProcessRetried || events[1].Kind != queue.EventProcessStarted { + t.Fatalf("attempt start events = %+v, want retried then started", events) + } + if events[0].Attempt != 1 || events[1].Attempt != 1 { + t.Fatalf("attempt metadata changed: %+v", events) + } +} + +// TestRedisWorker_PermanentFailureIncludesSkipRetryWithoutObserver verifies terminal settlement does not depend on observability being enabled. +func TestRedisWorker_PermanentFailureIncludesSkipRetryWithoutObserver(t *testing.T) { + server := &serverStub{} + w := newRedisWorker(server, backend.NewServeMux(), nil) + cause := errors.New("invalid application payload") + w.Register("job:permanent", func(context.Context, queue.Job) error { + return busruntime.Permanent(cause) + }) + if err := w.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers failed: %v", err) + } + + err := server.lastStartHandler.ProcessTask(context.Background(), backend.NewTask("job:permanent", nil)) + if !errors.Is(err, cause) { + t.Fatalf("expected returned error to preserve cause, got %v", err) + } + if !errors.Is(err, backend.SkipRetry) { + t.Fatalf("expected permanent error to include asynq SkipRetry, got %v", err) + } +} + +// TestRedisSettlementErrorDecisions verifies every terminal application outcome consumes no reserved transport retry. +func TestRedisSettlementErrorDecisions(t *testing.T) { + cause := errors.New("handler failed") + + retryErr := redisSettlementError(busruntime.DeliveryAttempt{Number: 0, MaxRetry: 2}, cause) + if !errors.Is(retryErr, cause) || errors.Is(retryErr, backend.SkipRetry) { + t.Fatalf("retry settlement = %v", retryErr) + } + + exhaustedErr := redisSettlementError(busruntime.DeliveryAttempt{Number: 2, MaxRetry: 2}, cause) + if !errors.Is(exhaustedErr, cause) || !errors.Is(exhaustedErr, backend.SkipRetry) { + t.Fatalf("exhausted settlement = %v", exhaustedErr) + } + + permanentErr := redisSettlementError( + busruntime.DeliveryAttempt{Number: 0, MaxRetry: 2}, + busruntime.Permanent(cause), + ) + if !errors.Is(permanentErr, cause) || !errors.Is(permanentErr, backend.SkipRetry) { + t.Fatalf("permanent settlement = %v", permanentErr) + } + + uncommittedErr := redisSettlementError( + busruntime.DeliveryAttempt{Number: 2, MaxRetry: 2}, + busruntime.Uncommitted(cause), + ) + if !errors.Is(uncommittedErr, cause) || errors.Is(uncommittedErr, backend.SkipRetry) { + t.Fatalf("uncommitted settlement = %v", uncommittedErr) + } + + skipRetryErr := redisSettlementError( + busruntime.DeliveryAttempt{Number: 2, MaxRetry: 2}, + errors.Join(cause, backend.SkipRetry), + ) + if !errors.Is(skipRetryErr, cause) || !errors.Is(skipRetryErr, backend.SkipRetry) { + t.Fatalf("existing skip-retry settlement = %v", skipRetryErr) + } +} + +// TestRedisApplicationMaxRetry verifies only a valid reserve header changes the handler-visible retry budget. +func TestRedisApplicationMaxRetry(t *testing.T) { + tests := []struct { + name string + task *backend.Task + transportMax int + want int + }{ + {name: "nil task", task: nil, transportMax: 4, want: 4}, + {name: "legacy task", task: backend.NewTask("job", nil), transportMax: 3, want: 3}, + {name: "reserved task", task: backend.NewTaskWithHeaders("job", nil, map[string]string{redisApplicationMaxRetryHeader: "2"}), transportMax: 3, want: 2}, + {name: "mismatched reserve", task: backend.NewTaskWithHeaders("job", nil, map[string]string{redisApplicationMaxRetryHeader: "2"}), transportMax: 4, want: 4}, + {name: "malformed reserve", task: backend.NewTaskWithHeaders("job", nil, map[string]string{redisApplicationMaxRetryHeader: "bad"}), transportMax: 3, want: 3}, + {name: "negative reserve", task: backend.NewTaskWithHeaders("job", nil, map[string]string{redisApplicationMaxRetryHeader: "-1"}), transportMax: 0, want: 0}, + {name: "overflow reserve", task: backend.NewTaskWithHeaders("job", nil, map[string]string{redisApplicationMaxRetryHeader: strconv.Itoa(int(^uint(0) >> 1))}), transportMax: int(^uint(0) >> 1), want: int(^uint(0) >> 1)}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := redisApplicationMaxRetry(test.task, test.transportMax); got != test.want { + t.Fatalf("application max retry = %d, want %d", got, test.want) + } + }) } } @@ -164,7 +337,7 @@ func TestRedisWorker_ProcessEventsUnwrapBusEnvelopeJobType(t *testing.T) { t.Fatalf("start workers failed: %v", err) } - payload := []byte(`{"job":{"type":"monitoring:check"}}`) + payload := []byte(`{"schema_version":1,"dispatch_id":"dsp_redis","job_id":"job_redis","chain_id":"chn_redis","job":{"type":"monitoring:check"}}`) if err := server.lastStartHandler.ProcessTask(context.Background(), backend.NewTask("bus:job", payload)); err != nil { t.Fatalf("process task failed: %v", err) } @@ -175,6 +348,9 @@ func TestRedisWorker_ProcessEventsUnwrapBusEnvelopeJobType(t *testing.T) { if event.JobType != "monitoring:check" { t.Fatalf("expected unwrapped observed job type, got %q", event.JobType) } + if event.DispatchID != "dsp_redis" || event.JobID != "job_redis" || event.ChainID != "chn_redis" { + t.Fatalf("expected correlated redis event, got %+v", event) + } } } @@ -189,14 +365,14 @@ func TestRedisWorker_NoObserverFastPath(t *testing.T) { t.Fatalf("expected job type job:plain, got %q", job.Type) } opts := queue.DriverOptions(job) - if opts.QueueName != "" { - t.Fatalf("expected empty queue name in no-observer path, got %q", opts.QueueName) + if opts.QueueName != "default" { + t.Fatalf("expected normalized queue name in no-observer path, got %q", opts.QueueName) } if opts.Attempt != 0 { t.Fatalf("expected zero attempt in no-observer path, got %d", opts.Attempt) } - if opts.MaxRetry != nil { - t.Fatalf("expected nil max retry in no-observer path, got %v", *opts.MaxRetry) + if opts.MaxRetry == nil || *opts.MaxRetry != 0 { + t.Fatalf("expected zero max retry in no-observer path, got %v", opts.MaxRetry) } return nil }) @@ -219,6 +395,7 @@ func TestRedisWorker_ObserverSeesDecoratedContext(t *testing.T) { var observed []string var handled []string + decoratorCalls := 0 observer := queue.ObserverFunc(func(ctx context.Context, event queue.Event) { if event.Kind != queue.EventProcessStarted && event.Kind != queue.EventProcessSucceeded { return @@ -228,6 +405,7 @@ func TestRedisWorker_ObserverSeesDecoratedContext(t *testing.T) { }) w := newRedisWorker(server, backend.NewServeMux(), observer) w.SetHandlerContextDecorator(func(ctx context.Context) context.Context { + decoratorCalls++ return context.WithValue(ctx, key, want) }) @@ -254,4 +432,59 @@ func TestRedisWorker_ObserverSeesDecoratedContext(t *testing.T) { if len(handled) != 1 || handled[0] != want { t.Fatalf("expected handler to see %q, got %#v", want, handled) } + if decoratorCalls != 1 { + t.Fatalf("expected decorator called once, got %d", decoratorCalls) + } +} + +// TestRedisWorkerReplacementDecoratorPreservesTransportDelivery verifies +// replacement application context cannot erase Asynq retry and queue facts. +func TestRedisWorkerReplacementDecoratorPreservesTransportDelivery(t *testing.T) { + type contextKey struct{} + key := contextKey{} + const decoratedValue = "replacement" + wantContextAttempt := busruntime.DeliveryAttempt{Number: 7, MaxRetry: 9} + var events []queue.Event + w := newRedisWorker(&serverStub{}, backend.NewServeMux(), queue.ObserverFunc(func(_ context.Context, event queue.Event) { + events = append(events, event) + })) + w.SetHandlerContextDecorator(func(context.Context) context.Context { + return context.WithValue(context.Background(), key, decoratedValue) + }) + handlerCalls := 0 + handler := func(ctx context.Context, job queue.Job) error { + handlerCalls++ + if got := ctx.Value(key); got != decoratedValue { + t.Fatalf("decorated context value = %v, want %q", got, decoratedValue) + } + if got, ok := busruntime.DeliveryAttemptFromContext(ctx); !ok || got != wantContextAttempt { + t.Fatalf("preserved context attempt = %+v, %t; want %+v", got, ok, wantContextAttempt) + } + opts := queue.DriverOptions(job) + if opts.Attempt != 2 || opts.MaxRetry == nil || *opts.MaxRetry != 4 || opts.QueueName != "critical" { + t.Fatalf("delivered transport options = attempt:%d max:%v queue:%q", opts.Attempt, opts.MaxRetry, opts.QueueName) + } + return nil + } + ctx := busruntime.WithDeliveryAttempt(context.Background(), wantContextAttempt) + err := w.processTask(ctx, backend.NewTask("job:replacement", []byte("ok")), handler, redisTransportDelivery{ + attempt: 2, + maxRetry: 4, + queue: "critical", + }) + if err != nil { + t.Fatalf("process replacement task: %v", err) + } + if handlerCalls != 1 { + t.Fatalf("handler calls = %d, want 1", handlerCalls) + } + wantKinds := []queue.EventKind{queue.EventProcessRetried, queue.EventProcessStarted, queue.EventProcessSucceeded} + if len(events) != len(wantKinds) { + t.Fatalf("events = %+v, want %d retry/start/success facts", events, len(wantKinds)) + } + for i, event := range events { + if event.Kind != wantKinds[i] || event.Attempt != 2 || event.MaxRetry != 4 || event.Queue != "critical" { + t.Fatalf("event[%d] = %+v, want kind:%q attempt:2 max:4 queue:critical", i, event, wantKinds[i]) + } + } } diff --git a/driver/sqlitequeue/sqlitequeue.go b/driver/sqlitequeue/sqlitequeue.go index b8b121a..64a3341 100644 --- a/driver/sqlitequeue/sqlitequeue.go +++ b/driver/sqlitequeue/sqlitequeue.go @@ -10,10 +10,12 @@ import ( _ "modernc.org/sqlite" ) +// Config configures the SQLite queue driver. type Config struct { queueconfig.DriverBaseConfig DB *sql.DB DSN string + DisableAutoMigrate bool ProcessingRecoveryGrace time.Duration ProcessingLeaseNoTimeout time.Duration } @@ -48,6 +50,7 @@ func New(dsn string, opts ...queue.Option) (*queue.Queue, error) { // }, // DB: nil, // optional; provide *sql.DB instead of DSN // DSN: "file:queue.db?_busy_timeout=5000", // optional if DB is set +// DisableAutoMigrate: false, // set true when schema migrations are managed externally // ProcessingRecoveryGrace: 2 * time.Second, // default if <=0: 2s // ProcessingLeaseNoTimeout: 5 * time.Minute, // default if <=0: 5m // }, @@ -62,6 +65,7 @@ func NewWithConfig(cfg Config, opts ...queue.Option) (*queue.Queue, error) { DriverBaseConfig: cfg.DriverBaseConfig, DB: cfg.DB, DSN: cfg.DSN, + DisableAutoMigrate: cfg.DisableAutoMigrate, ProcessingRecoveryGrace: cfg.ProcessingRecoveryGrace, ProcessingLeaseNoTimeout: cfg.ProcessingLeaseNoTimeout, }, opts...) diff --git a/driver/sqlitequeue/sqlitequeue_metadata_test.go b/driver/sqlitequeue/sqlitequeue_metadata_test.go new file mode 100644 index 0000000..e7fdf63 --- /dev/null +++ b/driver/sqlitequeue/sqlitequeue_metadata_test.go @@ -0,0 +1,344 @@ +package sqlitequeue + +import ( + "context" + "database/sql" + "errors" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/goforj/queue" + "github.com/goforj/queue/driver/sqlqueuecore" +) + +// TestSQLiteDirectMetadataRoundTrip verifies a fresh SQL schema retains direct +// correlation independently from the application type and payload. +func TestSQLiteDirectMetadataRoundTrip(t *testing.T) { + db := openSQLiteMetadataTestDB(t) + backend, err := sqlqueuecore.New(queue.DatabaseConfig{ + DB: db, + DriverName: "sqlite", + DefaultQueue: "default", + Workers: 1, + PollInterval: 5 * time.Millisecond, + }) + if err != nil { + t.Fatalf("new SQL backend: %v", err) + } + t.Cleanup(func() { _ = backend.Shutdown(context.Background()) }) + + received := make(chan queue.Job, 1) + backend.Register("reports:build", func(_ context.Context, job queue.Job) error { + received <- job + return nil + }) + if err := backend.StartWorkers(context.Background()); err != nil { + t.Fatalf("start SQL workers: %v", err) + } + + want := queue.DriverJobMetadata{ + SchemaVersion: queue.DriverJobMetadataVersion, + DispatchID: "dsp_sqlite", + JobID: "job_sqlite", + Queue: "default", + } + job := queue.DriverWithMetadata( + queue.NewJob("reports:build").Payload([]byte(`{"id":7}`)).OnQueue("default").Retry(2), + want, + ) + if err := backend.Dispatch(context.Background(), job); err != nil { + t.Fatalf("dispatch direct SQL job: %v", err) + } + got := waitForSQLiteMetadataJob(t, received) + if metadata := queue.DriverMetadata(got); metadata != want { + t.Fatalf("delivered metadata = %+v, want %+v", metadata, want) + } + if got.Type != job.Type || string(got.PayloadBytes()) != string(job.PayloadBytes()) { + t.Fatalf("delivered job = type:%q payload:%q, want type:%q payload:%q", got.Type, got.PayloadBytes(), job.Type, job.PayloadBytes()) + } +} + +// TestSQLiteDirectMetadataSurvivesRetry verifies SQL state transitions retain +// one logical job identity while only the physical attempt number advances. +func TestSQLiteDirectMetadataSurvivesRetry(t *testing.T) { + db := openSQLiteMetadataTestDB(t) + backend, err := sqlqueuecore.New(queue.DatabaseConfig{ + DB: db, + DriverName: "sqlite", + DefaultQueue: "default", + Workers: 1, + PollInterval: 5 * time.Millisecond, + }) + if err != nil { + t.Fatalf("new SQL backend: %v", err) + } + t.Cleanup(func() { _ = backend.Shutdown(context.Background()) }) + + deliveries := make(chan queue.Job, 2) + backend.Register("reports:retry", func(_ context.Context, job queue.Job) error { + deliveries <- job + if queue.DriverOptions(job).Attempt == 0 { + return errors.New("retry once") + } + return nil + }) + if err := backend.StartWorkers(context.Background()); err != nil { + t.Fatalf("start SQL workers: %v", err) + } + + want := queue.DriverJobMetadata{ + SchemaVersion: queue.DriverJobMetadataVersion, + DispatchID: "dsp_retry", + JobID: "job_retry", + Queue: "default", + } + job := queue.DriverWithMetadata(queue.NewJob("reports:retry").OnQueue("default").Retry(1), want) + if err := backend.Dispatch(context.Background(), job); err != nil { + t.Fatalf("dispatch retrying SQL job: %v", err) + } + first := waitForSQLiteMetadataJob(t, deliveries) + second := waitForSQLiteMetadataJob(t, deliveries) + if queue.DriverOptions(first).Attempt != 0 || queue.DriverOptions(second).Attempt != 1 { + t.Fatalf("delivery attempts = %d, %d; want 0, 1", queue.DriverOptions(first).Attempt, queue.DriverOptions(second).Attempt) + } + if firstMetadata, secondMetadata := queue.DriverMetadata(first), queue.DriverMetadata(second); firstMetadata != want || secondMetadata != want { + t.Fatalf("retry metadata = first:%+v second:%+v, want %+v", firstMetadata, secondMetadata, want) + } +} + +// TestSQLiteMetadataMigrationReadsLegacyAndUntrustedRows verifies additive +// migration preserves old envelopes while untrusted metadata cannot spoof IDs. +func TestSQLiteMetadataMigrationReadsLegacyAndUntrustedRows(t *testing.T) { + db := openSQLiteMetadataTestDB(t) + createLegacySQLiteQueueSchema(t, db) + now := time.Now().UnixMilli() + legacyPayload := []byte(`{"schema_version":1,"dispatch_id":"dsp_legacy","job_id":"job_legacy","job":{"type":"reports:legacy","payload":"eyJpZCI6MX0="}}`) + insertLegacySQLiteQueueJob(t, db, "bus:job", legacyPayload, now) + + backend, err := sqlqueuecore.New(queue.DatabaseConfig{ + DB: db, + DriverName: "sqlite", + DefaultQueue: "default", + Workers: 1, + PollInterval: 5 * time.Millisecond, + }) + if err != nil { + t.Fatalf("new SQL backend: %v", err) + } + t.Cleanup(func() { _ = backend.Shutdown(context.Background()) }) + + legacy := make(chan queue.Job, 1) + backend.Register("bus:job", func(_ context.Context, job queue.Job) error { + legacy <- job + return nil + }) + if err := backend.StartWorkers(context.Background()); err != nil { + t.Fatalf("start SQL workers: %v", err) + } + if !sqliteQueueColumnExists(t, db, "metadata_json") { + t.Fatal("additive migration did not create metadata_json") + } + legacyJob := waitForSQLiteMetadataJob(t, legacy) + legacyObserved := queue.ResolveObservedJobMetadataFromJob(legacyJob) + if legacyObserved.JobType != "reports:legacy" || legacyObserved.DispatchID != "dsp_legacy" || legacyObserved.JobID != "job_legacy" { + t.Fatalf("legacy envelope fallback = %+v", legacyObserved) + } + + untrusted := make(chan queue.Job, 2) + for _, jobType := range []string{"reports:malformed", "reports:unknown"} { + backend.Register(jobType, func(_ context.Context, job queue.Job) error { + untrusted <- job + return nil + }) + } + insertSQLiteQueueJobWithMetadata(t, db, "reports:malformed", []byte(`{"id":2}`), `{`, now+1) + insertSQLiteQueueJobWithMetadata(t, db, "reports:unknown", []byte(`{"id":3}`), `{"schema_version":99,"dispatch_id":"spoofed","job_id":"spoofed"}`, now+2) + for range 2 { + job := waitForSQLiteMetadataJob(t, untrusted) + metadata := queue.ResolveObservedJobMetadataFromJob(job) + if metadata.DispatchID != "" || metadata.JobID != "" || queue.DriverMetadata(job).SchemaVersion != 0 { + t.Fatalf("untrusted row produced correlation: job=%q metadata=%+v", job.Type, metadata) + } + } +} + +// TestSQLiteCallerManagedSchemaRequiresMetadataColumn verifies disabling +// migration fails before polling a schema that cannot retain direct IDs. +func TestSQLiteCallerManagedSchemaRequiresMetadataColumn(t *testing.T) { + db := openSQLiteMetadataTestDB(t) + createLegacySQLiteQueueSchema(t, db) + backend, err := sqlqueuecore.New(queue.DatabaseConfig{ + DB: db, + DriverName: "sqlite", + DefaultQueue: "default", + Workers: 1, + DisableAutoMigrate: true, + }) + if err != nil { + t.Fatalf("new SQL backend: %v", err) + } + backend.Register("reports:build", func(context.Context, queue.Job) error { return nil }) + err = backend.StartWorkers(context.Background()) + if err == nil || !strings.Contains(err.Error(), "missing required metadata_json column") { + t.Fatalf("start with legacy caller-managed schema = %v, want explicit metadata_json error", err) + } +} + +// TestSQLiteMetadataMigrationAllowsConcurrentStartup verifies two worker +// runtimes can race the additive migration without poisoning either lifecycle. +func TestSQLiteMetadataMigrationAllowsConcurrentStartup(t *testing.T) { + db := openSQLiteMetadataTestDB(t) + createLegacySQLiteQueueSchema(t, db) + config := queue.DatabaseConfig{ + DB: db, + DriverName: "sqlite", + DefaultQueue: "default", + Workers: 1, + PollInterval: 5 * time.Millisecond, + } + first, err := sqlqueuecore.New(config) + if err != nil { + t.Fatalf("new first SQL backend: %v", err) + } + second, err := sqlqueuecore.New(config) + if err != nil { + t.Fatalf("new second SQL backend: %v", err) + } + t.Cleanup(func() { + _ = first.Shutdown(context.Background()) + _ = second.Shutdown(context.Background()) + }) + + start := make(chan struct{}) + results := make(chan error, 2) + for _, backend := range []interface { + StartWorkers(context.Context) error + }{first, second} { + go func() { + <-start + results <- backend.StartWorkers(context.Background()) + }() + } + close(start) + for range 2 { + if err := <-results; err != nil { + t.Fatalf("concurrent worker startup: %v", err) + } + } + if !sqliteQueueColumnExists(t, db, "metadata_json") { + t.Fatal("concurrent startup did not install metadata_json") + } +} + +// openSQLiteMetadataTestDB opens one isolated on-disk database so multiple SQL +// connections observe the same migration and durable rows. +func openSQLiteMetadataTestDB(t *testing.T) *sql.DB { + t.Helper() + dsn := "file:" + filepath.Join(t.TempDir(), "metadata.db") + "?_busy_timeout=5000" + db, err := sql.Open("sqlite", dsn) + if err != nil { + t.Fatalf("open SQLite database: %v", err) + } + t.Cleanup(func() { + if err := db.Close(); err != nil && !errors.Is(err, sql.ErrConnDone) { + t.Errorf("close SQLite database: %v", err) + } + }) + return db +} + +// createLegacySQLiteQueueSchema creates the last compatible queue table shape +// before direct-delivery metadata was stored separately. +func createLegacySQLiteQueueSchema(t *testing.T, db *sql.DB) { + t.Helper() + _, err := db.Exec(`CREATE TABLE queue_jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + queue_name TEXT NOT NULL, + job_type TEXT NOT NULL, + payload BLOB NOT NULL, + timeout_seconds INTEGER NULL, + max_retry INTEGER NOT NULL DEFAULT 0, + backoff_millis INTEGER NOT NULL DEFAULT 0, + attempt INTEGER NOT NULL DEFAULT 0, + available_at INTEGER NOT NULL, + processing_started_at INTEGER NULL, + processing_token TEXT NULL, + last_error TEXT NULL, + state TEXT NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + )`) + if err != nil { + t.Fatalf("create legacy queue schema: %v", err) + } +} + +// insertLegacySQLiteQueueJob inserts one pre-metadata row using only columns an +// older producer knew how to populate. +func insertLegacySQLiteQueueJob(t *testing.T, db *sql.DB, jobType string, payload []byte, now int64) { + t.Helper() + _, err := db.Exec(`INSERT INTO queue_jobs + (queue_name, job_type, payload, timeout_seconds, max_retry, backoff_millis, attempt, available_at, state, created_at, updated_at) + VALUES ('default', ?, ?, NULL, 0, 0, 0, ?, 'pending', ?, ?)`, jobType, payload, now, now, now) + if err != nil { + t.Fatalf("insert legacy queue row: %v", err) + } +} + +// insertSQLiteQueueJobWithMetadata inserts a raw metadata fixture so malformed +// and future protocol versions are tested without public helpers filtering them. +func insertSQLiteQueueJobWithMetadata(t *testing.T, db *sql.DB, jobType string, payload []byte, metadata string, now int64) { + t.Helper() + _, err := db.Exec(`INSERT INTO queue_jobs + (queue_name, job_type, payload, metadata_json, timeout_seconds, max_retry, backoff_millis, attempt, available_at, state, created_at, updated_at) + VALUES ('default', ?, ?, ?, NULL, 0, 0, 0, ?, 'pending', ?, ?)`, jobType, payload, metadata, now, now, now) + if err != nil { + t.Fatalf("insert raw metadata queue row: %v", err) + } +} + +// sqliteQueueColumnExists inspects the migrated table without depending on +// implementation-private schema helpers. +func sqliteQueueColumnExists(t *testing.T, db *sql.DB, columnName string) bool { + t.Helper() + rows, err := db.Query(`PRAGMA table_info(queue_jobs)`) + if err != nil { + t.Fatalf("inspect SQLite queue columns: %v", err) + } + defer rows.Close() + for rows.Next() { + var ( + columnID int + name string + columnType string + notNull int + defaultValue sql.NullString + primaryKey int + ) + if err := rows.Scan(&columnID, &name, &columnType, ¬Null, &defaultValue, &primaryKey); err != nil { + t.Fatalf("scan SQLite queue column: %v", err) + } + if name == columnName { + return true + } + } + if err := rows.Err(); err != nil { + t.Fatalf("iterate SQLite queue columns: %v", err) + } + return false +} + +// waitForSQLiteMetadataJob waits for one polled delivery without allowing a +// broken migration to hang the test suite. +func waitForSQLiteMetadataJob(t *testing.T, jobs <-chan queue.Job) queue.Job { + t.Helper() + select { + case job := <-jobs: + return job + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for SQLite delivery") + return queue.Job{} + } +} diff --git a/driver/sqlqueuecore/database_test_helpers_test.go b/driver/sqlqueuecore/database_test_helpers_test.go new file mode 100644 index 0000000..79ef6b0 --- /dev/null +++ b/driver/sqlqueuecore/database_test_helpers_test.go @@ -0,0 +1,149 @@ +package sqlqueuecore + +import ( + "context" + "database/sql" + "database/sql/driver" + "errors" + "io" +) + +type databaseConnectorStub struct { + conn *databaseConnStub +} + +type databaseDriverStub struct { + conn *databaseConnStub +} + +type databaseConnStub struct { + exec func(context.Context, string, []driver.NamedValue) (driver.Result, error) + query func(context.Context, string, []driver.NamedValue) (driver.Rows, error) + beginErr error + commitErr error + rollbackErr error + rollbackCalls int + pingErr error + closeErr error + closeCalls int +} + +type databaseTxStub struct { + conn *databaseConnStub +} + +type databaseRowsStub struct { + columns []string + values [][]driver.Value + err error + index int +} + +// Connect exposes the scripted connection through database/sql without registering a process-global driver name. +func (c databaseConnectorStub) Connect(context.Context) (driver.Conn, error) { + return c.conn, nil +} + +// Driver returns the connector's fallback driver implementation. +func (c databaseConnectorStub) Driver() driver.Driver { + return databaseDriverStub{conn: c.conn} +} + +// Open returns the scripted connection when database/sql uses the fallback driver path. +func (d databaseDriverStub) Open(string) (driver.Conn, error) { + return d.conn, nil +} + +// Prepare rejects fallback statement preparation because every test scripts context-aware operations directly. +func (c *databaseConnStub) Prepare(string) (driver.Stmt, error) { + return nil, errors.New("unexpected database statement preparation") +} + +// Close records database ownership behavior without invalidating the reusable script fixture. +func (c *databaseConnStub) Close() error { + c.closeCalls++ + return c.closeErr +} + +// Begin starts a transaction through the context-aware implementation. +func (c *databaseConnStub) Begin() (driver.Tx, error) { + return c.BeginTx(context.Background(), driver.TxOptions{}) +} + +// BeginTx returns a transaction that delegates completion behavior to the scripted connection. +func (c *databaseConnStub) BeginTx(context.Context, driver.TxOptions) (driver.Tx, error) { + if c.beginErr != nil { + return nil, c.beginErr + } + return databaseTxStub{conn: c}, nil +} + +// Ping returns the configured connectivity result. +func (c *databaseConnStub) Ping(context.Context) error { + return c.pingErr +} + +// ExecContext delegates execution to the test-specific script. +func (c *databaseConnStub) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + if c.exec == nil { + return nil, errors.New("unexpected database execution") + } + return c.exec(ctx, query, args) +} + +// QueryContext delegates queries to the test-specific script. +func (c *databaseConnStub) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + if c.query == nil { + return nil, errors.New("unexpected database query") + } + return c.query(ctx, query, args) +} + +// Commit returns the configured transaction completion result. +func (t databaseTxStub) Commit() error { + return t.conn.commitErr +} + +// Rollback returns the configured transaction rollback result. +func (t databaseTxStub) Rollback() error { + t.conn.rollbackCalls++ + return t.conn.rollbackErr +} + +// Columns returns the shape expected by Scan. +func (r *databaseRowsStub) Columns() []string { + return r.columns +} + +// Close releases no resources because all rows are in-memory fixtures. +func (r *databaseRowsStub) Close() error { + return nil +} + +// Next returns scripted rows followed by an optional terminal iteration error. +func (r *databaseRowsStub) Next(dest []driver.Value) error { + if r.index < len(r.values) { + copy(dest, r.values[r.index]) + r.index++ + return nil + } + if r.err != nil { + err := r.err + r.err = nil + return err + } + return io.EOF +} + +// newDatabaseStub opens a database/sql handle backed by one deterministic scripted connection. +func newDatabaseStub(conn *databaseConnStub) *sql.DB { + return sql.OpenDB(databaseConnectorStub{conn: conn}) +} + +// databaseCountRows returns one integer result suitable for COUNT queries. +func databaseCountRows(count int64) driver.Rows { + return &databaseRowsStub{ + columns: []string{"count"}, + values: [][]driver.Value{{count}}, + } +} diff --git a/driver/sqlqueuecore/queue_database_failure_test.go b/driver/sqlqueuecore/queue_database_failure_test.go new file mode 100644 index 0000000..1f1edf0 --- /dev/null +++ b/driver/sqlqueuecore/queue_database_failure_test.go @@ -0,0 +1,1110 @@ +package sqlqueuecore + +import ( + "context" + "database/sql/driver" + "errors" + "fmt" + "strings" + "testing" + "time" + + "github.com/goforj/queue" + "github.com/goforj/queue/busruntime" +) + +// TestDatabaseDispatchPropagatesUniqueTransactionFailures verifies a unique +// dispatch never reports success when its transaction or database-clock claim +// cannot start. +func TestDatabaseDispatchPropagatesUniqueTransactionFailures(t *testing.T) { + beginErr := errors.New("begin unavailable") + clockErr := errors.New("clock unavailable") + tests := []struct { + name string + conn *databaseConnStub + want error + }{ + { + name: "begin transaction", + conn: &databaseConnStub{beginErr: beginErr}, + want: beginErr, + }, + { + name: "read database clock", + conn: &databaseConnStub{ + query: func(context.Context, string, []driver.NamedValue) (driver.Rows, error) { + return nil, clockErr + }, + }, + want: clockErr, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + db := newDatabaseStub(test.conn) + defer db.Close() + database := &databaseQueue{ + cfg: localDatabaseConfig{DriverName: "mysql", DefaultQueue: "default"}, + db: db, + } + job := queue.NewJob("reports:build"). + Payload([]byte(`{"report_id":42}`)). + OnQueue("default"). + UniqueFor(time.Second) + if err := database.Dispatch(context.Background(), job); !errors.Is(err, test.want) { + t.Fatalf("unique dispatch error = %v, want %v", err, test.want) + } + }) + } +} + +// TestLegacyUniqueJobKeyGoldenVector pins the exact persisted identity written +// by SQL producers before the canonical versioned key was introduced. +func TestLegacyUniqueJobKeyGoldenVector(t *testing.T) { + job := queue.NewJob("reports:build").Payload([]byte(`{"report_id":42}`)) + const want = "91631af060eb66e8a1fa0473091ca85f7bffb2b29d6552c26344eb011697e6c5" + if got := legacyUniqueJobKey(job, "default"); got != want { + t.Fatalf("legacy SQL unique key = %q, want %q", got, want) + } + if want == uniqueJobKey(job, "default") { + t.Fatal("legacy and canonical SQL unique keys unexpectedly match") + } +} + +// TestDatabaseDispatchRollsBackDualUniqueClaimFailures verifies every SQL +// dialect abandons the historical claim when either identity or queue-row +// acquisition cannot commit. +func TestDatabaseDispatchRollsBackDualUniqueClaimFailures(t *testing.T) { + job := queue.NewJob("reports:build"). + Payload([]byte(`{"report_id":42}`)). + OnQueue("default"). + UniqueFor(time.Minute) + legacyKey := legacyUniqueJobKey(job, "default") + canonicalKey := uniqueJobKey(job, "default") + claimErr := errors.New("claim unavailable") + queueErr := errors.New("queue insert unavailable") + + scenarios := []struct { + name string + insertErrKey string + updateErrKey string + collisionKey string + queueErr error + want error + wantQueue bool + }{ + {name: "legacy insert failure", insertErrKey: legacyKey, want: claimErr}, + {name: "legacy expiry update failure", updateErrKey: legacyKey, want: claimErr}, + {name: "legacy outstanding", collisionKey: legacyKey, want: queue.ErrDuplicate}, + {name: "canonical insert failure", insertErrKey: canonicalKey, want: claimErr}, + {name: "canonical expiry update failure", updateErrKey: canonicalKey, want: claimErr}, + {name: "canonical outstanding", collisionKey: canonicalKey, want: queue.ErrDuplicate}, + {name: "queue insert failure", queueErr: queueErr, want: queueErr, wantQueue: true}, + } + drivers := []string{"sqlite", "postgres", "mysql"} + + for _, driverName := range drivers { + for _, scenario := range scenarios { + t.Run(driverName+"/"+scenario.name, func(t *testing.T) { + queueInsert := false + claimed := make(map[string]bool) + conn := &databaseConnStub{ + query: func(context.Context, string, []driver.NamedValue) (driver.Rows, error) { + return databaseCountRows(1_000), nil + }, + exec: func(_ context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + if strings.Contains(query, "INSERT INTO queue_jobs") { + queueInsert = true + if scenario.queueErr != nil { + return nil, scenario.queueErr + } + return driver.RowsAffected(1), nil + } + if !strings.Contains(query, "queue_unique_locks") { + return nil, fmt.Errorf("unexpected transaction query: %s", query) + } + if driverName == "mysql" && strings.Contains(query, "INSERT") && !strings.Contains(query, "INSERT IGNORE") { + return nil, fmt.Errorf("mysql uniqueness insert did not ignore conflicts: %s", query) + } + if driverName != "mysql" && strings.Contains(query, "INSERT") && !strings.Contains(query, "ON CONFLICT") { + return nil, fmt.Errorf("%s uniqueness insert did not ignore conflicts: %s", driverName, query) + } + + keyIndex := 0 + updating := strings.Contains(query, "UPDATE queue_unique_locks") + if updating { + keyIndex = 1 + } + key, ok := args[keyIndex].Value.(string) + if !ok { + return nil, fmt.Errorf("uniqueness key argument = %#v", args[keyIndex].Value) + } + if updating { + if key == scenario.updateErrKey { + return nil, claimErr + } + if key == scenario.collisionKey { + return driver.RowsAffected(0), nil + } + claimed[key] = true + return driver.RowsAffected(1), nil + } + if key == scenario.insertErrKey { + return nil, claimErr + } + if key == scenario.updateErrKey || key == scenario.collisionKey { + return driver.RowsAffected(0), nil + } + claimed[key] = true + return driver.RowsAffected(1), nil + }, + } + db := newDatabaseStub(conn) + defer db.Close() + database := &databaseQueue{ + cfg: localDatabaseConfig{DriverName: driverName, DefaultQueue: "default"}, + db: db, + } + + err := database.Dispatch(context.Background(), job) + if !errors.Is(err, scenario.want) { + t.Fatalf("dual unique dispatch error = %v, want %v", err, scenario.want) + } + if queueInsert != scenario.wantQueue { + t.Fatalf("queue insert attempted = %t, want %t", queueInsert, scenario.wantQueue) + } + if conn.rollbackCalls != 1 { + t.Fatalf("transaction rollback calls = %d, want 1", conn.rollbackCalls) + } + canonicalPhase := scenario.insertErrKey == canonicalKey || + scenario.updateErrKey == canonicalKey || + scenario.collisionKey == canonicalKey || + scenario.queueErr != nil + if canonicalPhase && !claimed[legacyKey] { + t.Fatal("canonical phase began without first claiming the legacy identity") + } + }) + } + } +} + +// TestDatabaseClaimRejectsAmbiguousUpdateResults verifies a worker rolls back +// when the database cannot prove that exactly one pending row was fenced. +func TestDatabaseClaimRejectsAmbiguousUpdateResults(t *testing.T) { + rowsErr := errors.New("rows affected unavailable") + tests := []struct { + name string + result driver.Result + wantErr error + want string + }{ + { + name: "rows affected failure", + result: databaseResultStub{err: rowsErr}, + wantErr: rowsErr, + want: rowsErr.Error(), + }, + { + name: "multiple rows affected", + result: databaseResultStub{rows: 2}, + want: "database claim affected 2 rows, want 1", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + execCalls := 0 + conn := &databaseConnStub{ + exec: func(_ context.Context, _ string, _ []driver.NamedValue) (driver.Result, error) { + execCalls++ + if execCalls == 1 { + return driver.RowsAffected(0), nil + } + return test.result, nil + }, + query: func(context.Context, string, []driver.NamedValue) (driver.Rows, error) { + return &databaseRowsStub{ + columns: []string{ + "id", "queue_name", "job_type", "payload", "metadata_json", + "timeout_seconds", "max_retry", "backoff_millis", "attempt", "processing_token", + }, + values: [][]driver.Value{{ + int64(42), "default", "reports:build", []byte(`{"report_id":42}`), nil, + nil, int64(0), int64(0), int64(0), nil, + }}, + }, nil + }, + } + db := newDatabaseStub(conn) + defer db.Close() + database := &databaseQueue{ + cfg: localDatabaseConfig{ + DriverName: "mysql", + DefaultQueue: "default", + }, + db: db, + } + + job, err := database.claimOne(context.Background()) + if job != nil || err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("ambiguous claim = (%+v, %v), want nil job and %q", job, err, test.want) + } + if test.wantErr != nil && !errors.Is(err, test.wantErr) { + t.Fatalf("ambiguous claim error = %v, want wrapped %v", err, test.wantErr) + } + if conn.rollbackCalls != 1 { + t.Fatalf("claim rollback calls = %d, want 1", conn.rollbackCalls) + } + }) + } +} + +// TestDatabaseSettlementFailureIncludesLineageRepairError verifies telemetry +// retains both the original finalization failure and a failed recovery repair. +func TestDatabaseSettlementFailureIncludesLineageRepairError(t *testing.T) { + settlementErr := errors.New("settlement unavailable") + var events []queue.Event + var observedCtx context.Context + database := &databaseQueue{ + observer: queue.ObserverFunc(func(ctx context.Context, event queue.Event) { + observedCtx = ctx + events = append(events, event) + }), + } + job := &dbJob{ + id: 7, + jobType: "reports:build", + queueName: "critical", + recovered: true, + recoveryToken: "malformed", + processingToken: strings.Repeat("b", databaseProcessingTokenBytes*2), + } + + settlementCtx, _ := busruntime.WithDeliverySettlement(context.Background()) + wantIdentity, ok := busruntime.DeliverySettlementIdentityFromContext(settlementCtx) + if !ok { + t.Fatal("settlement context did not expose an identity") + } + database.handleSettlementFailure(settlementCtx, job, settlementErr) + if len(events) != 1 { + t.Fatalf("settlement failure events = %d, want 1", len(events)) + } + event := events[0] + if event.Kind != queue.EventSettlementFailed || !errors.Is(event.Err, settlementErr) { + t.Fatalf("settlement event = %+v, want original failure", event) + } + if !strings.Contains(event.Err.Error(), "restore recovered database settlement lineage") || + !strings.Contains(event.Err.Error(), "malformed") { + t.Fatalf("settlement event error = %v, want recovery repair context", event.Err) + } + if observed, observedOK := busruntime.DeliverySettlementIdentityFromContext(observedCtx); !observedOK || observed != wantIdentity { + t.Fatal("settlement observer did not retain the handler's delivery identity") + } + + database.observeSettlementFailure(context.Background(), nil, settlementErr) + if len(events) != 1 { + t.Fatalf("nil settlement job emitted an event: %+v", events[1:]) + } +} + +// TestDatabaseSettlementRejectsInvalidAndUnpersistableOutcomes verifies final +// state changes require a fenced job, a failed attempt, and a durable update. +func TestDatabaseSettlementRejectsInvalidAndUnpersistableOutcomes(t *testing.T) { + database := &databaseQueue{} + if err := database.markDone(context.Background(), nil); err == nil { + t.Fatal("markDone accepted a nil settlement job") + } + if err := database.markFailed(context.Background(), nil, errors.New("handler failed")); err == nil { + t.Fatal("markFailed accepted a nil settlement job") + } + job := &dbJob{ + id: 9, + processingToken: strings.Repeat("c", databaseProcessingTokenBytes*2), + maxRetry: 1, + } + if err := database.markFailed(context.Background(), job, nil); err == nil || + !strings.Contains(err.Error(), "successful attempt") { + t.Fatalf("successful failure settlement error = %v", err) + } + + updateErr := errors.New("pending settlement unavailable") + conn := &databaseConnStub{ + exec: func(context.Context, string, []driver.NamedValue) (driver.Result, error) { + return nil, updateErr + }, + } + db := newDatabaseStub(conn) + defer db.Close() + database.db = db + database.cfg.DriverName = "mysql" + if err := database.markFailed(context.Background(), job, errors.New("handler failed")); !errors.Is(err, updateErr) { + t.Fatalf("pending settlement error = %v, want %v", err, updateErr) + } +} + +// TestDatabaseUniqueClaimBoundsPersistentState verifies the periodic prune, +// minimum TTL, and lock-insert failures remain part of the surrounding +// transaction outcome. +func TestDatabaseUniqueClaimBoundsPersistentState(t *testing.T) { + job := queue.NewJob("reports:build").Payload([]byte(`{"report_id":42}`)) + + t.Run("periodic prune failure", func(t *testing.T) { + pruneErr := errors.New("prune unavailable") + conn := &databaseConnStub{ + query: func(context.Context, string, []driver.NamedValue) (driver.Rows, error) { + return databaseCountRows(1_000), nil + }, + exec: func(_ context.Context, query string, _ []driver.NamedValue) (driver.Result, error) { + if !strings.Contains(query, "DELETE FROM queue_unique_locks") { + return nil, fmt.Errorf("unexpected query after prune failure: %s", query) + } + return nil, pruneErr + }, + } + db := newDatabaseStub(conn) + defer db.Close() + tx, err := db.BeginTx(context.Background(), nil) + if err != nil { + t.Fatalf("begin unique transaction: %v", err) + } + defer tx.Rollback() + database := &databaseQueue{cfg: localDatabaseConfig{DriverName: "mysql"}} + database.uniqueClaims.Store(databaseUniquePruneInterval - 1) + if _, err := database.acquireUnique(context.Background(), tx, job, "default", time.Second); !errors.Is(err, pruneErr) { + t.Fatalf("periodic prune error = %v, want %v", err, pruneErr) + } + }) + + t.Run("sub-millisecond ttl", func(t *testing.T) { + var insertArgs []driver.NamedValue + conn := &databaseConnStub{ + query: func(context.Context, string, []driver.NamedValue) (driver.Rows, error) { + return databaseCountRows(1_000), nil + }, + exec: func(_ context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + if !strings.Contains(query, "INSERT IGNORE INTO queue_unique_locks") { + return nil, fmt.Errorf("unexpected uniqueness query: %s", query) + } + insertArgs = append([]driver.NamedValue(nil), args...) + return driver.RowsAffected(1), nil + }, + } + db := newDatabaseStub(conn) + defer db.Close() + tx, err := db.BeginTx(context.Background(), nil) + if err != nil { + t.Fatalf("begin unique transaction: %v", err) + } + defer tx.Rollback() + database := &databaseQueue{cfg: localDatabaseConfig{DriverName: "mysql"}} + acquired, err := database.acquireUnique(context.Background(), tx, job, "default", time.Nanosecond) + if err != nil || !acquired { + t.Fatalf("sub-millisecond unique claim = %t, %v", acquired, err) + } + if len(insertArgs) != 2 || insertArgs[1].Value != int64(1_001) { + t.Fatalf("unique insert arguments = %#v, want expiry 1001", insertArgs) + } + }) + + t.Run("insert failure", func(t *testing.T) { + insertErr := errors.New("lock insert unavailable") + conn := &databaseConnStub{ + query: func(context.Context, string, []driver.NamedValue) (driver.Rows, error) { + return databaseCountRows(1_000), nil + }, + exec: func(context.Context, string, []driver.NamedValue) (driver.Result, error) { + return nil, insertErr + }, + } + db := newDatabaseStub(conn) + defer db.Close() + tx, err := db.BeginTx(context.Background(), nil) + if err != nil { + t.Fatalf("begin unique transaction: %v", err) + } + defer tx.Rollback() + database := &databaseQueue{cfg: localDatabaseConfig{DriverName: "mysql"}} + if _, err := database.acquireUnique(context.Background(), tx, job, "default", time.Second); !errors.Is(err, insertErr) { + t.Fatalf("unique insert error = %v, want %v", err, insertErr) + } + }) +} + +// TestDatabaseAdditiveColumnMigrationHandlesInspectionAndRaces verifies a +// concurrent migration is accepted only after the required column becomes +// visible, while inspection and persistent ALTER failures remain fatal. +func TestDatabaseAdditiveColumnMigrationHandlesInspectionAndRaces(t *testing.T) { + migrations := []struct { + name string + driverName string + columnName string + migrate func(*databaseQueue, context.Context) error + }{ + { + name: "processing token", + driverName: "postgres", + columnName: "processing_token", + migrate: (*databaseQueue).ensureProcessingTokenColumn, + }, + { + name: "job metadata", + driverName: "mysql", + columnName: "metadata_json", + migrate: (*databaseQueue).ensureMetadataJSONColumn, + }, + } + inspectionErr := errors.New("column inspection unavailable") + alterErr := errors.New("alter lost migration race") + + for _, migration := range migrations { + t.Run(migration.name, func(t *testing.T) { + scenarios := []struct { + name string + inspectionErr error + recheckCount int64 + wantErr bool + wantExec int + }{ + {name: "inspection failure", inspectionErr: inspectionErr, wantErr: true}, + {name: "concurrent migration", recheckCount: 1, wantExec: 1}, + {name: "persistent alter failure", wantErr: true, wantExec: 1}, + } + for _, scenario := range scenarios { + t.Run(scenario.name, func(t *testing.T) { + queryCalls := 0 + execCalls := 0 + conn := &databaseConnStub{ + query: func(_ context.Context, _ string, args []driver.NamedValue) (driver.Rows, error) { + queryCalls++ + if len(args) != 1 || args[0].Value != migration.columnName { + return nil, fmt.Errorf("inspected column arguments = %#v", args) + } + if scenario.inspectionErr != nil { + return nil, scenario.inspectionErr + } + if queryCalls == 1 { + return databaseCountRows(0), nil + } + return databaseCountRows(scenario.recheckCount), nil + }, + exec: func(_ context.Context, query string, _ []driver.NamedValue) (driver.Result, error) { + execCalls++ + if !strings.Contains(query, migration.columnName) { + return nil, fmt.Errorf("migration query %q does not add %s", query, migration.columnName) + } + return nil, alterErr + }, + } + db := newDatabaseStub(conn) + defer db.Close() + database := &databaseQueue{ + cfg: localDatabaseConfig{DriverName: migration.driverName}, + db: db, + } + err := migration.migrate(database, context.Background()) + if (err != nil) != scenario.wantErr { + t.Fatalf("migration error = %v, wantErr %t", err, scenario.wantErr) + } + if execCalls != scenario.wantExec { + t.Fatalf("migration executions = %d, want %d", execCalls, scenario.wantExec) + } + }) + } + }) + } +} + +type databaseManagedSchemaFault struct { + missingTable string + missingColumn string + errorTable string + errorColumnsTable string + err error +} + +// databaseManagedQueueJobColumns is an independent test inventory of every +// queue_jobs field used by runtime SQL. +var databaseManagedQueueJobColumns = []string{ + "id", + "queue_name", + "job_type", + "payload", + "metadata_json", + "timeout_seconds", + "max_retry", + "backoff_millis", + "attempt", + "available_at", + "processing_started_at", + "processing_token", + "last_error", + "state", + "created_at", + "updated_at", +} + +// databaseManagedQueueUniqueLockColumns is an independent test inventory of +// every queue_unique_locks field used by runtime SQL. +var databaseManagedQueueUniqueLockColumns = []string{ + "lock_key", + "expires_at", +} + +// databaseManagedSchemaQuery returns a complete dialect-neutral schema +// inspection script with one optional structural or query failure. +func databaseManagedSchemaQuery(fault databaseManagedSchemaFault) func(context.Context, string, []driver.NamedValue) (driver.Rows, error) { + return func(_ context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + if strings.Contains(query, "sqlite_master") || strings.Contains(query, "information_schema.tables") || strings.Contains(query, "FROM pg_class") { + if len(args) != 1 { + return nil, fmt.Errorf("table query arguments = %#v", args) + } + tableName := fmt.Sprint(args[0].Value) + if tableName == fault.errorTable { + return nil, fault.err + } + if tableName == fault.missingTable { + return databaseCountRows(0), nil + } + return databaseCountRows(1), nil + } + + if len(args) != 1 { + return nil, fmt.Errorf("column query arguments = %#v", args) + } + tableName := fmt.Sprint(args[0].Value) + if tableName == fault.errorColumnsTable { + return nil, fault.err + } + columns := databaseManagedQueueUniqueLockColumns + if tableName == "queue_jobs" { + columns = databaseManagedQueueJobColumns + } + values := make([][]driver.Value, 0, len(columns)) + for _, columnName := range columns { + if tableName+"."+columnName != fault.missingColumn { + values = append(values, []driver.Value{columnName}) + } + } + return &databaseRowsStub{ + columns: []string{"column_name"}, + values: values, + }, nil + } +} + +// TestDatabaseManagedSchemaValidationRejectsIncompleteBackends verifies worker +// startup validates every table and column used by runtime SQL before polling. +func TestDatabaseManagedSchemaValidationRejectsIncompleteBackends(t *testing.T) { + validationErr := errors.New("schema inspection unavailable") + type managedSchemaCase struct { + name string + driverName string + fault databaseManagedSchemaFault + want string + } + tests := []managedSchemaCase{ + { + name: "mysql table inspection failure", + driverName: "mysql", + fault: databaseManagedSchemaFault{ + errorTable: "queue_jobs", + err: validationErr, + }, + want: "validate caller-managed queue_jobs table", + }, + { + name: "postgres missing queue jobs table", + driverName: "postgres", + fault: databaseManagedSchemaFault{ + missingTable: "queue_jobs", + }, + want: "missing required queue_jobs table", + }, + { + name: "mysql base column inspection failure", + driverName: "mysql", + fault: databaseManagedSchemaFault{ + errorColumnsTable: "queue_jobs", + err: validationErr, + }, + want: "validate caller-managed queue_jobs columns", + }, + { + name: "postgres uniqueness column inspection failure", + driverName: "postgres", + fault: databaseManagedSchemaFault{ + errorColumnsTable: "queue_unique_locks", + err: validationErr, + }, + want: "validate caller-managed queue_unique_locks columns", + }, + { + name: "postgres uniqueness table inspection failure", + driverName: "postgres", + fault: databaseManagedSchemaFault{ + errorTable: "queue_unique_locks", + err: validationErr, + }, + want: "validate caller-managed queue_unique_locks table", + }, + { + name: "mysql missing uniqueness table", + driverName: "mysql", + fault: databaseManagedSchemaFault{ + missingTable: "queue_unique_locks", + }, + want: "missing required queue_unique_locks table", + }, + } + + for _, columnName := range databaseManagedQueueJobColumns { + tests = append(tests, managedSchemaCase{ + name: "mysql missing queue jobs " + columnName, + driverName: "mysql", + fault: databaseManagedSchemaFault{ + missingColumn: "queue_jobs." + columnName, + }, + want: "missing required " + columnName + " column", + }) + } + for _, columnName := range databaseManagedQueueUniqueLockColumns { + tests = append(tests, managedSchemaCase{ + name: "postgres missing uniqueness " + columnName, + driverName: "postgres", + fault: databaseManagedSchemaFault{ + missingColumn: "queue_unique_locks." + columnName, + }, + want: "missing required " + columnName + " column", + }) + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + execCalls := 0 + conn := &databaseConnStub{ + query: databaseManagedSchemaQuery(test.fault), + exec: func(context.Context, string, []driver.NamedValue) (driver.Result, error) { + execCalls++ + return nil, errors.New("managed validation performed DDL") + }, + } + db := newDatabaseStub(conn) + defer db.Close() + database := &databaseQueue{ + cfg: localDatabaseConfig{ + DriverName: test.driverName, + AutoMigrate: false, + }, + db: db, + shutdownCh: make(chan struct{}), + } + err := database.StartWorkers(context.Background()) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("managed schema startup error = %v, want %q", err, test.want) + } + if test.fault.err != nil && !errors.Is(err, test.fault.err) { + t.Fatalf("managed schema startup error = %v, want wrapped %v", err, test.fault.err) + } + if database.started.Load() { + t.Fatal("workers started after caller-managed schema validation failed") + } + if execCalls != 0 { + t.Fatalf("managed schema validation executed %d statements, want read-only inspection", execCalls) + } + }) + } +} + +// TestDatabaseManagedSchemaPreflightAndStartupRetry verifies readiness and +// worker startup share read-only validation and recover after schema repair. +func TestDatabaseManagedSchemaPreflightAndStartupRetry(t *testing.T) { + missingUniqueTable := true + execCalls := 0 + queryCalls := 0 + conn := &databaseConnStub{ + query: func(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + queryCalls++ + fault := databaseManagedSchemaFault{} + if missingUniqueTable { + fault.missingTable = "queue_unique_locks" + } + return databaseManagedSchemaQuery(fault)(ctx, query, args) + }, + exec: func(context.Context, string, []driver.NamedValue) (driver.Result, error) { + execCalls++ + return nil, errors.New("managed validation performed DDL") + }, + } + db := newDatabaseStub(conn) + defer db.Close() + database := &databaseQueue{ + cfg: localDatabaseConfig{ + DriverName: "mysql", + AutoMigrate: true, + DisableAutoMigrate: true, + }, + db: db, + shutdownCh: make(chan struct{}), + } + + preflightErr := database.Preflight(context.Background()) + startErr := database.StartWorkers(context.Background()) + if preflightErr == nil || startErr == nil || preflightErr.Error() != startErr.Error() { + t.Fatalf("managed validation errors = (preflight: %v, startup: %v), want matching failures", preflightErr, startErr) + } + if database.started.Load() { + t.Fatal("failed managed startup latched the queue as started") + } + if queryCalls != 6 { + t.Fatalf("failed managed validation queries = %d, want three per attempt", queryCalls) + } + + missingUniqueTable = false + queryCalls = 0 + if err := database.Preflight(context.Background()); err != nil { + t.Fatalf("managed preflight after schema repair: %v", err) + } + if err := database.StartWorkers(context.Background()); err != nil { + t.Fatalf("managed startup after schema repair: %v", err) + } + if !database.started.Load() { + t.Fatal("managed startup did not become ready after schema repair") + } + if queryCalls != 8 { + t.Fatalf("successful managed validation queries = %d, want four per attempt", queryCalls) + } + if execCalls != 0 { + t.Fatalf("managed validation executed %d statements, want no DDL", execCalls) + } +} + +// TestDatabaseManagedSchemaColumnInspectionReportsFailures verifies catalog +// conversion and iteration errors remain visible instead of resembling absent columns. +func TestDatabaseManagedSchemaColumnInspectionReportsFailures(t *testing.T) { + iterationErr := errors.New("catalog iteration unavailable") + tests := []struct { + name string + rows driver.Rows + want string + }{ + { + name: "scan failure", + rows: &databaseRowsStub{ + columns: []string{"column_name"}, + values: [][]driver.Value{{nil}}, + }, + want: "scan queue_jobs column", + }, + { + name: "iteration failure", + rows: &databaseRowsStub{ + columns: []string{"column_name"}, + err: iterationErr, + }, + want: "inspect queue_jobs columns", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + conn := &databaseConnStub{ + query: func(context.Context, string, []driver.NamedValue) (driver.Rows, error) { + return test.rows, nil + }, + } + db := newDatabaseStub(conn) + defer db.Close() + database := &databaseQueue{ + cfg: localDatabaseConfig{DriverName: "mysql"}, + db: db, + } + _, err := database.managedQueueTableColumns(context.Background(), managedQueueJobsTable) + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("managed column inspection error = %v, want %q", err, test.want) + } + if test.name == "iteration failure" && !errors.Is(err, iterationErr) { + t.Fatalf("managed column inspection error = %v, want wrapped %v", err, iterationErr) + } + }) + } +} + +// TestDatabaseManagedSchemaSQLiteCatalog verifies SQLite readiness uses the +// same complete, bounded structural inspection as the server SQL dialects. +func TestDatabaseManagedSchemaSQLiteCatalog(t *testing.T) { + queryCalls := 0 + conn := &databaseConnStub{ + query: func(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + queryCalls++ + return databaseManagedSchemaQuery(databaseManagedSchemaFault{})(ctx, query, args) + }, + } + db := newDatabaseStub(conn) + defer db.Close() + database := &databaseQueue{ + cfg: localDatabaseConfig{DriverName: "sqlite"}, + db: db, + } + if err := database.requireManagedQueueSchema(context.Background()); err != nil { + t.Fatalf("sqlite managed schema validation: %v", err) + } + if queryCalls != 4 { + t.Fatalf("sqlite managed schema queries = %d, want 4", queryCalls) + } +} + +// TestDatabaseManagedSchemaRejectsNonBaseRelations verifies views cannot pass +// a readiness gate whose runtime contract requires writes and row locking. +func TestDatabaseManagedSchemaRejectsNonBaseRelations(t *testing.T) { + tests := []struct { + name string + driverName string + baseFilter string + }{ + { + name: "mysql view", + driverName: "mysql", + baseFilter: "table_type = 'BASE TABLE'", + }, + { + name: "postgres view", + driverName: "postgres", + baseFilter: "relkind IN ('r', 'p')", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + conn := &databaseConnStub{ + query: func(_ context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + if len(args) != 1 || fmt.Sprint(args[0].Value) != "queue_jobs" { + return nil, fmt.Errorf("table query arguments = %#v", args) + } + if !strings.Contains(query, test.baseFilter) { + return databaseCountRows(1), nil + } + return databaseCountRows(0), nil + }, + } + db := newDatabaseStub(conn) + defer db.Close() + database := &databaseQueue{ + cfg: localDatabaseConfig{DriverName: test.driverName}, + db: db, + } + err := database.requireManagedQueueSchema(context.Background()) + if err == nil || !strings.Contains(err.Error(), "missing required queue_jobs table") { + t.Fatalf("managed view validation error = %v, want missing base-table diagnostic", err) + } + }) + } +} + +// TestDatabasePreflightSkipsManagedValidationWhenMigrationsAreEnabled verifies +// readiness does not require a schema that worker startup is configured to create. +func TestDatabasePreflightSkipsManagedValidationWhenMigrationsAreEnabled(t *testing.T) { + queryCalls := 0 + conn := &databaseConnStub{ + query: func(context.Context, string, []driver.NamedValue) (driver.Rows, error) { + queryCalls++ + return nil, errors.New("unexpected schema inspection") + }, + } + db := newDatabaseStub(conn) + defer db.Close() + database := &databaseQueue{ + cfg: localDatabaseConfig{AutoMigrate: true}, + db: db, + } + if err := database.Preflight(context.Background()); err != nil { + t.Fatalf("auto-migrating preflight: %v", err) + } + if queryCalls != 0 { + t.Fatalf("auto-migrating preflight schema queries = %d, want 0", queryCalls) + } +} + +// TestDatabasePreflightPropagatesPingFailureBeforeManagedValidation verifies a +// connectivity failure remains the primary readiness diagnostic. +func TestDatabasePreflightPropagatesPingFailureBeforeManagedValidation(t *testing.T) { + pingErr := errors.New("database unavailable") + queryCalls := 0 + conn := &databaseConnStub{ + pingErr: pingErr, + query: func(context.Context, string, []driver.NamedValue) (driver.Rows, error) { + queryCalls++ + return nil, errors.New("unexpected schema inspection") + }, + } + db := newDatabaseStub(conn) + defer db.Close() + database := &databaseQueue{db: db} + if err := database.Preflight(context.Background()); !errors.Is(err, pingErr) { + t.Fatalf("database preflight = %v, want %v", err, pingErr) + } + if queryCalls != 0 { + t.Fatalf("database preflight queries after failed ping = %d, want 0", queryCalls) + } +} + +// TestDatabaseSQLiteColumnInspectionReportsFailures verifies caller-managed +// startup can distinguish query, row conversion, and row iteration failures. +func TestDatabaseSQLiteColumnInspectionReportsFailures(t *testing.T) { + queryErr := errors.New("pragma unavailable") + rowsErr := errors.New("pragma iteration failed") + tests := []struct { + name string + rows driver.Rows + err error + want string + }{ + {name: "query failure", err: queryErr, want: "inspect sqlite queue job column"}, + { + name: "scan failure", + rows: &databaseRowsStub{ + columns: []string{"cid", "name", "type", "notnull", "default", "pk"}, + values: [][]driver.Value{{"invalid", "metadata_json", "TEXT", int64(0), nil, int64(0)}}, + }, + want: "scan sqlite queue column", + }, + { + name: "iteration failure", + rows: &databaseRowsStub{ + columns: []string{"cid", "name", "type", "notnull", "default", "pk"}, + err: rowsErr, + }, + want: "inspect sqlite queue columns", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + conn := &databaseConnStub{ + query: func(context.Context, string, []driver.NamedValue) (driver.Rows, error) { + return test.rows, test.err + }, + } + db := newDatabaseStub(conn) + defer db.Close() + database := &databaseQueue{ + cfg: localDatabaseConfig{DriverName: "sqlite"}, + db: db, + } + _, err := database.queueJobColumnExists(context.Background(), "metadata_json") + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("sqlite column inspection error = %v, want %q", err, test.want) + } + }) + } +} + +// TestDatabaseSchemaStartupPropagatesCompatibilityFailures verifies schema +// initialization stops at each additive compatibility dependency. +func TestDatabaseSchemaStartupPropagatesCompatibilityFailures(t *testing.T) { + stageErr := errors.New("compatibility dependency unavailable") + for _, stage := range []string{"processing column", "metadata column", "mysql index", "database clock"} { + t.Run(stage, func(t *testing.T) { + conn := &databaseConnStub{ + exec: func(context.Context, string, []driver.NamedValue) (driver.Result, error) { + return driver.RowsAffected(0), nil + }, + query: func(_ context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + if len(args) == 1 { + switch args[0].Value { + case "processing_token": + if stage == "processing column" { + return nil, stageErr + } + return databaseCountRows(1), nil + case "metadata_json": + if stage == "metadata column" { + return nil, stageErr + } + return databaseCountRows(1), nil + case "idx_queue_unique_locks_expires": + if stage == "mysql index" { + return nil, stageErr + } + return databaseCountRows(1), nil + } + } + if stage == "database clock" && strings.Contains(query, "UNIX_TIMESTAMP") { + return nil, stageErr + } + return nil, fmt.Errorf("unexpected schema query at %s: %s", stage, query) + }, + } + db := newDatabaseStub(conn) + defer db.Close() + database := &databaseQueue{ + cfg: localDatabaseConfig{DriverName: "mysql"}, + db: db, + } + if err := database.ensureSchema(context.Background()); !errors.Is(err, stageErr) { + t.Fatalf("schema startup error = %v, want %v", err, stageErr) + } + }) + } +} + +// TestDatabaseMySQLIndexMigrationHandlesInspectionAndRaces verifies the +// additive expiry index accepts a concurrent winner but not an unverifiable +// ALTER failure. +func TestDatabaseMySQLIndexMigrationHandlesInspectionAndRaces(t *testing.T) { + inspectionErr := errors.New("index inspection unavailable") + alterErr := errors.New("index alter failed") + tests := []struct { + name string + inspectionErr error + recheckCount int64 + wantErr bool + wantExec int + }{ + {name: "inspection failure", inspectionErr: inspectionErr, wantErr: true}, + {name: "concurrent migration", recheckCount: 1, wantExec: 1}, + {name: "persistent alter failure", wantErr: true, wantExec: 1}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + queryCalls := 0 + execCalls := 0 + conn := &databaseConnStub{ + query: func(_ context.Context, _ string, args []driver.NamedValue) (driver.Rows, error) { + queryCalls++ + if len(args) != 1 || args[0].Value != "idx_queue_unique_locks_expires" { + return nil, fmt.Errorf("index query arguments = %#v", args) + } + if test.inspectionErr != nil { + return nil, test.inspectionErr + } + if queryCalls == 1 { + return databaseCountRows(0), nil + } + return databaseCountRows(test.recheckCount), nil + }, + exec: func(_ context.Context, query string, _ []driver.NamedValue) (driver.Result, error) { + execCalls++ + if !strings.Contains(query, "ADD INDEX idx_queue_unique_locks_expires") { + return nil, fmt.Errorf("unexpected index migration: %s", query) + } + return nil, alterErr + }, + } + db := newDatabaseStub(conn) + defer db.Close() + database := &databaseQueue{ + cfg: localDatabaseConfig{DriverName: "mysql"}, + db: db, + } + err := database.ensureMySQLUniqueExpiryIndex(context.Background()) + if (err != nil) != test.wantErr { + t.Fatalf("index migration error = %v, wantErr %t", err, test.wantErr) + } + if execCalls != test.wantExec { + t.Fatalf("index migration executions = %d, want %d", execCalls, test.wantExec) + } + }) + } +} diff --git a/driver/sqlqueuecore/queue_database_impl.go b/driver/sqlqueuecore/queue_database_impl.go index 6184566..2b0a707 100644 --- a/driver/sqlqueuecore/queue_database_impl.go +++ b/driver/sqlqueuecore/queue_database_impl.go @@ -2,9 +2,11 @@ package sqlqueuecore import ( "context" + "crypto/rand" "crypto/sha256" "database/sql" "encoding/hex" + "encoding/json" "errors" "fmt" "math" @@ -15,16 +17,54 @@ import ( "time" "github.com/goforj/queue" + "github.com/goforj/queue/busruntime" "github.com/goforj/queue/queuecore" ) +type managedQueueTable string + const ( defaultProcessingRecoveryGrace = 2 * time.Second defaultProcessingLeaseNoTimeout = 5 * time.Minute databaseFinalizeRetryCount = 3 databaseFinalizeRetryDelay = 25 * time.Millisecond + databaseFinalizeTimeout = 5 * time.Second + databaseUniquePruneInterval = 256 + databaseProcessingTokenBytes = 16 + databaseRecoveryMarker = "queue:internal:stale-processing-recovery:v1" + databaseRecoveryDiagnostic = "recovered stale processing job" + managedQueueJobsTable = managedQueueTable("queue_jobs") + managedQueueUniqueLocksTable = managedQueueTable("queue_unique_locks") ) +// managedQueueJobColumns captures the durable fields touched by dispatch, +// polling, recovery, settlement, administration, and statistics operations. +var managedQueueJobColumns = [...]string{ + "id", + "queue_name", + "job_type", + "payload", + "metadata_json", + "timeout_seconds", + "max_retry", + "backoff_millis", + "attempt", + "available_at", + "processing_started_at", + "processing_token", + "last_error", + "state", + "created_at", + "updated_at", +} + +// managedQueueUniqueLockColumns captures the durable fields required by +// distributed uniqueness acquisition and expiry pruning. +var managedQueueUniqueLockColumns = [...]string{ + "lock_key", + "expires_at", +} + // DatabaseConfig configures the SQL-backed database q. // @group Config type DatabaseConfig = queue.DatabaseConfig @@ -37,6 +77,7 @@ type localDatabaseConfig struct { PollInterval time.Duration DefaultQueue string AutoMigrate bool + DisableAutoMigrate bool ProcessingRecoveryGrace time.Duration ProcessingLeaseNoTimeout time.Duration Observer queue.Observer @@ -50,7 +91,9 @@ func (c localDatabaseConfig) normalize() localDatabaseConfig { if c.DefaultQueue == "" { c.DefaultQueue = "default" } - if !c.AutoMigrate { + if c.DisableAutoMigrate { + c.AutoMigrate = false + } else if !c.AutoMigrate { c.AutoMigrate = true } if c.ProcessingRecoveryGrace <= 0 { @@ -71,28 +114,55 @@ type databaseQueue struct { mu sync.RWMutex handlers map[string]queue.Handler - startOnce sync.Once + startMu sync.Mutex shutdownOnce sync.Once + shutdownDone chan struct{} + closeOnce sync.Once + closeDone chan struct{} + shutdownErr error workerWG sync.WaitGroup shutdownCh chan struct{} started atomic.Bool shuttingDown atomic.Bool + uniqueClaims atomic.Uint64 + continuation *busruntime.ContinuationScope observer queue.Observer } +type databaseRowQueryer interface { + QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row +} + +type databaseExecer interface { + ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) +} + type dbJob struct { - id int64 - queueName string - jobType string - payload []byte - timeoutSeconds sql.NullInt64 - maxRetry int - backoffMillis int64 - attempt int + id int64 + processingToken string + queueName string + jobType string + payload []byte + metadataJSON sql.NullString + timeoutSeconds sql.NullInt64 + maxRetry int + backoffMillis int64 + attempt int + recovered bool + recoveryToken string + applicationStateCommitted bool +} + +type databaseFailureSettlement struct { + state string + attempt int + availableAt int64 } +// New constructs a SQL queue while retaining caller ownership of any supplied database handle. func New(cfg queue.DatabaseConfig) (*databaseQueue, error) { + ownsDB := cfg.DB == nil local := localDatabaseConfig{ DB: cfg.DB, DriverName: cfg.DriverName, @@ -101,6 +171,7 @@ func New(cfg queue.DatabaseConfig) (*databaseQueue, error) { PollInterval: cfg.PollInterval, DefaultQueue: cfg.DefaultQueue, AutoMigrate: cfg.AutoMigrate, + DisableAutoMigrate: cfg.DisableAutoMigrate, ProcessingRecoveryGrace: cfg.ProcessingRecoveryGrace, ProcessingLeaseNoTimeout: cfg.ProcessingLeaseNoTimeout, Observer: cfg.Observer, @@ -113,6 +184,7 @@ func New(cfg queue.DatabaseConfig) (*databaseQueue, error) { PollInterval: local.PollInterval, DefaultQueue: local.DefaultQueue, AutoMigrate: local.AutoMigrate, + DisableAutoMigrate: local.DisableAutoMigrate, ProcessingRecoveryGrace: local.ProcessingRecoveryGrace, ProcessingLeaseNoTimeout: local.ProcessingLeaseNoTimeout, Observer: local.Observer, @@ -132,12 +204,13 @@ func New(cfg queue.DatabaseConfig) (*databaseQueue, error) { } d := &databaseQueue{ - cfg: local, - db: cfg.DB, - handlers: make(map[string]queue.Handler), - shutdownCh: make(chan struct{}), - ownsDB: cfg.DB != nil && cfg.DriverName != "" && cfg.DSN != "", - observer: cfg.Observer, + cfg: local, + db: cfg.DB, + handlers: make(map[string]queue.Handler), + shutdownCh: make(chan struct{}), + ownsDB: ownsDB, + continuation: busruntime.NewContinuationScope(), + observer: cfg.Observer, } if cfg.DriverName == "sqlite" { d.db.SetMaxOpenConns(1) @@ -152,6 +225,7 @@ func (d *databaseQueue) Driver() queue.Driver { return queue.DriverDatabase } +// Preflight verifies connectivity and any caller-managed schema without changing database state. func (d *databaseQueue) Preflight(ctx context.Context) error { if ctx == nil { ctx = context.Background() @@ -159,7 +233,13 @@ func (d *databaseQueue) Preflight(ctx context.Context) error { if err := ctx.Err(); err != nil { return err } - return d.db.PingContext(ctx) + if err := d.db.PingContext(ctx); err != nil { + return err + } + if d.cfg.AutoMigrate && !d.cfg.DisableAutoMigrate { + return nil + } + return d.requireManagedQueueSchema(ctx) } func (d *databaseQueue) Register(jobType string, handler queue.Handler) { @@ -171,52 +251,119 @@ func (d *databaseQueue) Register(jobType string, handler queue.Handler) { d.mu.Unlock() } +// StartWorkers prepares or validates durable storage before admitting the worker generation. func (d *databaseQueue) StartWorkers(ctx context.Context) error { if ctx == nil { ctx = context.Background() } + if err := ctx.Err(); err != nil { + return err + } + d.startMu.Lock() + defer d.startMu.Unlock() + if d.shuttingDown.Load() { + return queue.ErrQueuerShuttingDown + } if d.started.Load() { return nil } - var startErr error - d.startOnce.Do(func() { - if d.cfg.AutoMigrate { - if err := d.ensureSchema(ctx); err != nil { - startErr = err - return - } - } - for i := 0; i < d.cfg.Workers; i++ { - d.workerWG.Add(1) - go d.workerLoop() + if d.cfg.AutoMigrate && !d.cfg.DisableAutoMigrate { + if err := d.ensureSchema(ctx); err != nil { + return err } - d.started.Store(true) - }) - return startErr + } else if err := d.requireManagedQueueSchema(ctx); err != nil { + return err + } + for i := 0; i < d.cfg.Workers; i++ { + d.workerWG.Add(1) + go d.workerLoop() + } + d.started.Store(true) + return nil } +// Shutdown drains workers before closing only database handles opened by this queue. func (d *databaseQueue) Shutdown(ctx context.Context) error { + if err := d.DrainWorkers(ctx); err != nil { + return err + } + if ctx == nil { + ctx = context.Background() + } + d.startMu.Lock() + d.closeOnce.Do(func() { + d.closeDone = make(chan struct{}) + go d.finishResourceClose(d.closeDone) + }) + done := d.closeDone + d.startMu.Unlock() + select { + case <-done: + return d.takeShutdownError() + case <-ctx.Done(): + return ctx.Err() + } +} + +// DrainWorkers latches shutdown admission and joins the worker generation +// without closing producer resources that an admitted dispatch may still use. +func (d *databaseQueue) DrainWorkers(ctx context.Context) error { if ctx == nil { ctx = context.Background() } + d.startMu.Lock() d.shutdownOnce.Do(func() { d.shuttingDown.Store(true) close(d.shutdownCh) + d.shutdownDone = make(chan struct{}) + go d.finishWorkerDrain(d.shutdownDone) }) - if err := waitGroupWithContext(ctx, &d.workerWG); err != nil { - return err + done := d.shutdownDone + d.startMu.Unlock() + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() } +} + +// finishWorkerDrain waits once for the worker generation so callers with +// expired deadlines can retry without multiplying drain goroutines. +func (d *databaseQueue) finishWorkerDrain(done chan struct{}) { + d.workerWG.Wait() + close(done) +} + +// finishResourceClose releases the owned handle only after worker and root +// operation drains establish that no admitted dispatch can still use it. +func (d *databaseQueue) finishResourceClose(done chan struct{}) { + var closeErr error if d.ownsDB { - _ = d.db.Close() + closeErr = d.db.Close() } - return nil + d.startMu.Lock() + d.shutdownErr = closeErr + close(done) + d.startMu.Unlock() +} + +// takeShutdownError reports completed cleanup diagnostics once so a later +// outer runtime retry can converge after the owned resource is already closed. +func (d *databaseQueue) takeShutdownError() error { + d.startMu.Lock() + defer d.startMu.Unlock() + err := d.shutdownErr + d.shutdownErr = nil + return err } +// Dispatch commits a uniqueness claim and its queue row in one transaction when deduplication is requested. func (d *databaseQueue) Dispatch(ctx context.Context, job queue.Job) error { if ctx == nil { ctx = context.Background() } - if d.shuttingDown.Load() { + if d.shuttingDown.Load() && !d.continuation.Owns(ctx) { return queue.ErrQueuerShuttingDown } if err := queuecore.ValidateDriverJob(job); err != nil { @@ -243,16 +390,6 @@ func (d *databaseQueue) Dispatch(ctx context.Context, job queue.Job) error { availableAt = availableAt.Add(parsed.Delay) } - if parsed.UniqueTTL > 0 { - ok, err := d.acquireUnique(ctx, job, queueName, now.Add(parsed.UniqueTTL)) - if err != nil { - return err - } - if !ok { - return queuecore.ErrDuplicate - } - } - maxRetry := 0 if parsed.MaxRetry != nil { maxRetry = *parsed.MaxRetry @@ -270,26 +407,49 @@ func (d *databaseQueue) Dispatch(ctx context.Context, job queue.Job) error { } timeoutSeconds = seconds } + metadataJSON, err := databaseMetadataJSON(job) + if err != nil { + return err + } query := d.rebind( `INSERT INTO queue_jobs - (queue_name, job_type, payload, timeout_seconds, max_retry, backoff_millis, attempt, available_at, state, created_at, updated_at) - VALUES (?, ?, ?, ?, ?, ?, 0, ?, 'pending', ?, ?)`, + (queue_name, job_type, payload, metadata_json, timeout_seconds, max_retry, backoff_millis, attempt, available_at, state, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, 0, ?, 'pending', ?, ?)`, ) - _, err := d.db.ExecContext( - ctx, - query, + args := []any{ queueName, job.Type, payloadBytes, + metadataJSON, timeoutSeconds, maxRetry, backoffMillis, availableAt.UnixMilli(), now.UnixMilli(), now.UnixMilli(), - ) - return err + } + if parsed.UniqueTTL <= 0 { + _, err := d.db.ExecContext(ctx, query, args...) + return err + } + + tx, err := d.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + ok, err := d.acquireUnique(ctx, tx, job, queueName, parsed.UniqueTTL) + if err != nil { + return err + } + if !ok { + return queuecore.ErrDuplicate + } + if _, err := tx.ExecContext(ctx, query, args...); err != nil { + return err + } + return tx.Commit() } func (d *databaseQueue) Stats(ctx context.Context) (queue.StatsSnapshot, error) { @@ -421,8 +581,8 @@ func (d *databaseQueue) RetryJob(ctx context.Context, queueName, jobID string) e } now := time.Now().UnixMilli() query := d.rebind(`UPDATE queue_jobs -SET state='pending', available_at=?, processing_started_at=NULL, last_error=NULL, updated_at=? -WHERE id=? AND queue_name=?`) + SET state='pending', available_at=?, processing_started_at=NULL, processing_token=NULL, last_error=NULL, updated_at=? + WHERE id=? AND queue_name=?`) _, execErr := d.db.ExecContext(ctx, query, now, now, id, queuecore.NormalizeQueueName(queueName)) return execErr } @@ -440,8 +600,8 @@ func (d *databaseQueue) CancelJob(ctx context.Context, jobID string) error { } now := time.Now().UnixMilli() query := d.rebind(`UPDATE queue_jobs -SET state='dead', processing_started_at=NULL, last_error=?, updated_at=? -WHERE id=?`) + SET state='dead', processing_started_at=NULL, processing_token=NULL, last_error=?, updated_at=? + WHERE id=?`) _, execErr := d.db.ExecContext(ctx, query, "canceled from queue admin", now, id) return execErr } @@ -565,53 +725,152 @@ func (d *databaseQueue) workerLoop() { } } +// processJob commits deferred success facts only after the durable row reaches its final state. func (d *databaseQueue) processJob(job *dbJob) { handler, ok := d.lookup(job.jobType) if !ok { - d.markFailedWithRetry(job, fmt.Errorf("no handler registered for job type %q", job.jobType)) + if err := d.markFailedWithRetry(job, fmt.Errorf("no handler registered for job type %q", job.jobType)); err != nil { + d.handleSettlementFailure(context.Background(), job, err) + } return } - ctx := context.Background() + ctx, settlement := databaseSettlementContext(job) if job.timeoutSeconds.Valid && job.timeoutSeconds.Int64 > 0 { var cancel context.CancelFunc ctx, cancel = context.WithTimeout(ctx, time.Duration(job.timeoutSeconds.Int64)*time.Second) defer cancel() } - err := handler( + err := d.runHandlerWithContinuationPermit( ctx, - queuecore.DriverWithAttempt( - queue.NewJob(job.jobType). - Payload(job.payload). - OnQueue(job.queueName). - Retry(job.maxRetry), - job.attempt, - ), + handler, + databaseDeliveryJob(job), ) + job.applicationStateCommitted = settlement.ApplicationStateCommitted() + var settlementErr error if err == nil { - d.markDoneWithRetry(job) + settlementErr = d.markDoneWithRetry(job) + } else { + settlementErr = d.markFailedWithRetry(job, err) + } + if settlementErr != nil { + d.handleSettlementFailure(ctx, job, settlementErr) return } - d.markFailedWithRetry(job, err) + settlement.Commit() +} + +// handleSettlementFailure preserves inherited recovery lineage before reporting +// an exhausted physical settlement failure. Deferred facts remain uncommitted +// until a later generation positively settles the fenced row. +func (d *databaseQueue) handleSettlementFailure(ctx context.Context, job *dbJob, settlementErr error) { + repairCtx, cancel := context.WithTimeout(context.Background(), databaseFinalizeTimeout) + repairErr := d.restoreRecoveredSettlementLineage(repairCtx, job, settlementErr) + cancel() + if repairErr != nil { + settlementErr = errors.Join(settlementErr, fmt.Errorf("restore recovered database settlement lineage: %w", repairErr)) + } + d.observeSettlementFailure(ctx, job, settlementErr) +} + +// databaseSettlementContext exposes stale-processing evidence to orchestration +// while retaining the driver's post-handler commit boundary on every delivery. +func databaseSettlementContext(job *dbJob) (context.Context, *busruntime.DeliverySettlement) { + ctx, settlement := busruntime.WithDeliverySettlement(context.Background()) + if job != nil { + ctx = busruntime.WithDeliveryProvenance(ctx, busruntime.DeliveryProvenance{ + GenerationID: job.processingToken, + RecoveredGenerationID: job.recoveryToken, + Recovered: job.recovered, + }) + } + return ctx, settlement +} + +// runHandlerWithContinuationPermit limits shutdown-time descendant dispatch permission to this queue's active handler call. +func (d *databaseQueue) runHandlerWithContinuationPermit(ctx context.Context, handler queue.Handler, job queue.Job) error { + handlerCtx, release := d.continuation.Permit(ctx) + defer release() + return handler(handlerCtx, job) +} + +// databaseDeliveryJob restores persisted physical attempt metadata before the root orchestration adapter runs. +func databaseDeliveryJob(job *dbJob) queue.Job { + delivery := queuecore.DriverWithAttempt( + queue.NewJob(job.jobType). + Payload(job.payload). + OnQueue(job.queueName). + Retry(job.maxRetry), + job.attempt, + ) + return queue.DriverWithMetadata(delivery, databaseJobMetadata(job.metadataJSON)) +} + +// databaseMetadataJSON serializes only metadata versions supported by this +// root module so unknown producer state cannot become trusted SQL correlation. +func databaseMetadataJSON(job queue.Job) (sql.NullString, error) { + metadata := queue.DriverMetadata(job) + if metadata.SchemaVersion == 0 { + return sql.NullString{}, nil + } + encoded, err := json.Marshal(metadata) + if err != nil { + return sql.NullString{}, fmt.Errorf("marshal database job metadata: %w", err) + } + return sql.NullString{String: string(encoded), Valid: true}, nil } -func (d *databaseQueue) markDoneWithRetry(job *dbJob) { +// databaseJobMetadata accepts nullable legacy rows and ignores malformed or +// unknown-version metadata without changing application delivery. +func databaseJobMetadata(raw sql.NullString) queue.DriverJobMetadata { + if !raw.Valid || strings.TrimSpace(raw.String) == "" { + return queue.DriverJobMetadata{} + } + var metadata queue.DriverJobMetadata + if err := json.Unmarshal([]byte(raw.String), &metadata); err != nil { + return queue.DriverJobMetadata{} + } + if metadata.SchemaVersion != queue.DriverJobMetadataVersion { + return queue.DriverJobMetadata{} + } + return metadata +} + +// markDoneWithRetry bounds each finalization attempt and returns the last error for settlement telemetry. +func (d *databaseQueue) markDoneWithRetry(job *dbJob) error { + var lastErr error for i := 0; i < databaseFinalizeRetryCount; i++ { - if err := d.markDone(context.Background(), job); err == nil { - return + ctx, cancel := context.WithTimeout(context.Background(), databaseFinalizeTimeout) + err := d.markDone(ctx, job) + cancel() + if err == nil { + return nil } else if i < databaseFinalizeRetryCount-1 { + lastErr = err time.Sleep(databaseFinalizeRetryDelay) + } else { + lastErr = err } } + return fmt.Errorf("finalize successful database job: %w", lastErr) } -func (d *databaseQueue) markFailedWithRetry(job *dbJob, runErr error) { +// markFailedWithRetry persists retry or terminal state without hiding exhausted finalization attempts. +func (d *databaseQueue) markFailedWithRetry(job *dbJob, runErr error) error { + var lastErr error for i := 0; i < databaseFinalizeRetryCount; i++ { - if err := d.markFailed(context.Background(), job, runErr); err == nil { - return + ctx, cancel := context.WithTimeout(context.Background(), databaseFinalizeTimeout) + err := d.markFailed(ctx, job, runErr) + cancel() + if err == nil { + return nil } else if i < databaseFinalizeRetryCount-1 { + lastErr = err time.Sleep(databaseFinalizeRetryDelay) + } else { + lastErr = err } } + return fmt.Errorf("finalize failed database job: %w", lastErr) } func (d *databaseQueue) claimOne(ctx context.Context) (*dbJob, error) { @@ -637,20 +896,36 @@ func (d *databaseQueue) claimOne(ctx context.Context) (*dbJob, error) { _ = tx.Rollback() return nil, nil } - update := d.rebind(`UPDATE queue_jobs SET state='processing', processing_started_at=?, updated_at=? WHERE id=? AND state='pending'`) - res, err := tx.ExecContext(ctx, update, now, now, job.id) + processingToken, err := newDatabaseProcessingToken() + if err != nil { + _ = tx.Rollback() + return nil, err + } + update := d.rebind(`UPDATE queue_jobs + SET state='processing', processing_started_at=?, processing_token=?, updated_at=? + WHERE id=? AND state='pending'`) + res, err := tx.ExecContext(ctx, update, now, processingToken, now, job.id) if err != nil { _ = tx.Rollback() return nil, err } - rows, _ := res.RowsAffected() + rows, err := res.RowsAffected() + if err != nil { + _ = tx.Rollback() + return nil, fmt.Errorf("read database claim rows: %w", err) + } if rows == 0 { _ = tx.Rollback() continue } + if rows != 1 { + _ = tx.Rollback() + return nil, fmt.Errorf("database claim affected %d rows, want 1", rows) + } if err := tx.Commit(); err != nil { return nil, err } + job.processingToken = processingToken return job, nil } return nil, nil @@ -663,7 +938,9 @@ func (d *databaseQueue) recoverStaleProcessing(ctx context.Context, nowMillis in noTimeoutCutoff = 0 } query := d.rebind(`UPDATE queue_jobs -SET state='pending', available_at=?, processing_started_at=NULL, updated_at=?, last_error=? + SET state='pending', available_at=?, processing_started_at=NULL, + processing_token=CASE WHEN processing_token IS NOT NULL AND processing_token <> '' THEN processing_token ELSE ? END, + updated_at=?, last_error=? WHERE state='processing' AND processing_started_at IS NOT NULL AND ( (timeout_seconds IS NOT NULL AND timeout_seconds > 0 AND (processing_started_at + (timeout_seconds * 1000) + ?) <= ?) OR @@ -673,8 +950,9 @@ WHERE state='processing' AND processing_started_at IS NOT NULL AND ( ctx, query, nowMillis, + databaseRecoveryMarker, nowMillis, - "recovered stale processing job", + databaseRecoveryDiagnostic, graceMillis, nowMillis, noTimeoutCutoff, @@ -697,7 +975,8 @@ WHERE state='processing' AND processing_started_at IS NOT NULL AND ( } func (d *databaseQueue) selectPendingJob(ctx context.Context, tx *sql.Tx, now int64) (*dbJob, error) { - query := `SELECT id, queue_name, job_type, payload, timeout_seconds, max_retry, backoff_millis, attempt + query := `SELECT id, queue_name, job_type, payload, metadata_json, timeout_seconds, max_retry, backoff_millis, attempt, + processing_token FROM queue_jobs WHERE queue_name=? AND state='pending' AND available_at <= ? ORDER BY id ASC @@ -708,88 +987,369 @@ LIMIT 1` query = d.rebind(query) row := tx.QueryRowContext(ctx, query, d.cfg.DefaultQueue, now) job := &dbJob{} + var pendingProcessingToken sql.NullString if err := row.Scan( &job.id, &job.queueName, &job.jobType, &job.payload, + &job.metadataJSON, &job.timeoutSeconds, &job.maxRetry, &job.backoffMillis, &job.attempt, + &pendingProcessingToken, ); err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, nil } return nil, err } + job.recoveryToken, job.recovered = databaseRecoveryProof(pendingProcessingToken) return job, nil } +// databaseRecoveryProof recognizes only transport-owned state and returns the +// earlier processing generation when that opaque identity survived recovery. +func databaseRecoveryProof(processingToken sql.NullString) (string, bool) { + if !processingToken.Valid { + return "", false + } + if processingToken.String == databaseRecoveryMarker { + return "", true + } + if !databaseProcessingTokenValid(processingToken.String) { + return "", false + } + return processingToken.String, true +} + func (d *databaseQueue) usesOptimisticClaimLoop() bool { return d.cfg.DriverName == "sqlite" } +// markDone deletes exactly the processing row owned by one successful delivery. func (d *databaseQueue) markDone(ctx context.Context, job *dbJob) error { - query := d.rebind(`DELETE FROM queue_jobs WHERE id=?`) - _, err := d.db.ExecContext(ctx, query, job.id) - return err + id, processingToken, err := databaseProcessingClaim(job) + if err != nil { + return err + } + query := d.rebind(`DELETE FROM queue_jobs WHERE id=? AND state='processing' AND processing_token=?`) + result, err := d.db.ExecContext(ctx, query, id, processingToken) + if err != nil { + return err + } + return requireDatabaseSettlementRow(result) } +// markFailed writes exactly one retryable or terminal delivery transition. func (d *databaseQueue) markFailed(ctx context.Context, job *dbJob, runErr error) error { - nextAttempt := job.attempt + 1 + id, processingToken, err := databaseProcessingClaim(job) + if err != nil { + return err + } now := time.Now().UnixMilli() - if nextAttempt > job.maxRetry { - query := d.rebind(`UPDATE queue_jobs SET state='dead', attempt=?, last_error=?, updated_at=? WHERE id=?`) - _, err := d.db.ExecContext(ctx, query, nextAttempt, runErr.Error(), now, job.id) + settlement, err := classifyDatabaseFailure(job, runErr, now) + if err != nil { return err } - nextAt := now - if job.backoffMillis > 0 { - nextAt += job.backoffMillis + if settlement.state == "dead" { + query := d.rebind(`UPDATE queue_jobs + SET state='dead', attempt=?, processing_started_at=NULL, processing_token=NULL, last_error=?, updated_at=? + WHERE id=? AND state='processing' AND processing_token=?`) + result, err := d.db.ExecContext(ctx, query, settlement.attempt, runErr.Error(), now, id, processingToken) + if err != nil { + return err + } + return requireDatabaseSettlementRow(result) } query := d.rebind(`UPDATE queue_jobs -SET state='pending', attempt=?, available_at=?, last_error=?, processing_started_at=NULL, updated_at=? -WHERE id=?`) - _, err := d.db.ExecContext(ctx, query, nextAttempt, nextAt, runErr.Error(), now, job.id) - return err + SET state='pending', attempt=?, available_at=?, last_error=?, processing_started_at=NULL, processing_token=?, updated_at=? + WHERE id=? AND state='processing' AND processing_token=?`) + result, err := d.db.ExecContext(ctx, query, settlement.attempt, settlement.availableAt, runErr.Error(), databasePendingRecoveryToken(job, settlement), now, id, processingToken) + if err != nil { + return err + } + return requireDatabaseSettlementRow(result) } -func (d *databaseQueue) acquireUnique(ctx context.Context, job queue.Job, queueName string, expiresAt time.Time) (bool, error) { - now := time.Now().UnixMilli() - expiresAtMillis := expiresAt.UnixMilli() - key := uniqueJobKey(job, queueName) - insert := d.rebind(`INSERT INTO queue_unique_locks(lock_key, expires_at) VALUES(?, ?)`) - _, err := d.db.ExecContext(ctx, insert, key, expiresAtMillis) - if err == nil { - return true, nil +// restoreRecoveredSettlementLineage immediately returns an unsuccessfully +// finalized recovery delivery to the pending set without replacing the receipt +// owner's inherited generation or advancing the application attempt. +func (d *databaseQueue) restoreRecoveredSettlementLineage(ctx context.Context, job *dbJob, settlementErr error) error { + query := d.rebind(`UPDATE queue_jobs + SET state='pending', available_at=?, processing_started_at=NULL, + processing_token=?, last_error=?, updated_at=? + WHERE id=? AND state='processing' AND processing_token=? AND attempt=?`) + now := time.Now() + availableAt := now.Add(databaseSettlementRecoveryDelay(d.cfg.PollInterval)) + return restoreDatabaseSettlementLineage(ctx, d.db, query, job, settlementErr, availableAt.UnixMilli(), now.UnixMilli()) +} + +// restoreDatabaseSettlementLineage applies the fenced repair through an +// injectable executor so ownership, attempt, and no-op cases can be tested +// without weakening the databaseQueue's concrete connection contract. +func restoreDatabaseSettlementLineage(ctx context.Context, execer databaseExecer, query string, job *dbJob, settlementErr error, availableAtMillis, nowMillis int64) error { + recoveryToken, repair, err := databaseSettlementRecoveryToken(job) + if err != nil || !repair { + return err + } + if execer == nil { + return errors.New("database settlement recovery executor is nil") + } + if settlementErr == nil { + return errors.New("database settlement recovery error is nil") + } + id, processingToken, err := databaseProcessingClaim(job) + if err != nil { + return err + } + result, err := execer.ExecContext(ctx, query, availableAtMillis, recoveryToken, settlementErr.Error(), nowMillis, id, processingToken, job.attempt) + if err != nil { + return err + } + return requireDatabaseSettlementRow(result) +} + +// databaseSettlementRecoveryDelay prevents a persistent physical-settlement +// fault from immediately reclaiming the same repaired row in a tight loop. +func databaseSettlementRecoveryDelay(pollInterval time.Duration) time.Duration { + if pollInterval > databaseFinalizeRetryDelay { + return pollInterval + } + return databaseFinalizeRetryDelay +} + +// databaseSettlementRecoveryToken selects inherited recovery proof only when +// the current generation did not itself commit application state. +func databaseSettlementRecoveryToken(job *dbJob) (sql.NullString, bool, error) { + if job == nil || !job.recovered || job.applicationStateCommitted { + return sql.NullString{}, false, nil + } + if job.recoveryToken == "" { + return sql.NullString{String: databaseRecoveryMarker, Valid: true}, true, nil + } + if !databaseProcessingTokenValid(job.recoveryToken) { + return sql.NullString{}, false, fmt.Errorf("recovered database settlement generation %q is invalid", job.recoveryToken) + } + return sql.NullString{String: job.recoveryToken, Valid: true}, true, nil +} + +// databasePendingRecoveryToken preserves the current generation after it +// durably mutates application state; otherwise it retains inherited recovery +// proof only across same-attempt infrastructure redelivery. +func databasePendingRecoveryToken(job *dbJob, settlement databaseFailureSettlement) sql.NullString { + if job == nil || settlement.state != "pending" || settlement.attempt != job.attempt { + return sql.NullString{} + } + if job.applicationStateCommitted { + if !databaseProcessingTokenValid(job.processingToken) { + return sql.NullString{} + } + return sql.NullString{String: job.processingToken, Valid: true} + } + if !job.recovered { + return sql.NullString{} + } + if job.recoveryToken == "" { + return sql.NullString{String: databaseRecoveryMarker, Valid: true} + } + if !databaseProcessingTokenValid(job.recoveryToken) { + return sql.NullString{} + } + return sql.NullString{String: job.recoveryToken, Valid: true} +} + +// databaseProcessingClaim returns the fenced identity required to settle one exact processing generation. +func databaseProcessingClaim(job *dbJob) (int64, string, error) { + if job == nil { + return 0, "", fmt.Errorf("database settlement job is nil") + } + if job.id <= 0 { + return 0, "", fmt.Errorf("database settlement job id must be positive") } - if !isUniqueConstraintErr(err) { + if job.processingToken == "" { + return 0, "", fmt.Errorf("database settlement processing token is empty") + } + return job.id, job.processingToken, nil +} + +// newDatabaseProcessingToken creates one opaque physical generation identity +// that also fences settlement updates for the current claim. +func newDatabaseProcessingToken() (string, error) { + var token [databaseProcessingTokenBytes]byte + if _, err := rand.Read(token[:]); err != nil { + return "", fmt.Errorf("create database processing token: %w", err) + } + return hex.EncodeToString(token[:]), nil +} + +// databaseProcessingTokenValid accepts only the canonical lowercase encoding +// generated for fenced SQL claims. +func databaseProcessingTokenValid(token string) bool { + if len(token) != databaseProcessingTokenBytes*2 || token != strings.ToLower(token) { + return false + } + decoded, err := hex.DecodeString(token) + if err != nil || len(decoded) != databaseProcessingTokenBytes { + return false + } + return true +} + +// requireDatabaseSettlementRow rejects stale or lost finalization updates that cannot prove ownership of one delivery. +func requireDatabaseSettlementRow(result sql.Result) error { + if result == nil { + return fmt.Errorf("database settlement returned no result") + } + rows, err := result.RowsAffected() + if err != nil { + return fmt.Errorf("read database settlement rows: %w", err) + } + if rows != 1 { + return fmt.Errorf("database settlement affected %d rows, want 1", rows) + } + return nil +} + +// observeSettlementFailure emits the physical delivery identity whose durable row could not be finalized. +func (d *databaseQueue) observeSettlementFailure(ctx context.Context, job *dbJob, err error) { + if job == nil { + return + } + metadata := queue.ResolveObservedJobMetadataFromJob(databaseDeliveryJob(job)) + queuecore.SafeObserve(ctx, d.observer, queue.Event{ + Kind: queue.EventSettlementFailed, + Driver: queue.DriverDatabase, + Queue: queuecore.NormalizeQueueName(job.queueName), + JobType: metadata.JobType, + JobKey: metadata.JobKey, + DispatchID: metadata.DispatchID, + JobID: metadata.JobID, + ChainID: metadata.ChainID, + BatchID: metadata.BatchID, + Attempt: job.attempt, + MaxRetry: job.maxRetry, + Err: err, + Time: time.Now(), + }) +} + +// classifyDatabaseFailure derives the durable state transition from the physical attempt and handler result. +func classifyDatabaseFailure(job *dbJob, runErr error, now int64) (databaseFailureSettlement, error) { + decision := busruntime.ClassifyAttempt(busruntime.DeliveryAttempt{ + Number: job.attempt, + MaxRetry: job.maxRetry, + }, runErr) + + switch decision { + case busruntime.AttemptRetry: + return databasePendingSettlement(job, job.attempt+1, now), nil + case busruntime.AttemptFailed: + return databaseFailureSettlement{state: "dead", attempt: job.attempt + 1}, nil + case busruntime.AttemptRedeliver: + return databasePendingSettlement(job, job.attempt, now), nil + default: + return databaseFailureSettlement{}, fmt.Errorf("cannot persist a successful attempt as failed") + } +} + +// databasePendingSettlement applies configured backoff without deciding whether the application retry counter advances. +func databasePendingSettlement(job *dbJob, attempt int, now int64) databaseFailureSettlement { + availableAt := now + if job.backoffMillis > 0 { + availableAt += job.backoffMillis + } + return databaseFailureSettlement{ + state: "pending", + attempt: attempt, + availableAt: availableAt, + } +} + +// acquireUnique claims the exact historical physical identity and canonical +// logical identity in the queue-row transaction so stable legacy claims remain effective. +func (d *databaseQueue) acquireUnique(ctx context.Context, tx *sql.Tx, job queue.Job, queueName string, ttl time.Duration) (bool, error) { + now, err := d.databaseNowMillis(ctx, tx) + if err != nil { return false, err } + if d.uniqueClaims.Add(1)%databaseUniquePruneInterval == 0 { + if err := d.pruneExpiredUniqueLocks(ctx, tx, now); err != nil { + return false, err + } + } + ttlMillis := ttl.Milliseconds() + if ttlMillis < 1 { + ttlMillis = 1 + } + expiresAt := now + ttlMillis + keys := [...]string{legacyUniqueJobKey(job, queueName), uniqueJobKey(job, queueName)} + for _, key := range keys { + acquired, acquireErr := d.acquireUniqueKey(ctx, tx, key, now, expiresAt) + if acquireErr != nil || !acquired { + return false, acquireErr + } + } + return true, nil +} + +// uniqueJobKey preserves the shared versioned identity verbatim for diagnosable SQL state. +func uniqueJobKey(job queue.Job, queueName string) string { + return queuecore.UniqueKey(job, queueName) +} +// legacyUniqueJobKey reproduces the unversioned SQL identity so a rolling upgrade honors outstanding claims from older producers. +func legacyUniqueJobKey(job queue.Job, queueName string) string { + digest := sha256.Sum256(append([]byte(queueName+":"+job.Type+":"), job.PayloadBytes()...)) + return hex.EncodeToString(digest[:]) +} + +// acquireUniqueKey couples one lock claim to the surrounding queue-row transaction. +func (d *databaseQueue) acquireUniqueKey(ctx context.Context, tx *sql.Tx, key string, now, expiresAt int64) (bool, error) { + insert := `INSERT INTO queue_unique_locks(lock_key, expires_at) VALUES(?, ?) ON CONFLICT(lock_key) DO NOTHING` + if d.cfg.DriverName != "pgx" && d.cfg.DriverName != "postgres" && d.cfg.DriverName != "sqlite" { + insert = `INSERT IGNORE INTO queue_unique_locks(lock_key, expires_at) VALUES(?, ?)` + } + res, err := tx.ExecContext(ctx, d.rebind(insert), key, expiresAt) + if err != nil { + return false, err + } + if rows, rowsErr := res.RowsAffected(); rowsErr == nil && rows == 1 { + return true, nil + } update := d.rebind(`UPDATE queue_unique_locks SET expires_at=? WHERE lock_key=? AND expires_at <= ?`) - res, err := d.db.ExecContext(ctx, update, expiresAtMillis, key, now) + res, err = tx.ExecContext(ctx, update, expiresAt, key, now) if err != nil { return false, err } - rows, _ := res.RowsAffected() - return rows == 1, nil + rows, err := res.RowsAffected() + return rows == 1, err } -func uniqueJobKey(job queue.Job, queueName string) string { - hash := sha256.Sum256(append([]byte(queueName+":"+job.Type+":"), job.PayloadBytes()...)) - return hex.EncodeToString(hash[:]) +// databaseNowMillis reads the backend clock so producer clock skew cannot shorten or extend distributed claims. +func (d *databaseQueue) databaseNowMillis(ctx context.Context, queryer databaseRowQueryer) (int64, error) { + query := `SELECT CAST(UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000 AS SIGNED)` + switch d.cfg.DriverName { + case "pgx", "postgres": + query = `SELECT CAST(EXTRACT(EPOCH FROM clock_timestamp()) * 1000 AS BIGINT)` + case "sqlite": + query = `SELECT CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER)` + } + var now int64 + if err := queryer.QueryRowContext(ctx, query).Scan(&now); err != nil { + return 0, fmt.Errorf("read database time for uniqueness: %w", err) + } + return now, nil } -func isUniqueConstraintErr(err error) bool { - if err == nil { - return false +// pruneExpiredUniqueLocks bounds persistent identity state without touching live claims. +func (d *databaseQueue) pruneExpiredUniqueLocks(ctx context.Context, execer databaseExecer, now int64) error { + query := d.rebind(`DELETE FROM queue_unique_locks WHERE expires_at <= ?`) + if _, err := execer.ExecContext(ctx, query, now); err != nil { + return fmt.Errorf("prune expired uniqueness claims: %w", err) } - msg := strings.ToLower(err.Error()) - return strings.Contains(msg, "duplicate") || - strings.Contains(msg, "unique constraint") || - strings.Contains(msg, "unique violation") + return nil } func (d *databaseQueue) ensureSchema(ctx context.Context) error { @@ -799,9 +1359,258 @@ func (d *databaseQueue) ensureSchema(ctx context.Context) error { return fmt.Errorf("ensure queue schema failed: %w", err) } } + if err := d.ensureProcessingTokenColumn(ctx); err != nil { + return err + } + if err := d.ensureMetadataJSONColumn(ctx); err != nil { + return err + } + if d.cfg.DriverName == "mysql" { + if err := d.ensureMySQLUniqueExpiryIndex(ctx); err != nil { + return err + } + } + now, err := d.databaseNowMillis(ctx, d.db) + if err != nil { + return err + } + return d.pruneExpiredUniqueLocks(ctx, d.db, now) +} + +// ensureProcessingTokenColumn upgrades existing queue tables additively while nullable storage keeps older binaries and rows readable. +func (d *databaseQueue) ensureProcessingTokenColumn(ctx context.Context) error { + exists, err := d.processingTokenColumnExists(ctx) + if err != nil { + return err + } + if exists { + return nil + } + statement := `ALTER TABLE queue_jobs ADD COLUMN processing_token VARCHAR(64) NULL` + switch d.cfg.DriverName { + case "pgx", "postgres", "sqlite": + statement = `ALTER TABLE queue_jobs ADD COLUMN processing_token TEXT NULL` + } + if _, err := d.db.ExecContext(ctx, statement); err != nil { + // Concurrent startup may observe an already-completed additive migration after its own ALTER loses the race. + exists, checkErr := d.processingTokenColumnExists(ctx) + if checkErr == nil && exists { + return nil + } + return fmt.Errorf("ensure database processing token column: %w", err) + } + return nil +} + +// processingTokenColumnExists inspects the active dialect without relying on non-portable ALTER TABLE guards. +func (d *databaseQueue) processingTokenColumnExists(ctx context.Context) (bool, error) { + return d.queueJobColumnExists(ctx, "processing_token") +} + +// ensureMetadataJSONColumn upgrades legacy queue tables before direct jobs can +// rely on out-of-payload correlation surviving a durable delivery. +func (d *databaseQueue) ensureMetadataJSONColumn(ctx context.Context) error { + exists, err := d.metadataJSONColumnExists(ctx) + if err != nil { + return err + } + if exists { + return nil + } + if _, err := d.db.ExecContext(ctx, `ALTER TABLE queue_jobs ADD COLUMN metadata_json TEXT NULL`); err != nil { + // Concurrent startup may observe an already-completed additive migration after its own ALTER loses the race. + exists, checkErr := d.metadataJSONColumnExists(ctx) + if checkErr == nil && exists { + return nil + } + return fmt.Errorf("ensure database job metadata column: %w", err) + } + return nil +} + +// requireManagedQueueSchema keeps readiness and worker startup aligned with +// every table and column that runtime SQL can touch without performing DDL. +func (d *databaseQueue) requireManagedQueueSchema(ctx context.Context) error { + tableExists, err := d.queueJobsTableExists(ctx) + if err != nil { + return fmt.Errorf("validate caller-managed queue_jobs table: %w", err) + } + if !tableExists { + return fmt.Errorf("caller-managed schema is missing required queue_jobs table") + } + jobColumns, err := d.managedQueueTableColumns(ctx, managedQueueJobsTable) + if err != nil { + return fmt.Errorf("validate caller-managed queue_jobs columns: %w", err) + } + for _, columnName := range managedQueueJobColumns { + if _, exists := jobColumns[columnName]; !exists { + return fmt.Errorf("caller-managed queue_jobs schema is missing required %s column", columnName) + } + } + + tableExists, err = d.queueUniqueLocksTableExists(ctx) + if err != nil { + return fmt.Errorf("validate caller-managed queue_unique_locks table: %w", err) + } + if !tableExists { + return fmt.Errorf("caller-managed schema is missing required queue_unique_locks table") + } + uniqueLockColumns, err := d.managedQueueTableColumns(ctx, managedQueueUniqueLocksTable) + if err != nil { + return fmt.Errorf("validate caller-managed queue_unique_locks columns: %w", err) + } + for _, columnName := range managedQueueUniqueLockColumns { + if _, exists := uniqueLockColumns[columnName]; !exists { + return fmt.Errorf("caller-managed queue_unique_locks schema is missing required %s column", columnName) + } + } + return nil +} + +// queueJobsTableExists reports whether the caller installed the durable job table. +func (d *databaseQueue) queueJobsTableExists(ctx context.Context) (bool, error) { + return d.managedQueueTableExists(ctx, managedQueueJobsTable) +} + +// queueUniqueLocksTableExists reports whether the caller installed the distributed uniqueness table. +func (d *databaseQueue) queueUniqueLocksTableExists(ctx context.Context) (bool, error) { + return d.managedQueueTableExists(ctx, managedQueueUniqueLocksTable) +} + +// managedQueueTableExists inspects one trusted runtime table name through the active dialect. +func (d *databaseQueue) managedQueueTableExists(ctx context.Context, tableName managedQueueTable) (bool, error) { + var count int + switch d.cfg.DriverName { + case "sqlite": + err := d.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?`, string(tableName)).Scan(&count) + return count > 0, err + case "pgx", "postgres": + err := d.db.QueryRowContext(ctx, d.rebind(`SELECT COUNT(*) FROM pg_class WHERE oid = to_regclass(?) AND relkind IN ('r', 'p')`), string(tableName)).Scan(&count) + return count > 0, err + default: + err := d.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM information_schema.tables WHERE table_schema = DATABASE() AND table_name = ? AND table_type = 'BASE TABLE'`, string(tableName)).Scan(&count) + return count > 0, err + } +} + +// managedQueueTableColumns reads one complete catalog snapshot so frequent +// readiness checks do not issue a separate database roundtrip per field. +func (d *databaseQueue) managedQueueTableColumns(ctx context.Context, tableName managedQueueTable) (map[string]struct{}, error) { + query := `SELECT column_name + FROM information_schema.columns + WHERE table_schema = DATABASE() AND table_name = ?` + switch d.cfg.DriverName { + case "sqlite": + query = `SELECT name FROM pragma_table_info(?)` + case "pgx", "postgres": + query = `SELECT attname + FROM pg_attribute + WHERE attrelid = to_regclass(?) AND attnum > 0 AND NOT attisdropped` + } + rows, err := d.db.QueryContext(ctx, d.rebind(query), string(tableName)) + if err != nil { + return nil, fmt.Errorf("inspect %s columns: %w", tableName, err) + } + defer rows.Close() + + columns := make(map[string]struct{}) + for rows.Next() { + var columnName string + if err := rows.Scan(&columnName); err != nil { + return nil, fmt.Errorf("scan %s column: %w", tableName, err) + } + columns[columnName] = struct{}{} + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("inspect %s columns: %w", tableName, err) + } + return columns, nil +} + +// metadataJSONColumnExists reports whether direct-delivery metadata has an +// additive persistence slot in the active queue table. +func (d *databaseQueue) metadataJSONColumnExists(ctx context.Context) (bool, error) { + return d.queueJobColumnExists(ctx, "metadata_json") +} + +// queueJobColumnExists inspects one trusted queue_jobs column name through the +// active dialect without depending on non-portable ALTER TABLE guards. +func (d *databaseQueue) queueJobColumnExists(ctx context.Context, columnName string) (bool, error) { + if d.cfg.DriverName == "sqlite" { + rows, err := d.db.QueryContext(ctx, `PRAGMA table_info(queue_jobs)`) + if err != nil { + return false, fmt.Errorf("inspect sqlite queue job column %q: %w", columnName, err) + } + defer rows.Close() + for rows.Next() { + var ( + columnID int + name string + columnType string + notNull int + defaultValue sql.NullString + primaryKey int + ) + if err := rows.Scan(&columnID, &name, &columnType, ¬Null, &defaultValue, &primaryKey); err != nil { + return false, fmt.Errorf("scan sqlite queue column: %w", err) + } + if name == columnName { + return true, nil + } + } + if err := rows.Err(); err != nil { + return false, fmt.Errorf("inspect sqlite queue columns: %w", err) + } + return false, nil + } + + query := `SELECT COUNT(*) + FROM information_schema.columns + WHERE table_schema = DATABASE() AND table_name = 'queue_jobs' AND column_name = ?` + if d.cfg.DriverName == "pgx" || d.cfg.DriverName == "postgres" { + query = `SELECT COUNT(*) + FROM pg_attribute + WHERE attrelid = to_regclass('queue_jobs') AND attname = ? AND NOT attisdropped` + } + var count int + if err := d.db.QueryRowContext(ctx, d.rebind(query), columnName).Scan(&count); err != nil { + return false, fmt.Errorf("inspect database queue job column %q: %w", columnName, err) + } + return count > 0, nil +} + +// ensureMySQLUniqueExpiryIndex migrates existing lock tables whose original CREATE TABLE predates expiry pruning. +func (d *databaseQueue) ensureMySQLUniqueExpiryIndex(ctx context.Context) error { + exists, err := d.mysqlIndexExists(ctx, "idx_queue_unique_locks_expires") + if err != nil { + return err + } + if exists { + return nil + } + if _, err := d.db.ExecContext(ctx, `ALTER TABLE queue_unique_locks ADD INDEX idx_queue_unique_locks_expires (expires_at)`); err != nil { + // Multiple producers may migrate concurrently, so a successful peer wins even if this ALTER observed the race. + exists, checkErr := d.mysqlIndexExists(ctx, "idx_queue_unique_locks_expires") + if checkErr == nil && exists { + return nil + } + return fmt.Errorf("ensure mysql uniqueness expiry index: %w", err) + } return nil } +// mysqlIndexExists checks the active schema instead of relying on version-specific CREATE INDEX syntax. +func (d *databaseQueue) mysqlIndexExists(ctx context.Context, indexName string) (bool, error) { + const query = `SELECT COUNT(*) +FROM information_schema.statistics +WHERE table_schema = DATABASE() AND table_name = 'queue_unique_locks' AND index_name = ?` + var count int + if err := d.db.QueryRowContext(ctx, query, indexName).Scan(&count); err != nil { + return false, fmt.Errorf("inspect mysql uniqueness expiry index: %w", err) + } + return count > 0, nil +} + func (d *databaseQueue) schemaStatements() []string { switch d.cfg.DriverName { case "pgx", "postgres": @@ -811,13 +1620,15 @@ func (d *databaseQueue) schemaStatements() []string { queue_name TEXT NOT NULL, job_type TEXT NOT NULL, payload BYTEA NOT NULL, + metadata_json TEXT NULL, timeout_seconds BIGINT NULL, max_retry INTEGER NOT NULL DEFAULT 0, backoff_millis BIGINT NOT NULL DEFAULT 0, attempt INTEGER NOT NULL DEFAULT 0, - available_at BIGINT NOT NULL, - processing_started_at BIGINT NULL, - last_error TEXT NULL, + available_at BIGINT NOT NULL, + processing_started_at BIGINT NULL, + processing_token TEXT NULL, + last_error TEXT NULL, state TEXT NOT NULL, created_at BIGINT NOT NULL, updated_at BIGINT NOT NULL @@ -827,6 +1638,7 @@ func (d *databaseQueue) schemaStatements() []string { lock_key TEXT PRIMARY KEY, expires_at BIGINT NOT NULL )`, + `CREATE INDEX IF NOT EXISTS idx_queue_unique_locks_expires ON queue_unique_locks(expires_at)`, } case "sqlite": return []string{ @@ -835,13 +1647,15 @@ func (d *databaseQueue) schemaStatements() []string { queue_name TEXT NOT NULL, job_type TEXT NOT NULL, payload BLOB NOT NULL, + metadata_json TEXT NULL, timeout_seconds INTEGER NULL, max_retry INTEGER NOT NULL DEFAULT 0, backoff_millis INTEGER NOT NULL DEFAULT 0, attempt INTEGER NOT NULL DEFAULT 0, - available_at INTEGER NOT NULL, - processing_started_at INTEGER NULL, - last_error TEXT NULL, + available_at INTEGER NOT NULL, + processing_started_at INTEGER NULL, + processing_token TEXT NULL, + last_error TEXT NULL, state TEXT NOT NULL, created_at INTEGER NOT NULL, updated_at INTEGER NOT NULL @@ -851,6 +1665,7 @@ func (d *databaseQueue) schemaStatements() []string { lock_key TEXT PRIMARY KEY, expires_at INTEGER NOT NULL )`, + `CREATE INDEX IF NOT EXISTS idx_queue_unique_locks_expires ON queue_unique_locks(expires_at)`, } default: return []string{ @@ -859,22 +1674,24 @@ func (d *databaseQueue) schemaStatements() []string { queue_name VARCHAR(191) NOT NULL, job_type VARCHAR(191) NOT NULL, payload LONGBLOB NOT NULL, + metadata_json TEXT NULL, timeout_seconds BIGINT NULL, max_retry INT NOT NULL DEFAULT 0, backoff_millis BIGINT NOT NULL DEFAULT 0, attempt INT NOT NULL DEFAULT 0, - available_at BIGINT NOT NULL, - processing_started_at BIGINT NULL, - last_error TEXT NULL, + available_at BIGINT NOT NULL, + processing_started_at BIGINT NULL, + processing_token VARCHAR(64) NULL, + last_error TEXT NULL, state VARCHAR(16) NOT NULL, created_at BIGINT NOT NULL, updated_at BIGINT NOT NULL, KEY idx_queue_jobs_ready (state, available_at, id) )`, `CREATE TABLE IF NOT EXISTS queue_unique_locks ( - lock_key VARCHAR(255) NOT NULL PRIMARY KEY, - expires_at BIGINT NOT NULL - )`, + lock_key VARCHAR(255) NOT NULL PRIMARY KEY, + expires_at BIGINT NOT NULL + )`, } } } @@ -886,27 +1703,6 @@ func defaultWorkerCount(n int) int { return n } -func waitGroupWithContext(ctx context.Context, wg *sync.WaitGroup) error { - if wg == nil { - return nil - } - done := make(chan struct{}) - go func() { - defer close(done) - wg.Wait() - }() - if ctx == nil { - <-done - return nil - } - select { - case <-done: - return nil - case <-ctx.Done(): - return ctx.Err() - } -} - func (d *databaseQueue) rebind(query string) string { if d.cfg.DriverName != "pgx" && d.cfg.DriverName != "postgres" { return query diff --git a/driver/sqlqueuecore/queue_database_impl_test.go b/driver/sqlqueuecore/queue_database_impl_test.go new file mode 100644 index 0000000..4062c9c --- /dev/null +++ b/driver/sqlqueuecore/queue_database_impl_test.go @@ -0,0 +1,856 @@ +package sqlqueuecore + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "errors" + "runtime" + "strings" + "testing" + "time" + + "github.com/goforj/queue" + "github.com/goforj/queue/busruntime" + "github.com/goforj/queue/queuecore" +) + +// TestDatabaseStartAfterShutdownRejectsFalseRestart verifies direct core callers cannot receive success after workers and resources stopped. +func TestDatabaseStartAfterShutdownRejectsFalseRestart(t *testing.T) { + database := &databaseQueue{} + database.started.Store(true) + database.shuttingDown.Store(true) + if err := database.StartWorkers(context.Background()); !errors.Is(err, queue.ErrQueuerShuttingDown) { + t.Fatalf("start after shutdown = %v, want ErrQueuerShuttingDown", err) + } +} + +// TestDatabaseShutdownRetriesShareOneDrain verifies caller deadlines do not +// multiply waiter goroutines and later cleanup reports close diagnostics once +// before converging. +func TestDatabaseShutdownRetriesShareOneDrain(t *testing.T) { + closeErr := errors.New("close database") + connection := &databaseConnStub{closeErr: closeErr} + db := newDatabaseStub(connection) + if err := db.PingContext(context.Background()); err != nil { + t.Fatalf("open database connection: %v", err) + } + + database := &databaseQueue{ + db: db, + ownsDB: true, + shutdownCh: make(chan struct{}), + } + releaseWorker := make(chan struct{}) + database.workerWG.Add(1) + go func() { + defer database.workerWG.Done() + <-releaseWorker + }() + + shutdownWithDeadline := func() { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond) + defer cancel() + if err := database.Shutdown(ctx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("timed shutdown = %v, want context deadline exceeded", err) + } + } + + shutdownWithDeadline() + sharedDone := database.shutdownDone + if sharedDone == nil { + t.Fatal("shutdown did not retain its drain completion channel") + } + goroutinesAfterFirstDeadline := runtime.NumGoroutine() + for range 32 { + shutdownWithDeadline() + if database.shutdownDone != sharedDone { + t.Fatal("shutdown retry replaced the shared drain completion channel") + } + } + runtime.Gosched() + if got := runtime.NumGoroutine(); got > goroutinesAfterFirstDeadline+2 { + t.Fatalf("shutdown retries grew goroutines from %d to %d", goroutinesAfterFirstDeadline, got) + } + if connection.closeCalls != 0 { + t.Fatalf("database close calls before worker drain = %d, want 0", connection.closeCalls) + } + + close(releaseWorker) + if err := database.Shutdown(context.Background()); !errors.Is(err, closeErr) { + t.Fatalf("converged shutdown = %v, want %v", err, closeErr) + } + if err := database.Shutdown(context.Background()); err != nil { + t.Fatalf("repeated converged shutdown = %v, want nil after diagnostic was reported", err) + } + if connection.closeCalls != 1 { + t.Fatalf("database close calls = %d, want 1", connection.closeCalls) + } +} + +// TestDatabaseDrainWorkersSeparatesResourceOwnership verifies native root +// shutdown can join workers before deciding whether it owns the database close. +func TestDatabaseDrainWorkersSeparatesResourceOwnership(t *testing.T) { + tests := []struct { + name string + ownsDB bool + }{ + {name: "owned database", ownsDB: true}, + {name: "caller-owned database", ownsDB: false}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + connection := &databaseConnStub{} + db := newDatabaseStub(connection) + if err := db.PingContext(context.Background()); err != nil { + t.Fatalf("open database connection: %v", err) + } + database := &databaseQueue{ + db: db, + ownsDB: test.ownsDB, + shutdownCh: make(chan struct{}), + } + + if err := database.DrainWorkers(context.Background()); err != nil { + t.Fatalf("drain workers: %v", err) + } + if connection.closeCalls != 0 { + t.Fatalf("database close calls during worker drain = %d, want 0", connection.closeCalls) + } + if err := db.PingContext(context.Background()); err != nil { + t.Fatalf("database unavailable after worker drain: %v", err) + } + if err := database.Shutdown(context.Background()); err != nil { + t.Fatalf("shutdown database queue: %v", err) + } + wantCloses := 0 + if test.ownsDB { + wantCloses = 1 + } + if connection.closeCalls != wantCloses { + t.Fatalf("database close calls after shutdown = %d, want %d", connection.closeCalls, wantCloses) + } + if err := database.Shutdown(context.Background()); err != nil { + t.Fatalf("repeat database shutdown: %v", err) + } + if connection.closeCalls != wantCloses { + t.Fatalf("database close calls after repeat = %d, want %d", connection.closeCalls, wantCloses) + } + if !test.ownsDB { + if err := db.PingContext(context.Background()); err != nil { + t.Fatalf("caller-owned database unavailable after shutdown: %v", err) + } + if err := db.Close(); err != nil { + t.Fatalf("close caller-owned database: %v", err) + } + } + }) + } +} + +// TestLocalDatabaseConfigDisableAutoMigrate verifies the additive opt-out preserves the established default while overriding legacy true values. +func TestLocalDatabaseConfigDisableAutoMigrate(t *testing.T) { + if normalized := (localDatabaseConfig{}).normalize(); !normalized.AutoMigrate { + t.Fatal("default configuration no longer enables compatibility migrations") + } + normalized := (localDatabaseConfig{AutoMigrate: true, DisableAutoMigrate: true}).normalize() + if normalized.AutoMigrate { + t.Fatal("DisableAutoMigrate did not override migration startup") + } +} + +// TestDatabaseContinuationPermissionIsScopedAndEphemeral verifies only this queue's active handler may dispatch during drain. +func TestDatabaseContinuationPermissionIsScopedAndEphemeral(t *testing.T) { + database := &databaseQueue{continuation: busruntime.NewContinuationScope()} + database.shuttingDown.Store(true) + invalidJob := queue.Job{} + + foreign := busruntime.NewContinuationScope() + foreignCtx, releaseForeign := foreign.Permit(context.Background()) + defer releaseForeign() + if err := database.Dispatch(foreignCtx, invalidJob); !errors.Is(err, queue.ErrQueuerShuttingDown) { + t.Fatalf("foreign continuation dispatch = %v, want ErrQueuerShuttingDown", err) + } + + var escaped context.Context + err := database.runHandlerWithContinuationPermit(context.Background(), func(ctx context.Context, _ queue.Job) error { + escaped = ctx + if !database.continuation.Owns(ctx) { + t.Fatal("active SQL handler did not own its continuation permit") + } + dispatchErr := database.Dispatch(ctx, invalidJob) + if errors.Is(dispatchErr, queue.ErrQueuerShuttingDown) || dispatchErr == nil { + t.Fatalf("owned continuation dispatch = %v, want validation error after shutdown gate", dispatchErr) + } + return nil + }, invalidJob) + if err != nil { + t.Fatalf("run handler with continuation permit: %v", err) + } + if database.continuation.Owns(escaped) { + t.Fatal("handler context retained SQL continuation ownership after return") + } + if err := database.Dispatch(escaped, invalidJob); !errors.Is(err, queue.ErrQueuerShuttingDown) { + t.Fatalf("escaped continuation dispatch = %v, want ErrQueuerShuttingDown", err) + } +} + +type databaseResultStub struct { + rows int64 + err error +} + +type databaseExecerStub struct { + calls int + query string + args []any + result sql.Result + err error +} + +// LastInsertId returns an unused identifier for the sql.Result contract. +func (r databaseResultStub) LastInsertId() (int64, error) { return 0, nil } + +// RowsAffected returns the configured settlement evidence. +func (r databaseResultStub) RowsAffected() (int64, error) { return r.rows, r.err } + +// ExecContext records one recovery-lineage repair without requiring a live SQL +// driver in this dependency-light core module. +func (e *databaseExecerStub) ExecContext(_ context.Context, query string, args ...any) (sql.Result, error) { + e.calls++ + e.query = query + e.args = append([]any(nil), args...) + return e.result, e.err +} + +// TestDatabaseDeliveryJobRestoresAttemptMetadata verifies SQL persistence reaches the shared orchestration context intact. +func TestDatabaseDeliveryJobRestoresAttemptMetadata(t *testing.T) { + wantPayload := []byte(`{"report_id":42}`) + wantMetadata := queue.DriverJobMetadata{ + SchemaVersion: queue.DriverJobMetadataVersion, + DispatchID: "dsp_sql", + JobID: "job_sql", + ChainID: "chn_sql", + BatchID: "bat_sql", + Queue: "critical", + } + encodedMetadata, err := json.Marshal(wantMetadata) + if err != nil { + t.Fatalf("marshal metadata fixture: %v", err) + } + job := databaseDeliveryJob(&dbJob{ + jobType: "reports:build", + payload: wantPayload, + metadataJSON: sql.NullString{String: string(encodedMetadata), Valid: true}, + queueName: "critical", + attempt: 2, + maxRetry: 4, + }) + opts := queuecore.DriverOptions(job) + if job.Type != "reports:build" || !bytes.Equal(job.PayloadBytes(), wantPayload) { + t.Fatalf("delivery job = type:%q payload:%q", job.Type, job.PayloadBytes()) + } + if opts.QueueName != "critical" || opts.Attempt != 2 || opts.MaxRetry == nil || *opts.MaxRetry != 4 { + t.Fatalf("delivery options = %+v", opts) + } + if metadata := queue.DriverMetadata(job); metadata != wantMetadata { + t.Fatalf("delivery metadata = %+v, want %+v", metadata, wantMetadata) + } +} + +// TestDatabaseSettlementContextMarksOnlyRecoveredRows ensures an ordinary +// duplicate cannot request winner-fact replay without stale-processing proof. +func TestDatabaseSettlementContextMarksOnlyRecoveredRows(t *testing.T) { + tests := []struct { + name string + job *dbJob + want busruntime.DeliveryProvenance + wantPresent bool + }{ + {name: "nil"}, + { + name: "ordinary", + job: &dbJob{processingToken: "current-generation"}, + want: busruntime.DeliveryProvenance{GenerationID: "current-generation"}, + wantPresent: true, + }, + { + name: "identified recovery", + job: &dbJob{ + processingToken: "current-generation", + recoveryToken: "earlier-generation", + recovered: true, + }, + want: busruntime.DeliveryProvenance{ + GenerationID: "current-generation", + RecoveredGenerationID: "earlier-generation", + Recovered: true, + }, + wantPresent: true, + }, + { + name: "legacy recovery", + job: &dbJob{processingToken: "current-generation", recovered: true}, + want: busruntime.DeliveryProvenance{GenerationID: "current-generation", Recovered: true}, + wantPresent: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx, settlement := databaseSettlementContext(test.job) + if settlement == nil { + t.Fatal("database settlement context omitted commit boundary") + } + got, present := busruntime.DeliveryProvenanceFromContext(ctx) + if present != test.wantPresent || got != test.want { + t.Fatalf("delivery provenance = %+v present:%t, want %+v/%t", got, present, test.want, test.wantPresent) + } + }) + } +} + +// TestDatabaseRecoveryProofUsesOnlyTransportState verifies application error +// text cannot collide with the internal stale-processing recovery marker. +func TestDatabaseRecoveryProofUsesOnlyTransportState(t *testing.T) { + tests := []struct { + name string + processingToken sql.NullString + lastError sql.NullString + wantToken string + want bool + }{ + { + name: "application error matches marker", + lastError: sql.NullString{String: databaseRecoveryMarker, Valid: true}, + }, + { + name: "ordinary processing token", + processingToken: sql.NullString{String: "ordinary-claim", Valid: true}, + lastError: sql.NullString{String: databaseRecoveryMarker, Valid: true}, + }, + { + name: "transport recovery marker", + processingToken: sql.NullString{String: databaseRecoveryMarker, Valid: true}, + lastError: sql.NullString{String: databaseRecoveryDiagnostic, Valid: true}, + want: true, + }, + { + name: "identified transport generation", + processingToken: sql.NullString{String: strings.Repeat("a", databaseProcessingTokenBytes*2), Valid: true}, + wantToken: strings.Repeat("a", databaseProcessingTokenBytes*2), + want: true, + }, + { + name: "uppercase generation is not canonical", + processingToken: sql.NullString{String: strings.Repeat("A", databaseProcessingTokenBytes*2), Valid: true}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + token, recovered := databaseRecoveryProof(test.processingToken) + if recovered != test.want || token != test.wantToken { + t.Fatalf("recovery proof = token:%q recovered:%t, want %q/%t (processing_token=%q, last_error=%q)", token, recovered, test.wantToken, test.want, test.processingToken.String, test.lastError.String) + } + }) + } +} + +// TestDatabasePendingRecoveryTokenPreservesPendingRecovery verifies the exact +// durable owner survives same-attempt infrastructure redelivery. +func TestDatabasePendingRecoveryTokenPreservesPendingRecovery(t *testing.T) { + const ( + recoveryToken = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + currentToken = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + ) + tests := []struct { + name string + job *dbJob + settlement databaseFailureSettlement + wantToken string + }{ + {name: "nil job", settlement: databaseFailureSettlement{state: "pending"}}, + { + name: "ordinary same attempt redelivery", + job: &dbJob{attempt: 2}, + settlement: databaseFailureSettlement{state: "pending", attempt: 2}, + }, + { + name: "recovered same attempt redelivery", + job: &dbJob{attempt: 2, recovered: true, recoveryToken: recoveryToken, processingToken: currentToken}, + settlement: databaseFailureSettlement{state: "pending", attempt: 2}, + wantToken: recoveryToken, + }, + { + name: "legacy same attempt redelivery", + job: &dbJob{attempt: 2, recovered: true, processingToken: currentToken}, + settlement: databaseFailureSettlement{state: "pending", attempt: 2}, + wantToken: databaseRecoveryMarker, + }, + { + name: "recovered current generation committed application state", + job: &dbJob{attempt: 2, recovered: true, recoveryToken: recoveryToken, processingToken: currentToken, applicationStateCommitted: true}, + settlement: databaseFailureSettlement{state: "pending", attempt: 2}, + wantToken: currentToken, + }, + { + name: "ordinary current generation committed application state", + job: &dbJob{attempt: 2, processingToken: currentToken, applicationStateCommitted: true}, + settlement: databaseFailureSettlement{state: "pending", attempt: 2}, + wantToken: currentToken, + }, + { + name: "application retry starts a new owner", + job: &dbJob{attempt: 2, recovered: true, recoveryToken: recoveryToken, processingToken: currentToken, applicationStateCommitted: true}, + settlement: databaseFailureSettlement{state: "pending", attempt: 3}, + }, + { + name: "recovered terminal failure", + job: &dbJob{attempt: 2, recovered: true, recoveryToken: recoveryToken, processingToken: currentToken}, + settlement: databaseFailureSettlement{state: "dead", attempt: 3}, + }, + { + name: "malformed recovered generation", + job: &dbJob{attempt: 2, recovered: true, recoveryToken: "malformed", processingToken: currentToken}, + settlement: databaseFailureSettlement{state: "pending", attempt: 2}, + }, + { + name: "malformed committed current generation", + job: &dbJob{attempt: 2, processingToken: "malformed", applicationStateCommitted: true}, + settlement: databaseFailureSettlement{state: "pending", attempt: 2}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + token := databasePendingRecoveryToken(test.job, test.settlement) + if token.Valid != (test.wantToken != "") || token.String != test.wantToken { + t.Fatalf("recovery token = %#v, want %q", token, test.wantToken) + } + }) + } +} + +// TestDatabaseSettlementRecoveryTokenRepairsOnlyInheritedLineage verifies an +// exhausted recovery settlement never replaces a receipt owner with the current +// physical generation or repairs an ordinary first delivery. +func TestDatabaseSettlementRecoveryTokenRepairsOnlyInheritedLineage(t *testing.T) { + const ( + recoveryToken = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + currentToken = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + ) + tests := []struct { + name string + job *dbJob + wantToken string + wantRepair bool + wantError bool + }{ + {name: "nil job"}, + {name: "ordinary delivery", job: &dbJob{processingToken: currentToken}}, + { + name: "identified inherited owner", + job: &dbJob{recovered: true, recoveryToken: recoveryToken, processingToken: currentToken}, + wantToken: recoveryToken, + wantRepair: true, + }, + { + name: "legacy inherited marker", + job: &dbJob{recovered: true, processingToken: currentToken}, + wantToken: databaseRecoveryMarker, + wantRepair: true, + }, + { + name: "current generation superseded provenance", + job: &dbJob{ + recovered: true, + recoveryToken: recoveryToken, + processingToken: currentToken, + applicationStateCommitted: true, + }, + }, + { + name: "malformed inherited owner", + job: &dbJob{recovered: true, recoveryToken: "malformed", processingToken: currentToken}, + wantError: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + token, repair, err := databaseSettlementRecoveryToken(test.job) + if (err != nil) != test.wantError { + t.Fatalf("databaseSettlementRecoveryToken() error = %v, wantError %t", err, test.wantError) + } + if repair != test.wantRepair || token.Valid != test.wantRepair || token.String != test.wantToken { + t.Fatalf("databaseSettlementRecoveryToken() = %#v repair:%t, want %q/%t", token, repair, test.wantToken, test.wantRepair) + } + }) + } +} + +// TestRestoreDatabaseSettlementLineageFencesPendingRepair verifies the repair +// targets one exact processing generation while preserving attempt and owner. +func TestRestoreDatabaseSettlementLineageFencesPendingRepair(t *testing.T) { + const ( + query = "fenced recovery update" + recoveryToken = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + currentToken = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + nowMillis = int64(123456) + availableAtMillis = int64(123531) + ) + job := &dbJob{ + id: 41, + attempt: 3, + recovered: true, + recoveryToken: recoveryToken, + processingToken: currentToken, + } + settlementErr := errors.New("delete remained unavailable") + execer := &databaseExecerStub{result: databaseResultStub{rows: 1}} + if err := restoreDatabaseSettlementLineage(context.Background(), execer, query, job, settlementErr, availableAtMillis, nowMillis); err != nil { + t.Fatalf("restore database settlement lineage: %v", err) + } + if execer.calls != 1 || execer.query != query { + t.Fatalf("repair execution = calls:%d query:%q, want 1/%q", execer.calls, execer.query, query) + } + wantArgs := []any{ + availableAtMillis, + sql.NullString{String: recoveryToken, Valid: true}, + settlementErr.Error(), + nowMillis, + job.id, + currentToken, + job.attempt, + } + if len(execer.args) != len(wantArgs) { + t.Fatalf("repair argument count = %d, want %d: %#v", len(execer.args), len(wantArgs), execer.args) + } + for index := range wantArgs { + if execer.args[index] != wantArgs[index] { + t.Fatalf("repair argument %d = %#v, want %#v", index, execer.args[index], wantArgs[index]) + } + } + if job.attempt != 3 || job.processingToken != currentToken || job.recoveryToken != recoveryToken { + t.Fatalf("repair mutated in-memory claim: %+v", job) + } +} + +// TestDatabaseSettlementRecoveryDelayBoundsFaultLoop verifies repaired rows +// honor the slower of queue polling and the driver's finalization retry floor. +func TestDatabaseSettlementRecoveryDelayBoundsFaultLoop(t *testing.T) { + tests := []struct { + name string + pollInterval time.Duration + want time.Duration + }{ + {name: "zero poll interval", want: databaseFinalizeRetryDelay}, + {name: "short poll interval", pollInterval: time.Millisecond, want: databaseFinalizeRetryDelay}, + {name: "equal poll interval", pollInterval: databaseFinalizeRetryDelay, want: databaseFinalizeRetryDelay}, + {name: "long poll interval", pollInterval: 150 * time.Millisecond, want: 150 * time.Millisecond}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := databaseSettlementRecoveryDelay(test.pollInterval); got != test.want { + t.Fatalf("databaseSettlementRecoveryDelay(%s) = %s, want %s", test.pollInterval, got, test.want) + } + }) + } +} + +// TestRestoreDatabaseSettlementLineageRejectsUnprovableRepair covers every +// failure branch that must leave the currently fenced row untouched. +func TestRestoreDatabaseSettlementLineageRejectsUnprovableRepair(t *testing.T) { + const currentToken = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + settlementErr := errors.New("settlement failed") + tests := []struct { + name string + execer databaseExecer + job *dbJob + err error + wantError bool + wantCalls int + }{ + { + name: "ordinary delivery is unchanged", + execer: &databaseExecerStub{result: databaseResultStub{rows: 1}}, + job: &dbJob{id: 1, processingToken: currentToken}, + }, + { + name: "nil executor", + job: &dbJob{id: 1, recovered: true, processingToken: currentToken}, + wantError: true, + }, + { + name: "nil settlement error", + execer: &databaseExecerStub{result: databaseResultStub{rows: 1}}, + job: &dbJob{id: 1, recovered: true, processingToken: currentToken}, + wantError: true, + }, + { + name: "missing fenced id", + execer: &databaseExecerStub{result: databaseResultStub{rows: 1}}, + job: &dbJob{recovered: true, processingToken: currentToken}, + err: settlementErr, + wantError: true, + }, + { + name: "execution failure", + execer: &databaseExecerStub{err: errors.New("database offline")}, + job: &dbJob{id: 1, recovered: true, processingToken: currentToken}, + err: settlementErr, + wantError: true, + wantCalls: 1, + }, + { + name: "lost fence", + execer: &databaseExecerStub{result: databaseResultStub{}}, + job: &dbJob{id: 1, recovered: true, processingToken: currentToken}, + err: settlementErr, + wantError: true, + wantCalls: 1, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + repairErr := test.err + if test.name != "nil settlement error" && repairErr == nil { + repairErr = settlementErr + } + err := restoreDatabaseSettlementLineage(context.Background(), test.execer, "repair", test.job, repairErr, 2, 1) + if (err != nil) != test.wantError { + t.Fatalf("restoreDatabaseSettlementLineage() error = %v, wantError %t", err, test.wantError) + } + if execer, ok := test.execer.(*databaseExecerStub); ok && execer.calls != test.wantCalls { + t.Fatalf("repair calls = %d, want %d", execer.calls, test.wantCalls) + } + }) + } +} + +// TestDatabaseMetadataJSONPersistsOnlySupportedMetadata verifies legacy jobs +// store SQL NULL while direct jobs retain the exact root correlation contract. +func TestDatabaseMetadataJSONPersistsOnlySupportedMetadata(t *testing.T) { + if got, err := databaseMetadataJSON(queue.NewJob("reports:legacy")); err != nil || got.Valid { + t.Fatalf("legacy metadata JSON = %#v, %v; want SQL NULL", got, err) + } + + want := queue.DriverJobMetadata{ + SchemaVersion: queue.DriverJobMetadataVersion, + DispatchID: "dsp_sql", + JobID: "job_sql", + Queue: "critical", + } + encoded, err := databaseMetadataJSON(queue.DriverWithMetadata(queue.NewJob("reports:direct"), want)) + if err != nil { + t.Fatalf("encode direct metadata: %v", err) + } + if !encoded.Valid { + t.Fatal("direct metadata unexpectedly encoded as SQL NULL") + } + var got queue.DriverJobMetadata + if err := json.Unmarshal([]byte(encoded.String), &got); err != nil { + t.Fatalf("decode direct metadata: %v", err) + } + if got != want { + t.Fatalf("metadata round trip = %+v, want %+v", got, want) + } +} + +// TestDatabaseDeliveryJobRejectsUntrustedMetadata verifies nullable, malformed, +// and unknown-version rows remain deliverable without accepting spoofed IDs. +func TestDatabaseDeliveryJobRejectsUntrustedMetadata(t *testing.T) { + tests := []struct { + name string + raw sql.NullString + }{ + {name: "null"}, + {name: "empty", raw: sql.NullString{Valid: true}}, + {name: "malformed", raw: sql.NullString{String: `{`, Valid: true}}, + {name: "unknown", raw: sql.NullString{String: `{"schema_version":99,"dispatch_id":"spoofed","job_id":"spoofed"}`, Valid: true}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + job := databaseDeliveryJob(&dbJob{ + jobType: "reports:build", + payload: []byte(`{"id":7}`), + metadataJSON: test.raw, + queueName: "critical", + }) + if metadata := queue.DriverMetadata(job); metadata.SchemaVersion != 0 { + t.Fatalf("untrusted driver metadata = %+v", metadata) + } + observed := queue.ResolveObservedJobMetadataFromJob(job) + if observed.DispatchID != "" || observed.JobID != "" || observed.ChainID != "" || observed.BatchID != "" { + t.Fatalf("untrusted observed correlation = %+v", observed) + } + if observed.JobType != "reports:build" || observed.JobKey == "" { + t.Fatalf("application identity was not delivered: %+v", observed) + } + }) + } +} + +// TestDatabaseDeliveryJobRetainsLegacyEnvelopeFallback verifies NULL metadata +// does not sever correlation for workflow envelopes already persisted by v1. +func TestDatabaseDeliveryJobRetainsLegacyEnvelopeFallback(t *testing.T) { + payload := []byte(`{"schema_version":1,"dispatch_id":"dsp_legacy","job_id":"job_legacy","chain_id":"chn_legacy","job":{"type":"reports:build","payload":"eyJpZCI6N30="}}`) + job := databaseDeliveryJob(&dbJob{ + jobType: "bus:chain:node", + payload: payload, + queueName: "critical", + }) + metadata := queue.ResolveObservedJobMetadataFromJob(job) + if metadata.JobType != "reports:build" || metadata.DispatchID != "dsp_legacy" || metadata.JobID != "job_legacy" || metadata.ChainID != "chn_legacy" { + t.Fatalf("legacy metadata fallback = %+v", metadata) + } +} + +// TestClassifyDatabaseFailure verifies persisted attempt counters distinguish application retry from infrastructure redelivery. +func TestClassifyDatabaseFailure(t *testing.T) { + cause := errors.New("failed") + tests := []struct { + name string + job dbJob + err error + want databaseFailureSettlement + wantClassErr bool + }{ + { + name: "application retry advances attempt", + job: dbJob{attempt: 1, maxRetry: 3, backoffMillis: 250}, + err: cause, + want: databaseFailureSettlement{state: "pending", attempt: 2, availableAt: 1250}, + }, + { + name: "permanent failure becomes dead early", + job: dbJob{attempt: 0, maxRetry: 3, backoffMillis: 250}, + err: busruntime.Permanent(cause), + want: databaseFailureSettlement{state: "dead", attempt: 1}, + }, + { + name: "exhausted failure becomes dead", + job: dbJob{attempt: 3, maxRetry: 3, backoffMillis: 250}, + err: cause, + want: databaseFailureSettlement{state: "dead", attempt: 4}, + }, + { + name: "uncommitted redelivery preserves attempt", + job: dbJob{attempt: 2, maxRetry: 3, backoffMillis: 250}, + err: busruntime.Uncommitted(cause), + want: databaseFailureSettlement{state: "pending", attempt: 2, availableAt: 1250}, + }, + { + name: "success cannot enter failure persistence", + job: dbJob{attempt: 0, maxRetry: 3}, + wantClassErr: true, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := classifyDatabaseFailure(&test.job, test.err, 1000) + if test.wantClassErr { + if err == nil { + t.Fatalf("classifyDatabaseFailure() = %+v, nil; want error", got) + } + return + } + if err != nil { + t.Fatalf("classifyDatabaseFailure(): %v", err) + } + if got != test.want { + t.Fatalf("settlement = %+v, want %+v", got, test.want) + } + }) + } +} + +// TestRequireDatabaseSettlementRow verifies only one affected durable row commits a delivery outcome. +func TestRequireDatabaseSettlementRow(t *testing.T) { + rowsErr := errors.New("rows unavailable") + tests := []struct { + name string + result sql.Result + wantErr bool + }{ + {name: "nil", wantErr: true}, + {name: "zero", result: databaseResultStub{}, wantErr: true}, + {name: "one", result: databaseResultStub{rows: 1}}, + {name: "many", result: databaseResultStub{rows: 2}, wantErr: true}, + {name: "rows error", result: databaseResultStub{err: rowsErr}, wantErr: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := requireDatabaseSettlementRow(test.result) + if (err != nil) != test.wantErr { + t.Fatalf("requireDatabaseSettlementRow() error = %v, wantErr %t", err, test.wantErr) + } + }) + } +} + +// TestDatabaseProcessingClaimRequiresGeneration verifies settlement cannot fall back to an unfenced row identifier. +func TestDatabaseProcessingClaimRequiresGeneration(t *testing.T) { + tests := []struct { + name string + job *dbJob + wantErr bool + }{ + {name: "nil", wantErr: true}, + {name: "missing id", job: &dbJob{processingToken: "claim"}, wantErr: true}, + {name: "missing token", job: &dbJob{id: 7}, wantErr: true}, + {name: "fenced claim", job: &dbJob{id: 7, processingToken: "claim"}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + id, token, err := databaseProcessingClaim(test.job) + if (err != nil) != test.wantErr { + t.Fatalf("databaseProcessingClaim() = (%d, %q, %v), wantErr %t", id, token, err, test.wantErr) + } + if !test.wantErr && (id != test.job.id || token != test.job.processingToken) { + t.Fatalf("databaseProcessingClaim() = (%d, %q), want (%d, %q)", id, token, test.job.id, test.job.processingToken) + } + }) + } +} + +// TestNewDatabaseProcessingToken verifies processing generations fit every additive dialect column. +func TestNewDatabaseProcessingToken(t *testing.T) { + token, err := newDatabaseProcessingToken() + if err != nil { + t.Fatalf("newDatabaseProcessingToken(): %v", err) + } + if !databaseProcessingTokenValid(token) { + t.Fatalf("processing token %q is not canonical", token) + } + if len(token) != databaseProcessingTokenBytes*2 || len(token) > 64 { + t.Fatalf("processing token length = %d, want %d within additive column", len(token), databaseProcessingTokenBytes*2) + } + for _, malformed := range []string{ + "", + strings.Repeat("a", databaseProcessingTokenBytes*2-1), + strings.Repeat("a", databaseProcessingTokenBytes*2+1), + strings.Repeat("z", databaseProcessingTokenBytes*2), + strings.Repeat("A", databaseProcessingTokenBytes*2), + } { + if databaseProcessingTokenValid(malformed) { + t.Fatalf("malformed processing token %q was accepted", malformed) + } + } +} + +// TestDatabaseSchemaStatementsIncludeAdditiveColumns verifies fresh schemas +// never depend on either compatibility migration pass. +func TestDatabaseSchemaStatementsIncludeAdditiveColumns(t *testing.T) { + for _, driverName := range []string{"sqlite", "pgx", "mysql"} { + t.Run(driverName, func(t *testing.T) { + statements := (&databaseQueue{cfg: localDatabaseConfig{DriverName: driverName}}).schemaStatements() + if len(statements) == 0 || !strings.Contains(statements[0], "processing_token") { + t.Fatalf("fresh %s queue schema does not include processing_token", driverName) + } + if !strings.Contains(statements[0], "metadata_json TEXT NULL") { + t.Fatalf("fresh %s queue schema does not include nullable metadata_json", driverName) + } + }) + } +} diff --git a/driver/sqlqueuecore/wrapper.go b/driver/sqlqueuecore/wrapper.go index 5949967..39dd4b3 100644 --- a/driver/sqlqueuecore/wrapper.go +++ b/driver/sqlqueuecore/wrapper.go @@ -15,6 +15,7 @@ type ModuleConfig struct { queueconfig.DriverBaseConfig DB *sql.DB DSN string + DisableAutoMigrate bool ProcessingRecoveryGrace time.Duration ProcessingLeaseNoTimeout time.Duration } @@ -22,14 +23,16 @@ type ModuleConfig struct { // NewQueue creates a high-level queue.Queue from the shared SQL implementation // for a specific SQL driver name. func NewQueue(driverName string, cfg ModuleConfig, opts ...queue.Option) (*queue.Queue, error) { + observer := driverbridge.NewObserverSink(cfg.Observer) backend, err := New(queue.DatabaseConfig{ DB: cfg.DB, DriverName: driverName, DSN: cfg.DSN, + DisableAutoMigrate: cfg.DisableAutoMigrate, DefaultQueue: queue.PhysicalQueueName(cfg.DefaultQueue, cfg.DefaultQueue), ProcessingRecoveryGrace: cfg.ProcessingRecoveryGrace, ProcessingLeaseNoTimeout: cfg.ProcessingLeaseNoTimeout, - Observer: cfg.Observer, + Observer: observer, Logger: cfg.Logger, }) if err != nil { @@ -38,7 +41,7 @@ func NewQueue(driverName string, cfg ModuleConfig, opts ...queue.Option) (*queue rootCfg := queue.Config{ Driver: queue.DriverDatabase, DefaultQueue: cfg.DefaultQueue, - Observer: cfg.Observer, + Observer: observer, Logger: cfg.Logger, } return driverbridge.NewQueueFromDriver(rootCfg, backend, nil, opts...) diff --git a/driver/sqsqueue/queue_sqs_impl.go b/driver/sqsqueue/queue_sqs_impl.go index 41b2b7e..9909e6c 100644 --- a/driver/sqsqueue/queue_sqs_impl.go +++ b/driver/sqsqueue/queue_sqs_impl.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "strings" "sync" "time" @@ -14,19 +15,21 @@ import ( "github.com/aws/aws-sdk-go-v2/service/sqs" "github.com/aws/aws-sdk-go-v2/service/sqs/types" "github.com/goforj/queue" + "github.com/goforj/queue/internal/uniqueness" "github.com/goforj/queue/queuecore" ) type sqsMessage struct { - Type string `json:"type"` - Payload []byte `json:"payload,omitempty"` - Queue string `json:"queue"` - Attempt int `json:"attempt,omitempty"` - MaxRetry int `json:"max_retry,omitempty"` - BackoffMillis int64 `json:"backoff_millis,omitempty"` - TimeoutMillis int64 `json:"timeout_millis,omitempty"` - AvailableAtMS int64 `json:"available_at_ms,omitempty"` - PublishedAtMS int64 `json:"published_at_ms,omitempty"` + Type string `json:"type"` + Payload []byte `json:"payload,omitempty"` + Queue string `json:"queue"` + Metadata json.RawMessage `json:"metadata,omitempty"` + Attempt int `json:"attempt,omitempty"` + MaxRetry int `json:"max_retry,omitempty"` + BackoffMillis int64 `json:"backoff_millis,omitempty"` + TimeoutMillis int64 `json:"timeout_millis,omitempty"` + AvailableAtMS int64 `json:"available_at_ms,omitempty"` + PublishedAtMS int64 `json:"published_at_ms,omitempty"` } type sqsClient interface { @@ -43,7 +46,7 @@ type sqsQueue struct { mu sync.Mutex client sqsClient queueURLs map[string]string - unique map[string]time.Time + unique uniqueness.MemoryStore } func (q *sqsQueue) physicalQueueName() string { @@ -57,7 +60,6 @@ func newSQSQueue(cfg Config) *sqsQueue { return &sqsQueue{ cfg: normalizeConfig(cfg), queueURLs: make(map[string]string), - unique: make(map[string]time.Time), } } @@ -104,10 +106,14 @@ func (q *sqsQueue) Shutdown(_ context.Context) error { return nil } +// Dispatch requires a service message identifier before reporting SQS acceptance. func (q *sqsQueue) Dispatch(ctx context.Context, job queue.Job) error { if ctx == nil { ctx = context.Background() } + if err := ctx.Err(); err != nil { + return err + } if err := queuecore.ValidateDriverJob(job); err != nil { return err } @@ -118,36 +124,40 @@ func (q *sqsQueue) Dispatch(ctx context.Context, job queue.Job) error { if err := q.ensureClient(ctx); err != nil { return err } - if parsed.UniqueTTL > 0 && !q.claimUnique(job, parsed.QueueName, parsed.UniqueTTL) { - return queuecore.ErrDuplicate + var ( + uniqueKey string + uniqueToken uint64 + ) + if parsed.UniqueTTL > 0 { + var acquired bool + uniqueKey, uniqueToken, acquired = q.claimUnique(job, parsed.QueueName, parsed.UniqueTTL) + if !acquired { + return queuecore.ErrDuplicate + } } - msg := sqsMessage{ - Type: job.Type, - Payload: job.PayloadBytes(), - Queue: parsed.QueueName, - PublishedAtMS: time.Now().UnixMilli(), - } - if parsed.MaxRetry != nil { - msg.MaxRetry = *parsed.MaxRetry - } - if parsed.Backoff != nil && *parsed.Backoff > 0 { - msg.BackoffMillis = parsed.Backoff.Milliseconds() - } - if parsed.Timeout != nil && *parsed.Timeout > 0 { - msg.TimeoutMillis = parsed.Timeout.Milliseconds() - } - if parsed.Delay > 0 { - msg.AvailableAtMS = time.Now().Add(parsed.Delay).UnixMilli() + msg, err := sqsMessageForJob(job, parsed) + if err != nil { + q.unique.Release(uniqueKey, uniqueToken) + return err } body, err := json.Marshal(msg) if err != nil { + q.unique.Release(uniqueKey, uniqueToken) return err } queueURL, err := q.ensureQueue(ctx, parsed.QueueName) if err != nil { + q.unique.Release(uniqueKey, uniqueToken) return err } + q.mu.Lock() + client := q.client + q.mu.Unlock() + if client == nil { + q.unique.Release(uniqueKey, uniqueToken) + return fmt.Errorf("sqs client unavailable during dispatch") + } input := &sqs.SendMessageInput{ QueueUrl: &queueURL, MessageBody: aws.String(string(body)), @@ -161,10 +171,55 @@ func (q *sqsQueue) Dispatch(ctx context.Context, job queue.Job) error { input.DelaySeconds = seconds } } - _, err = q.client.SendMessage(ctx, input) + output, err := client.SendMessage(ctx, input) + if err == nil { + err = sqsSendAccepted(output) + } + // Send failures and missing receipts are ambiguous: the service may have committed before its response was lost. return err } +// sqsMessageForJob converts one validated queue job into the stable SQS wire +// representation while keeping direct-delivery metadata optional. +func sqsMessageForJob(job queue.Job, options queue.DriverJobOptions) (sqsMessage, error) { + message := sqsMessage{ + Type: job.Type, + Payload: job.PayloadBytes(), + Queue: options.QueueName, + PublishedAtMS: time.Now().UnixMilli(), + } + metadata := queue.DriverMetadata(job) + if metadata.SchemaVersion != 0 { + encoded, err := json.Marshal(metadata) + if err != nil { + return sqsMessage{}, fmt.Errorf("encode SQS driver job metadata: %w", err) + } + message.Metadata = encoded + } + if options.MaxRetry != nil { + message.MaxRetry = *options.MaxRetry + } + if options.Backoff != nil && *options.Backoff > 0 { + message.BackoffMillis = options.Backoff.Milliseconds() + } + if options.Timeout != nil && *options.Timeout > 0 { + message.TimeoutMillis = options.Timeout.Milliseconds() + } + if options.Delay > 0 { + message.AvailableAtMS = time.Now().Add(options.Delay).UnixMilli() + } + return message, nil +} + +// sqsSendAccepted requires the service-generated receipt that proves SQS accepted the message. +func sqsSendAccepted(output *sqs.SendMessageOutput) error { + if output == nil || output.MessageId == nil || strings.TrimSpace(*output.MessageId) == "" { + return fmt.Errorf("sqs send message returned no message id") + } + return nil +} + +// ensureQueue resolves one queue through a stable client snapshot so concurrent shutdown cannot dereference nil. func (q *sqsQueue) ensureQueue(ctx context.Context, queueName string) (string, error) { q.mu.Lock() if url, ok := q.queueURLs[queueName]; ok && url != "" { @@ -173,6 +228,9 @@ func (q *sqsQueue) ensureQueue(ctx context.Context, queueName string) (string, e } client := q.client q.mu.Unlock() + if client == nil { + return "", fmt.Errorf("sqs client unavailable while resolving queue") + } url, err := getOrCreateSQSQueue(ctx, client, queueName) if err != nil { @@ -184,54 +242,32 @@ func (q *sqsQueue) ensureQueue(ctx context.Context, queueName string) (string, e return url, nil } +// getOrCreateSQSQueue resolves a queue and tolerates an absent lookup response +// because SQS queue creation is idempotent by name. func getOrCreateSQSQueue(ctx context.Context, client sqsClient, queueName string) (string, error) { out, err := client.GetQueueUrl(ctx, &sqs.GetQueueUrlInput{QueueName: &queueName}) - if err == nil && out.QueueUrl != nil && *out.QueueUrl != "" { + if err == nil && out != nil && out.QueueUrl != nil && *out.QueueUrl != "" { return *out.QueueUrl, nil } var notFound *types.QueueDoesNotExist - if err != nil && !isQueueDoesNotExist(err, ¬Found) { + if err != nil && !errors.As(err, ¬Found) { return "", err } createOut, createErr := client.CreateQueue(ctx, &sqs.CreateQueueInput{QueueName: &queueName}) if createErr != nil { return "", createErr } - if createOut.QueueUrl == nil || *createOut.QueueUrl == "" { + if createOut == nil || createOut.QueueUrl == nil || *createOut.QueueUrl == "" { return "", fmt.Errorf("created queue %q but no queue url returned", queueName) } return *createOut.QueueUrl, nil } -func isQueueDoesNotExist(err error, target **types.QueueDoesNotExist) bool { - if err == nil { - return false - } - var notFound *types.QueueDoesNotExist - if ok := errors.As(err, ¬Found); ok { - if target != nil { - *target = notFound - } - return true - } - return false -} - -func (q *sqsQueue) claimUnique(job queue.Job, queueName string, ttl time.Duration) bool { - now := time.Now() - key := queueName + ":" + job.Type + ":" + string(job.PayloadBytes()) - q.mu.Lock() - defer q.mu.Unlock() - for candidate, expiresAt := range q.unique { - if expiresAt.Before(now) { - delete(q.unique, candidate) - } - } - if expiresAt, ok := q.unique[key]; ok && expiresAt.After(now) { - return false - } - q.unique[key] = now.Add(ttl) - return true +// claimUnique returns the ownership token needed to compensate a rejected send. +func (q *sqsQueue) claimUnique(job queue.Job, queueName string, ttl time.Duration) (string, uint64, bool) { + key := queuecore.UniqueKey(job, queueName) + token, ok := q.unique.Acquire(key, ttl) + return key, token, ok } func newSQSClient(ctx context.Context, cfg Config) (sqsClient, error) { diff --git a/driver/sqsqueue/queue_sqs_impl_test.go b/driver/sqsqueue/queue_sqs_impl_test.go new file mode 100644 index 0000000..cce219d --- /dev/null +++ b/driver/sqsqueue/queue_sqs_impl_test.go @@ -0,0 +1,407 @@ +package sqsqueue + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "testing" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/sqs" + "github.com/aws/aws-sdk-go-v2/service/sqs/types" + "github.com/goforj/queue" +) + +// TestSQSDirectDeliveryMetadataRoundTrip verifies producer framing, worker +// reconstruction, retry preservation, and legacy-envelope observation. +func TestSQSDirectDeliveryMetadataRoundTrip(t *testing.T) { + wantMetadata := queue.DriverJobMetadata{ + SchemaVersion: queue.DriverJobMetadataVersion, + DispatchID: "dsp_sqs_direct", + JobID: "job_sqs_direct", + Queue: "critical", + } + wantPayload := []byte(`{"report_id":42}`) + job := queue.DriverWithMetadata( + queue.NewJob("reports:build").Payload(wantPayload).OnQueue("critical").Retry(3), + wantMetadata, + ) + message, err := sqsMessageForJob(job, queue.DriverOptions(job)) + if err != nil { + t.Fatalf("build direct message: %v", err) + } + var wireMetadata queue.DriverJobMetadata + if err := json.Unmarshal(message.Metadata, &wireMetadata); err != nil || wireMetadata != wantMetadata { + t.Fatalf("wire metadata = %+v, want %+v (err=%v)", wireMetadata, wantMetadata, err) + } + + wire, err := json.Marshal(message) + if err != nil { + t.Fatalf("marshal direct message: %v", err) + } + var decoded sqsMessage + if err := json.Unmarshal(wire, &decoded); err != nil { + t.Fatalf("unmarshal direct message: %v", err) + } + delivery := sqsDeliveryJob(decoded) + if delivery.Type != "reports:build" || !bytes.Equal(delivery.PayloadBytes(), wantPayload) { + t.Fatalf("delivery = type:%q payload:%q", delivery.Type, delivery.PayloadBytes()) + } + if got := queue.DriverMetadata(delivery); got != wantMetadata { + t.Fatalf("reconstructed metadata = %+v, want %+v", got, wantMetadata) + } + observed := queue.ResolveObservedJobMetadataFromJob(delivery) + if observed.DispatchID != wantMetadata.DispatchID || observed.JobID != wantMetadata.JobID || observed.JobType != job.Type { + t.Fatalf("direct observation = %+v", observed) + } + var events []queue.Event + worker := &sqsWorker{observer: queue.ObserverFunc(func(_ context.Context, event queue.Event) { + events = append(events, event) + })} + worker.observeRepublishFailure(context.Background(), decoded, errors.New("republish failed")) + if len(events) != 1 || events[0].DispatchID != wantMetadata.DispatchID || events[0].JobID != wantMetadata.JobID { + t.Fatalf("direct republish observation = %+v", events) + } + + decoded.Attempt++ + client := &sqsWorkerClientStub{} + worker.client = client + worker.queueURL = "https://example.local/queue/critical" + if err := worker.republish(decoded); err != nil { + t.Fatalf("republish direct message: %v", err) + } + if len(client.sendInputs) != 1 { + t.Fatalf("republished messages = %d, want 1", len(client.sendInputs)) + } + retry := decodeSQSBody(t, client.sendInputs[0]) + retryJob := sqsDeliveryJob(retry) + if got := queue.DriverMetadata(retryJob); got != wantMetadata { + t.Fatalf("retry metadata = %+v, want %+v", got, wantMetadata) + } + if got := queue.DriverOptions(retryJob).Attempt; got != 1 { + t.Fatalf("retry attempt = %d, want 1", got) + } + + legacyPayload := []byte(`{"schema_version":1,"dispatch_id":"dsp_sqs_legacy","job_id":"job_sqs_legacy","job":{"type":"reports:legacy","payload":"e30="}}`) + legacy := queue.ResolveObservedJobMetadataFromJob(sqsDeliveryJob(sqsMessage{Type: "bus:job", Payload: legacyPayload})) + if legacy.JobType != "reports:legacy" || legacy.DispatchID != "dsp_sqs_legacy" || legacy.JobID != "job_sqs_legacy" { + t.Fatalf("legacy observation = %+v", legacy) + } + + plainJob := queue.NewJob("reports:plain").OnQueue("default") + plain, err := sqsMessageForJob(plainJob, queue.DriverOptions(plainJob)) + if err != nil { + t.Fatalf("build metadata-absent message: %v", err) + } + plainWire, err := json.Marshal(plain) + if err != nil { + t.Fatalf("marshal metadata-absent message: %v", err) + } + var fields map[string]json.RawMessage + if err := json.Unmarshal(plainWire, &fields); err != nil { + t.Fatalf("inspect metadata-absent message: %v", err) + } + if _, ok := fields["metadata"]; ok { + t.Fatalf("metadata-absent wire unexpectedly contains metadata: %s", plainWire) + } +} + +// TestSQSUntrustedMetadataRemainsAnOpaqueRetrySidecar verifies valid +// application bytes survive malformed metadata and future fields survive republish. +func TestSQSUntrustedMetadataRemainsAnOpaqueRetrySidecar(t *testing.T) { + for _, raw := range []string{`"malformed"`, `{"schema_version":"bad","dispatch_id":"spoofed"}`} { + wire := []byte(`{"type":"reports:build","payload":"AQI=","queue":"critical","metadata":` + raw + `}`) + var message sqsMessage + if err := json.Unmarshal(wire, &message); err != nil { + t.Fatalf("decode message with metadata %s: %v", raw, err) + } + delivery := sqsDeliveryJob(message) + if delivery.Type != "reports:build" || !bytes.Equal(delivery.PayloadBytes(), []byte{1, 2}) { + t.Fatalf("delivery with metadata %s = type:%q payload:%v", raw, delivery.Type, delivery.PayloadBytes()) + } + if metadata := queue.DriverMetadata(delivery); metadata != (queue.DriverJobMetadata{}) { + t.Fatalf("untrusted metadata %s became trusted: %+v", raw, metadata) + } + } + + future := json.RawMessage(`{"schema_version":99,"dispatch_id":"future","future_field":{"id":7}}`) + client := &sqsWorkerClientStub{} + worker := &sqsWorker{client: client, queueURL: "https://example.local/queue/critical"} + if err := worker.republish(sqsMessage{Type: "reports:build", Queue: "critical", Metadata: future}); err != nil { + t.Fatalf("republish future metadata: %v", err) + } + if len(client.sendInputs) != 1 { + t.Fatalf("republished messages = %d, want 1", len(client.sendInputs)) + } + retry := decodeSQSBody(t, client.sendInputs[0]) + if !bytes.Equal(retry.Metadata, future) { + t.Fatalf("future retry metadata = %s, want %s", retry.Metadata, future) + } + if metadata := queue.DriverMetadata(sqsDeliveryJob(retry)); metadata != (queue.DriverJobMetadata{}) { + t.Fatalf("future metadata became trusted: %+v", metadata) + } +} + +// TestSQSQueueAmbiguousDispatchRetainsUniqueClaim verifies a lost send response fails closed against duplicate retries. +func TestSQSQueueAmbiguousDispatchRetainsUniqueClaim(t *testing.T) { + sendErr := errors.New("send response lost") + client := &sqsWorkerClientStub{sendErr: sendErr} + q := newSQSQueue(Config{}) + q.client = client + q.queueURLs["default"] = "https://example.local/queue/default" + job := queue.NewJob("reports:build").Payload([]byte(`{"id":1}`)).OnQueue("default").UniqueFor(time.Minute) + + if err := q.Dispatch(context.Background(), job); !errors.Is(err, sendErr) { + t.Fatalf("first dispatch error = %v, want send rejection", err) + } + if err := q.Dispatch(context.Background(), job); !errors.Is(err, queue.ErrDuplicate) { + t.Fatalf("ambiguous dispatch did not retain claim: %v", err) + } +} + +// TestSQSQueueMissingReceiptRetainsUniqueClaim verifies a receipt-less response cannot admit an immediate duplicate retry. +func TestSQSQueueMissingReceiptRetainsUniqueClaim(t *testing.T) { + client := &sqsWorkerClientStub{sendNil: true} + q := newSQSQueue(Config{}) + q.client = client + q.queueURLs["default"] = "https://example.local/queue/default" + job := queue.NewJob("reports:receipt").OnQueue("default").UniqueFor(time.Minute) + if err := q.Dispatch(context.Background(), job); err == nil { + t.Fatal("missing SQS message id was accepted") + } + client.sendNil = false + if err := q.Dispatch(context.Background(), job); !errors.Is(err, queue.ErrDuplicate) { + t.Fatalf("missing receipt did not retain uniqueness claim: %v", err) + } +} + +// TestSQSQueueCanceledDispatchStopsBeforeClaim verifies cancellation cannot send or consume uniqueness state. +func TestSQSQueueCanceledDispatchStopsBeforeClaim(t *testing.T) { + client := &sqsWorkerClientStub{} + q := newSQSQueue(Config{}) + q.client = client + q.queueURLs["default"] = "https://example.local/queue/default" + job := queue.NewJob("reports:canceled").OnQueue("default").UniqueFor(time.Minute) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := q.Dispatch(ctx, job); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled dispatch = %v, want context.Canceled", err) + } + key, token, ok := q.claimUnique(job, "default", time.Minute) + if !ok { + t.Fatal("canceled dispatch consumed uniqueness state") + } + q.unique.Release(key, token) + if len(client.sendInputs) != 0 { + t.Fatalf("canceled dispatch sent %d messages", len(client.sendInputs)) + } +} + +// TestSQSQueueDriverAndPreflight verifies driver identity, nil-context +// normalization, queue discovery, and cancellation before client activity. +func TestSQSQueueDriverAndPreflight(t *testing.T) { + client := &sqsWorkerClientStub{queueURL: "https://example.local/queue/critical"} + q := newSQSQueue(Config{}) + q.cfg.DefaultQueue = "critical" + q.client = client + if got := q.Driver(); got != queue.DriverSQS { + t.Fatalf("driver = %q, want %q", got, queue.DriverSQS) + } + if got := q.physicalQueueName(); got != "critical" { + t.Fatalf("physical queue = %q, want critical", got) + } + if err := q.Preflight(nil); err != nil { + t.Fatalf("preflight with configured client: %v", err) + } + if len(client.getQueueInputs) != 1 || aws.ToString(client.getQueueInputs[0].QueueName) != "critical" { + t.Fatalf("queue lookups = %+v, want one critical lookup", client.getQueueInputs) + } + if got := q.queueURLs["critical"]; got != client.queueURL { + t.Fatalf("cached queue URL = %q, want %q", got, client.queueURL) + } + + canceled, cancel := context.WithCancel(context.Background()) + cancel() + if err := q.Preflight(canceled); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled preflight error = %v, want context.Canceled", err) + } + if len(client.getQueueInputs) != 1 { + t.Fatalf("canceled preflight performed %d queue lookups, want 0 additional lookups", len(client.getQueueInputs)) + } + if got := newSQSQueue(Config{}).physicalQueueName(); got != "default" { + t.Fatalf("default physical queue = %q, want default", got) + } +} + +// TestGetOrCreateSQSQueueBoundaries verifies malformed success responses and +// service errors are returned without nil dereferences. +func TestGetOrCreateSQSQueueBoundaries(t *testing.T) { + getErr := errors.New("lookup failed") + createErr := errors.New("create failed") + tests := []struct { + name string + client *sqsWorkerClientStub + wantURL string + wantErr error + wantAnyErr bool + wantCreates int + }{ + { + name: "existing queue", + client: &sqsWorkerClientStub{queueURL: "https://example.local/queue/existing"}, + wantURL: "https://example.local/queue/existing", + }, + { + name: "missing queue is created", + client: &sqsWorkerClientStub{ + getQueueErr: &types.QueueDoesNotExist{}, + createOutput: &sqs.CreateQueueOutput{QueueUrl: aws.String("https://example.local/queue/created")}, + }, + wantURL: "https://example.local/queue/created", + wantCreates: 1, + }, + { + name: "nil lookup success falls back to creation", + client: &sqsWorkerClientStub{ + getNilSuccess: true, + createOutput: &sqs.CreateQueueOutput{QueueUrl: aws.String("https://example.local/queue/from-nil")}, + }, + wantURL: "https://example.local/queue/from-nil", + wantCreates: 1, + }, + { + name: "lookup rejection", + client: &sqsWorkerClientStub{getQueueErr: getErr}, + wantErr: getErr, + }, + { + name: "creation rejection", + client: &sqsWorkerClientStub{ + getQueueErr: &types.QueueDoesNotExist{}, + createErr: createErr, + }, + wantErr: createErr, + wantCreates: 1, + }, + { + name: "nil creation success", + client: &sqsWorkerClientStub{ + getQueueErr: &types.QueueDoesNotExist{}, + createNilSuccess: true, + }, + wantAnyErr: true, + wantCreates: 1, + }, + { + name: "empty creation URL", + client: &sqsWorkerClientStub{ + getQueueErr: &types.QueueDoesNotExist{}, + createOutput: &sqs.CreateQueueOutput{}, + }, + wantAnyErr: true, + wantCreates: 1, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := getOrCreateSQSQueue(context.Background(), test.client, "reports") + if test.wantErr != nil && !errors.Is(err, test.wantErr) { + t.Fatalf("queue resolution error = %v, want %v", err, test.wantErr) + } + if test.wantAnyErr && err == nil { + t.Fatal("malformed service response unexpectedly succeeded") + } + if test.wantErr == nil && !test.wantAnyErr && err != nil { + t.Fatalf("queue resolution: %v", err) + } + if got != test.wantURL { + t.Fatalf("queue URL = %q, want %q", got, test.wantURL) + } + if len(test.client.createInputs) != test.wantCreates { + t.Fatalf("create calls = %d, want %d", len(test.client.createInputs), test.wantCreates) + } + }) + } +} + +// TestSQSQueueDelayEncodingBoundsServiceDelay verifies SQS receives only its +// supported delay while the wire message retains the full delivery deadline. +func TestSQSQueueDelayEncodingBoundsServiceDelay(t *testing.T) { + client := &sqsWorkerClientStub{} + q := newSQSQueue(Config{}) + q.client = client + q.queueURLs["default"] = "https://example.local/queue/default" + started := time.Now() + job := queue.NewJob("reports:delayed").OnQueue("default").Delay(901 * time.Second) + if err := q.Dispatch(context.Background(), job); err != nil { + t.Fatalf("dispatch delayed job: %v", err) + } + finished := time.Now() + if len(client.sendInputs) != 1 { + t.Fatalf("send calls = %d, want 1", len(client.sendInputs)) + } + if got := client.sendInputs[0].DelaySeconds; got != 900 { + t.Fatalf("service delay = %d seconds, want 900", got) + } + message := decodeSQSBody(t, client.sendInputs[0]) + minimumDeadline := started.Add(901 * time.Second).Add(-time.Millisecond).UnixMilli() + maximumDeadline := finished.Add(901 * time.Second).Add(time.Millisecond).UnixMilli() + if message.AvailableAtMS < minimumDeadline || message.AvailableAtMS > maximumDeadline { + t.Fatalf("wire availability = %d, want the original 901-second deadline in [%d, %d]", message.AvailableAtMS, minimumDeadline, maximumDeadline) + } +} + +// TestSQSQueueEnsureQueueRejectsMissingClient verifies shutdown races fail with +// a diagnostic instead of dereferencing an unavailable client. +func TestSQSQueueEnsureQueueRejectsMissingClient(t *testing.T) { + q := newSQSQueue(Config{}) + if _, err := q.ensureQueue(context.Background(), "default"); err == nil { + t.Fatal("queue resolution without a client unexpectedly succeeded") + } +} + +// TestSQSQueueRejectedResolutionReleasesUniqueClaim verifies a failure before +// send does not retain uniqueness state for a message SQS never accepted. +func TestSQSQueueRejectedResolutionReleasesUniqueClaim(t *testing.T) { + q := newSQSQueue(Config{}) + q.client = &sqsWorkerClientStub{} + job := queue.NewJob("reports:resolve").OnQueue("default").UniqueFor(time.Minute) + + if err := q.Dispatch(context.Background(), job); err == nil || errors.Is(err, queue.ErrDuplicate) { + t.Fatalf("queue resolution error = %v, want a pre-send rejection", err) + } + key, token, acquired := q.claimUnique(job, "default", time.Minute) + if !acquired { + t.Fatal("pre-send queue resolution failure retained the uniqueness claim") + } + q.unique.Release(key, token) +} + +// TestSQSQueueShutdownRaceBeforeSendReleasesUniqueClaim verifies a concurrent +// shutdown after queue resolution cannot retain a claim for an unsent message. +func TestSQSQueueShutdownRaceBeforeSendReleasesUniqueClaim(t *testing.T) { + q := newSQSQueue(Config{}) + client := &sqsWorkerClientStub{queueURL: "https://example.local/queue/default"} + client.queueURLHook = func() { + if err := q.Shutdown(context.Background()); err != nil { + t.Errorf("shutdown during queue resolution: %v", err) + } + } + q.client = client + job := queue.NewJob("reports:shutdown-race").OnQueue("default").UniqueFor(time.Minute) + + if err := q.Dispatch(context.Background(), job); err == nil || errors.Is(err, queue.ErrDuplicate) { + t.Fatalf("shutdown-race dispatch error = %v, want an unavailable-client rejection", err) + } + if len(client.sendInputs) != 0 { + t.Fatalf("shutdown-race sends = %d, want 0", len(client.sendInputs)) + } + key, token, acquired := q.claimUnique(job, "default", time.Minute) + if !acquired { + t.Fatal("shutdown before send retained the uniqueness claim") + } + q.unique.Release(key, token) +} diff --git a/driver/sqsqueue/sqsqueue.go b/driver/sqsqueue/sqsqueue.go index 7fb32d0..a2e6984 100644 --- a/driver/sqsqueue/sqsqueue.go +++ b/driver/sqsqueue/sqsqueue.go @@ -56,10 +56,11 @@ func New(region string, opts ...queue.Option) (*queue.Queue, error) { // _ = q func NewWithConfig(cfg Config, opts ...queue.Option) (*queue.Queue, error) { cfg = normalizeConfig(cfg) + observer := driverbridge.NewObserverSink(cfg.Observer) rootCfg := queue.Config{ Driver: queue.DriverSQS, DefaultQueue: cfg.DefaultQueue, - Observer: cfg.Observer, + Observer: observer, } defaultQueue := queue.PhysicalQueueName(cfg.DefaultQueue, cfg.DefaultQueue) return driverbridge.NewQueueFromDriver(rootCfg, newSQSQueue(cfg), func(workers int) (any, error) { @@ -70,7 +71,7 @@ func NewWithConfig(cfg Config, opts ...queue.Option) (*queue.Queue, error) { SQSAccessKey: cfg.AccessKey, SQSSecretKey: cfg.SecretKey, Workers: workers, - Observer: cfg.Observer, + Observer: observer, }), nil }, opts...) } diff --git a/driver/sqsqueue/worker_sqs_impl.go b/driver/sqsqueue/worker_sqs_impl.go index c185c8f..5546632 100644 --- a/driver/sqsqueue/worker_sqs_impl.go +++ b/driver/sqsqueue/worker_sqs_impl.go @@ -3,6 +3,8 @@ package sqsqueue import ( "context" "encoding/json" + "fmt" + "strings" "sync" "time" @@ -10,9 +12,12 @@ import ( "github.com/aws/aws-sdk-go-v2/service/sqs" sqstypes "github.com/aws/aws-sdk-go-v2/service/sqs/types" "github.com/goforj/queue" + "github.com/goforj/queue/busruntime" "github.com/goforj/queue/queuecore" ) +const sqsSettlementTimeout = 15 * time.Second + type sqsWorkerClient interface { GetQueueUrl(ctx context.Context, params *sqs.GetQueueUrlInput, optFns ...func(*sqs.Options)) (*sqs.GetQueueUrlOutput, error) CreateQueue(ctx context.Context, params *sqs.CreateQueueInput, optFns ...func(*sqs.Options)) (*sqs.CreateQueueOutput, error) @@ -44,6 +49,7 @@ type sqsWorker struct { wg sync.WaitGroup startStop sync.Mutex observer queue.Observer + stopDone chan struct{} } func newSQSWorker(cfg sqsWorkerConfig) *sqsWorker { @@ -104,20 +110,40 @@ func (w *sqsWorker) StartWorkers(ctx context.Context) error { return nil } -func (w *sqsWorker) Shutdown(_ context.Context) error { +// Shutdown stops receive loops while allowing in-flight replacement and deletion calls to finish independently. +func (w *sqsWorker) Shutdown(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } w.startStop.Lock() if !w.started { w.startStop.Unlock() return nil } - cancel := w.cancel - w.started = false + if w.stopDone == nil { + w.stopDone = make(chan struct{}) + cancel := w.cancel + done := w.stopDone + if cancel != nil { + cancel() + } + go func() { + w.wg.Wait() + w.startStop.Lock() + w.started = false + w.stopDone = nil + w.startStop.Unlock() + close(done) + }() + } + done := w.stopDone w.startStop.Unlock() - if cancel != nil { - cancel() + select { + case <-done: + return nil + case <-ctx.Done(): + return ctx.Err() } - w.wg.Wait() - return nil } func (w *sqsWorker) loop(ctx context.Context) { @@ -146,24 +172,25 @@ func (w *sqsWorker) loop(ctx context.Context) { } } +// process commits positive facts only after the original SQS receipt is deleted. func (w *sqsWorker) process(ctx context.Context, message sqstypes.Message) { if message.Body == nil { - w.delete(ctx, message) + w.deleteAndObserve(ctx, message, sqsMessage{}) return } var incoming sqsMessage if err := json.Unmarshal([]byte(*message.Body), &incoming); err != nil { - w.delete(ctx, message) + w.deleteAndObserve(ctx, message, sqsMessage{}) return } if incoming.AvailableAtMS > 0 { remaining := time.Until(time.UnixMilli(incoming.AvailableAtMS)) if remaining > 0 { - if err := w.republish(ctx, incoming); err != nil { + if err := w.republish(incoming); err != nil { w.observeRepublishFailure(ctx, incoming, err) return } - w.delete(ctx, message) + w.deleteAndObserve(ctx, message, incoming) return } } @@ -172,10 +199,12 @@ func (w *sqsWorker) process(ctx context.Context, message sqstypes.Message) { handler, ok := w.handlers[incoming.Type] w.mu.RUnlock() if !ok { - w.delete(ctx, message) + w.deleteAndObserve(ctx, message, incoming) return } - runCtx := context.Background() + attempt := busruntime.DeliveryAttempt{Number: incoming.Attempt, MaxRetry: incoming.MaxRetry} + runCtx := busruntime.WithDeliveryAttempt(context.Background(), attempt) + runCtx, settlement := busruntime.WithDeliverySettlement(runCtx) if incoming.TimeoutMillis > 0 { var cancel context.CancelFunc runCtx, cancel = context.WithTimeout(runCtx, time.Duration(incoming.TimeoutMillis)*time.Millisecond) @@ -183,36 +212,37 @@ func (w *sqsWorker) process(ctx context.Context, message sqstypes.Message) { } err := handler( runCtx, - queuecore.DriverWithAttempt( - queue.NewJob(incoming.Type). - Payload(incoming.Payload). - OnQueue(incoming.Queue). - Retry(incoming.MaxRetry), - incoming.Attempt, - ), + sqsDeliveryJob(incoming), ) - if err == nil { - w.delete(ctx, message) + switch busruntime.ClassifyAttempt(attempt, err) { + case busruntime.AttemptSucceeded, busruntime.AttemptFailed: + if w.deleteAndObserve(runCtx, message, incoming) { + settlement.Commit() + } return - } - if incoming.Attempt >= incoming.MaxRetry { - w.delete(ctx, message) + case busruntime.AttemptRedeliver: + // Leaving the receipt undeleted lets SQS redeliver the same application attempt after its visibility timeout. return + case busruntime.AttemptRetry: } + settledMessage := incoming incoming.Attempt++ if incoming.BackoffMillis > 0 { incoming.AvailableAtMS = time.Now().Add(time.Duration(incoming.BackoffMillis) * time.Millisecond).UnixMilli() } else { incoming.AvailableAtMS = 0 } - if err := w.republish(ctx, incoming); err != nil { + if err := w.republish(incoming); err != nil { w.observeRepublishFailure(ctx, incoming, err) return } - w.delete(ctx, message) + if w.deleteAndObserve(runCtx, message, settledMessage) { + settlement.Commit() + } } -func (w *sqsWorker) republish(ctx context.Context, message sqsMessage) error { +// republish creates a confirmed replacement before the original receipt can be deleted. +func (w *sqsWorker) republish(message sqsMessage) error { body, err := json.Marshal(message) if err != nil { return err @@ -231,33 +261,101 @@ func (w *sqsWorker) republish(ctx context.Context, message sqsMessage) error { input.DelaySeconds = seconds } } - _, err = w.client.SendMessage(ctx, input) - return err + ctx, cancel := sqsSettlementContext() + defer cancel() + output, err := w.client.SendMessage(ctx, input) + if err != nil { + return err + } + return sqsSendAccepted(output) } -func (w *sqsWorker) delete(ctx context.Context, message sqstypes.Message) { - if message.ReceiptHandle == nil { - return +// delete settles one SQS receipt through a bounded context independent of receive-loop cancellation. +func (w *sqsWorker) delete(message sqstypes.Message) error { + if message.ReceiptHandle == nil || strings.TrimSpace(*message.ReceiptHandle) == "" { + return fmt.Errorf("sqs receipt handle is required for settlement") } - _, _ = w.client.DeleteMessage(ctx, &sqs.DeleteMessageInput{ + ctx, cancel := sqsSettlementContext() + defer cancel() + _, err := w.client.DeleteMessage(ctx, &sqs.DeleteMessageInput{ QueueUrl: &w.queueURL, ReceiptHandle: message.ReceiptHandle, }) + return err +} + +// deleteAndObserve reports deletion ambiguity and returns whether the delivery reached positive settlement. +func (w *sqsWorker) deleteAndObserve(ctx context.Context, message sqstypes.Message, incoming sqsMessage) bool { + if err := w.delete(message); err != nil { + w.observeSettlementFailure(ctx, incoming, fmt.Errorf("delete sqs message: %w", err)) + return false + } + return true } func (w *sqsWorker) observeRepublishFailure(ctx context.Context, message sqsMessage, err error) { + metadata := queue.ResolveObservedJobMetadataFromJob(sqsDeliveryJob(message)) + queuecore.SafeObserve(ctx, w.observer, queue.Event{ + Kind: queue.EventRepublishFailed, + Driver: queue.DriverSQS, + Queue: queuecore.NormalizeQueueName(message.Queue), + JobType: metadata.JobType, + JobKey: metadata.JobKey, + DispatchID: metadata.DispatchID, + JobID: metadata.JobID, + ChainID: metadata.ChainID, + BatchID: metadata.BatchID, + Attempt: message.Attempt, + MaxRetry: message.MaxRetry, + Err: err, + Time: time.Now(), + }) +} + +// observeSettlementFailure emits the canonical worker fact for an uncommitted SQS deletion. +func (w *sqsWorker) observeSettlementFailure(ctx context.Context, message sqsMessage, err error) { + metadata := queue.ResolveObservedJobMetadataFromJob(sqsDeliveryJob(message)) queuecore.SafeObserve(ctx, w.observer, queue.Event{ - Kind: queue.EventRepublishFailed, - Driver: queue.DriverSQS, - Queue: queuecore.NormalizeQueueName(message.Queue), - JobType: queue.ResolveObservedJobType(message.Type, message.Payload), - Attempt: message.Attempt, - MaxRetry: message.MaxRetry, - Err: err, - Time: time.Now(), + Kind: queue.EventSettlementFailed, + Driver: queue.DriverSQS, + Queue: queuecore.NormalizeQueueName(message.Queue), + JobType: metadata.JobType, + JobKey: metadata.JobKey, + DispatchID: metadata.DispatchID, + JobID: metadata.JobID, + ChainID: metadata.ChainID, + BatchID: metadata.BatchID, + Attempt: message.Attempt, + MaxRetry: message.MaxRetry, + Err: err, + Time: time.Now(), }) } +// sqsDeliveryJob reconstructs one SQS delivery while retaining supported +// direct-delivery metadata separately from the application payload. +func sqsDeliveryJob(message sqsMessage) queue.Job { + job := queuecore.DriverWithAttempt( + queue.NewJob(message.Type). + Payload(message.Payload). + OnQueue(message.Queue). + Retry(message.MaxRetry), + message.Attempt, + ) + if len(message.Metadata) > 0 { + var metadata queue.DriverJobMetadata + if err := json.Unmarshal(message.Metadata, &metadata); err == nil { + job = queue.DriverWithMetadata(job, metadata) + } + } + return job +} + +// sqsSettlementContext lets in-flight work finish settlement after the receive loop is canceled without waiting forever. +func sqsSettlementContext() (context.Context, context.CancelFunc) { + return context.WithTimeout(context.Background(), sqsSettlementTimeout) +} + func defaultWorkerCount(n int) int { if n <= 0 { return 1 diff --git a/driver/sqsqueue/worker_sqs_impl_test.go b/driver/sqsqueue/worker_sqs_impl_test.go index 970b984..6d500b7 100644 --- a/driver/sqsqueue/worker_sqs_impl_test.go +++ b/driver/sqsqueue/worker_sqs_impl_test.go @@ -11,20 +11,58 @@ import ( "github.com/aws/aws-sdk-go-v2/service/sqs" sqstypes "github.com/aws/aws-sdk-go-v2/service/sqs/types" "github.com/goforj/queue" + "github.com/goforj/queue/busruntime" "github.com/goforj/queue/queuecore" ) type sqsWorkerClientStub struct { - sendInputs []*sqs.SendMessageInput - deleteInputs []*sqs.DeleteMessageInput - sendErr error + getQueueInputs []*sqs.GetQueueUrlInput + createInputs []*sqs.CreateQueueInput + sendInputs []*sqs.SendMessageInput + deleteInputs []*sqs.DeleteMessageInput + queueURL string + queueURLHook func() + getQueueErr error + createOutput *sqs.CreateQueueOutput + createErr error + getNilSuccess bool + createNilSuccess bool + sendErr error + deleteErr error + sendNil bool + sendEmptyID bool } -func (s *sqsWorkerClientStub) GetQueueUrl(context.Context, *sqs.GetQueueUrlInput, ...func(*sqs.Options)) (*sqs.GetQueueUrlOutput, error) { +// GetQueueUrl supplies scripted lookup outcomes so service-boundary tests stay deterministic. +func (s *sqsWorkerClientStub) GetQueueUrl(_ context.Context, params *sqs.GetQueueUrlInput, _ ...func(*sqs.Options)) (*sqs.GetQueueUrlOutput, error) { + s.getQueueInputs = append(s.getQueueInputs, params) + if s.queueURLHook != nil { + s.queueURLHook() + } + if s.getQueueErr != nil { + return nil, s.getQueueErr + } + if s.getNilSuccess { + return nil, nil + } + if s.queueURL != "" { + return &sqs.GetQueueUrlOutput{QueueUrl: aws.String(s.queueURL)}, nil + } return nil, errors.New("not implemented") } -func (s *sqsWorkerClientStub) CreateQueue(context.Context, *sqs.CreateQueueInput, ...func(*sqs.Options)) (*sqs.CreateQueueOutput, error) { +// CreateQueue supplies scripted creation outcomes so service-boundary tests stay deterministic. +func (s *sqsWorkerClientStub) CreateQueue(_ context.Context, params *sqs.CreateQueueInput, _ ...func(*sqs.Options)) (*sqs.CreateQueueOutput, error) { + s.createInputs = append(s.createInputs, params) + if s.createErr != nil { + return nil, s.createErr + } + if s.createNilSuccess { + return nil, nil + } + if s.createOutput != nil { + return s.createOutput, nil + } return nil, errors.New("not implemented") } @@ -34,7 +72,88 @@ func (s *sqsWorkerClientStub) ReceiveMessage(context.Context, *sqs.ReceiveMessag func (s *sqsWorkerClientStub) DeleteMessage(_ context.Context, params *sqs.DeleteMessageInput, _ ...func(*sqs.Options)) (*sqs.DeleteMessageOutput, error) { s.deleteInputs = append(s.deleteInputs, params) - return &sqs.DeleteMessageOutput{}, nil + return &sqs.DeleteMessageOutput{}, s.deleteErr +} + +// TestSQSWorkerDeleteFailureEmitsSettlementEvent verifies delete ambiguity is visible and retains logical correlation. +func TestSQSWorkerDeleteFailureEmitsSettlementEvent(t *testing.T) { + deleteErr := errors.New("delete response lost") + stub := &sqsWorkerClientStub{deleteErr: deleteErr} + var events []queue.Event + committed := false + var handlerSettlement busruntime.DeliverySettlementIdentity + var handlerSettlementOK bool + var observedSettlement busruntime.DeliverySettlementIdentity + var observedSettlementOK bool + w := &sqsWorker{ + handlers: map[string]queue.Handler{"bus:job": func(ctx context.Context, _ queue.Job) error { + handlerSettlement, handlerSettlementOK = busruntime.DeliverySettlementIdentityFromContext(ctx) + if !busruntime.DeferUntilDeliveryCommitted(ctx, func() { committed = true }) { + t.Fatal("handler context did not carry a settlement boundary") + } + return nil + }}, + client: stub, + queueURL: "https://example.local/queue/default", + observer: queue.ObserverFunc(func(ctx context.Context, event queue.Event) { + observedSettlement, observedSettlementOK = busruntime.DeliverySettlementIdentityFromContext(ctx) + events = append(events, event) + }), + } + payload := []byte(`{"schema_version":1,"dispatch_id":"dsp_sqs_settle","job_id":"job_sqs_settle","job":{"type":"reports:build","payload":"eyJpZCI6MX0="}}`) + body, err := json.Marshal(sqsMessage{Type: "bus:job", Queue: "critical", Payload: payload, Attempt: 2, MaxRetry: 4}) + if err != nil { + t.Fatalf("marshal body: %v", err) + } + w.process(context.Background(), sqstypes.Message{Body: aws.String(string(body)), ReceiptHandle: aws.String("rh-settle")}) + if len(events) != 1 || events[0].Kind != queue.EventSettlementFailed || !errors.Is(events[0].Err, deleteErr) { + t.Fatalf("settlement events = %+v, want one delete failure", events) + } + if events[0].Layer != queue.EventLayerWorker || events[0].JobType != "reports:build" || events[0].DispatchID != "dsp_sqs_settle" { + t.Fatalf("settlement correlation = %+v", events[0]) + } + if committed { + t.Fatal("delete failure committed deferred handler success") + } + if !handlerSettlementOK || !observedSettlementOK || observedSettlement != handlerSettlement { + t.Fatal("settlement observer did not retain the handler's delivery identity") + } +} + +// TestSQSWorkerRetrySettlementFailureUsesDeliveredAttempt verifies replacement metadata does not overwrite the unsettled receipt's correlation. +func TestSQSWorkerRetrySettlementFailureUsesDeliveredAttempt(t *testing.T) { + stub := &sqsWorkerClientStub{deleteErr: errors.New("delete failed")} + var events []queue.Event + var handlerSettlement busruntime.DeliverySettlementIdentity + var handlerSettlementOK bool + var observedSettlement busruntime.DeliverySettlementIdentity + var observedSettlementOK bool + w := &sqsWorker{ + handlers: map[string]queue.Handler{"job:retry:settlement": func(ctx context.Context, _ queue.Job) error { + handlerSettlement, handlerSettlementOK = busruntime.DeliverySettlementIdentityFromContext(ctx) + return errors.New("retry me") + }}, + client: stub, + queueURL: "https://example.local/queue/default", + observer: queue.ObserverFunc(func(ctx context.Context, event queue.Event) { + observedSettlement, observedSettlementOK = busruntime.DeliverySettlementIdentityFromContext(ctx) + events = append(events, event) + }), + } + body, err := json.Marshal(sqsMessage{Type: "job:retry:settlement", Queue: "critical", Attempt: 1, MaxRetry: 3}) + if err != nil { + t.Fatalf("marshal body: %v", err) + } + w.process(context.Background(), sqstypes.Message{Body: aws.String(string(body)), ReceiptHandle: aws.String("rh-retry")}) + if len(stub.sendInputs) != 1 { + t.Fatalf("replacement sends = %d, want 1", len(stub.sendInputs)) + } + if len(events) != 1 || events[0].Kind != queue.EventSettlementFailed || events[0].Attempt != 1 { + t.Fatalf("settlement events = %+v, want original attempt 1", events) + } + if !handlerSettlementOK || !observedSettlementOK || observedSettlement != handlerSettlement { + t.Fatal("retry settlement observer did not retain the handler's delivery identity") + } } func (s *sqsWorkerClientStub) SendMessage(_ context.Context, params *sqs.SendMessageInput, _ ...func(*sqs.Options)) (*sqs.SendMessageOutput, error) { @@ -42,7 +161,13 @@ func (s *sqsWorkerClientStub) SendMessage(_ context.Context, params *sqs.SendMes if s.sendErr != nil { return nil, s.sendErr } - return &sqs.SendMessageOutput{}, nil + if s.sendNil { + return nil, nil + } + if s.sendEmptyID { + return &sqs.SendMessageOutput{}, nil + } + return &sqs.SendMessageOutput{MessageId: aws.String("msg-1")}, nil } func decodeSQSBody(t *testing.T, input *sqs.SendMessageInput) sqsMessage { @@ -57,6 +182,28 @@ func decodeSQSBody(t *testing.T, input *sqs.SendMessageInput) sqsMessage { return out } +// TestSQSSendAcceptedRequiresMessageID verifies only a service receipt crosses the publish boundary. +func TestSQSSendAcceptedRequiresMessageID(t *testing.T) { + tests := []struct { + name string + output *sqs.SendMessageOutput + wantErr bool + }{ + {name: "nil output", wantErr: true}, + {name: "missing id", output: &sqs.SendMessageOutput{}, wantErr: true}, + {name: "blank id", output: &sqs.SendMessageOutput{MessageId: aws.String(" ")}, wantErr: true}, + {name: "accepted", output: &sqs.SendMessageOutput{MessageId: aws.String("msg-1")}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + err := sqsSendAccepted(test.output) + if (err != nil) != test.wantErr { + t.Fatalf("sqsSendAccepted() error = %v, wantErr %t", err, test.wantErr) + } + }) + } +} + func TestSQSWorker_ProcessFutureMessageRepublishesAndDeletes(t *testing.T) { stub := &sqsWorkerClientStub{} w := &sqsWorker{ @@ -118,6 +265,34 @@ func TestSQSWorker_ProcessFutureMessageRepublishFailureDoesNotDelete(t *testing. } } +// TestSQSWorkerMissingSendReceiptDoesNotDelete verifies an ambiguous replacement send leaves the original redeliverable. +func TestSQSWorkerMissingSendReceiptDoesNotDelete(t *testing.T) { + tests := []struct { + name string + stub *sqsWorkerClientStub + }{ + {name: "nil output", stub: &sqsWorkerClientStub{sendNil: true}}, + {name: "empty message id", stub: &sqsWorkerClientStub{sendEmptyID: true}}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + w := &sqsWorker{handlers: map[string]queue.Handler{}, client: test.stub, queueURL: "https://example.local/queue/default"} + body, err := json.Marshal(sqsMessage{ + Type: "job:future", + Queue: "default", + AvailableAtMS: time.Now().Add(2 * time.Second).UnixMilli(), + }) + if err != nil { + t.Fatalf("marshal body: %v", err) + } + w.process(context.Background(), sqstypes.Message{Body: aws.String(string(body)), ReceiptHandle: aws.String("rh-1")}) + if len(test.stub.sendInputs) != 1 || len(test.stub.deleteInputs) != 0 { + t.Fatalf("send/delete calls = %d/%d, want 1/0", len(test.stub.sendInputs), len(test.stub.deleteInputs)) + } + }) + } +} + func TestSQSWorker_RepublishFailureEmitsObserverEvent(t *testing.T) { stub := &sqsWorkerClientStub{sendErr: errors.New("send failed")} var events []queue.Event @@ -125,7 +300,7 @@ func TestSQSWorker_RepublishFailureEmitsObserverEvent(t *testing.T) { handlers: map[string]queue.Handler{}, client: stub, queueURL: "https://example.local/queue/default", - observer: queue.ObserverFunc(func(e queue.Event) { events = append(events, e) }), + observer: queue.ObserverFunc(func(_ context.Context, e queue.Event) { events = append(events, e) }), } body, err := json.Marshal(sqsMessage{ @@ -141,6 +316,9 @@ func TestSQSWorker_RepublishFailureEmitsObserverEvent(t *testing.T) { if len(events) == 0 || events[0].Kind != queue.EventRepublishFailed || events[0].Driver != queue.DriverSQS || events[0].Queue != "critical" { t.Fatalf("expected republish_failed event for sqs, got %+v", events) } + if events[0].Layer != queue.EventLayerWorker { + t.Fatalf("republish_failed layer = %q, want worker", events[0].Layer) + } } func TestSQSWorker_RepublishFailureUnwrapsBusEnvelopeJobType(t *testing.T) { @@ -150,14 +328,14 @@ func TestSQSWorker_RepublishFailureUnwrapsBusEnvelopeJobType(t *testing.T) { handlers: map[string]queue.Handler{}, client: stub, queueURL: "https://example.local/queue/default", - observer: queue.ObserverFunc(func(e queue.Event) { events = append(events, e) }), + observer: queue.ObserverFunc(func(_ context.Context, e queue.Event) { events = append(events, e) }), } body, err := json.Marshal(sqsMessage{ Type: "bus:job", Queue: "critical", AvailableAtMS: time.Now().Add(2 * time.Second).UnixMilli(), - Payload: []byte(`{"job":{"type":"monitoring:check"}}`), + Payload: []byte(`{"schema_version":1,"dispatch_id":"dsp_sqs","job_id":"job_sqs","batch_id":"bat_sqs","job":{"type":"monitoring:check"}}`), }) if err != nil { t.Fatalf("marshal body: %v", err) @@ -170,11 +348,15 @@ func TestSQSWorker_RepublishFailureUnwrapsBusEnvelopeJobType(t *testing.T) { if events[0].JobType != "monitoring:check" { t.Fatalf("expected unwrapped observed job type, got %q", events[0].JobType) } + if events[0].DispatchID != "dsp_sqs" || events[0].JobID != "job_sqs" || events[0].BatchID != "bat_sqs" { + t.Fatalf("expected correlated sqs event, got %+v", events[0]) + } } func TestSQSWorker_ProcessSuccessInvokesHandlerAndDeletes(t *testing.T) { stub := &sqsWorkerClientStub{} called := 0 + committed := false w := &sqsWorker{ handlers: map[string]queue.Handler{ "job:ok": func(ctx context.Context, job queue.Job) error { @@ -189,6 +371,9 @@ func TestSQSWorker_ProcessSuccessInvokesHandlerAndDeletes(t *testing.T) { if opts.MaxRetry == nil || *opts.MaxRetry != 3 { t.Fatalf("expected max retry 3, got %+v", opts.MaxRetry) } + if !busruntime.DeferUntilDeliveryCommitted(ctx, func() { committed = true }) { + t.Fatal("handler context did not carry a settlement boundary") + } return nil }, }, @@ -221,6 +406,9 @@ func TestSQSWorker_ProcessSuccessInvokesHandlerAndDeletes(t *testing.T) { if len(stub.deleteInputs) != 1 { t.Fatalf("expected one delete on success, got %d", len(stub.deleteInputs)) } + if !committed { + t.Fatal("successful delete did not commit deferred handler success") + } } func TestSQSWorker_ProcessFailureRetryAndTerminal(t *testing.T) { @@ -321,6 +509,72 @@ func TestSQSWorker_ProcessFailureRetryAndTerminal(t *testing.T) { }) } +// TestSQSWorker_AttemptDecisionSettlement verifies terminal work is deleted while uncommitted work remains available for redelivery. +func TestSQSWorker_AttemptDecisionSettlement(t *testing.T) { + t.Run("permanent failure deletes without republishing", func(t *testing.T) { + stub := &sqsWorkerClientStub{} + w := &sqsWorker{ + handlers: map[string]queue.Handler{ + "job:permanent": func(ctx context.Context, _ queue.Job) error { + attempt, ok := busruntime.DeliveryAttemptFromContext(ctx) + if !ok || attempt.Number != 0 || attempt.MaxRetry != 3 { + t.Fatalf("unexpected delivery attempt: %+v, present=%t", attempt, ok) + } + return busruntime.Permanent(errors.New("invalid job")) + }, + }, + client: stub, + queueURL: "https://example.local/queue/default", + } + body, err := json.Marshal(sqsMessage{Type: "job:permanent", Queue: "default", MaxRetry: 3}) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + w.process(context.Background(), sqstypes.Message{Body: aws.String(string(body)), ReceiptHandle: aws.String("rh-permanent")}) + + if len(stub.sendInputs) != 0 { + t.Fatalf("permanent failure must not republish, got %d sends", len(stub.sendInputs)) + } + if len(stub.deleteInputs) != 1 { + t.Fatalf("permanent failure must delete its receipt, got %d deletes", len(stub.deleteInputs)) + } + }) + + t.Run("uncommitted failure leaves the original receipt", func(t *testing.T) { + stub := &sqsWorkerClientStub{} + w := &sqsWorker{ + handlers: map[string]queue.Handler{ + "job:uncommitted": func(ctx context.Context, _ queue.Job) error { + attempt, ok := busruntime.DeliveryAttemptFromContext(ctx) + if !ok || attempt.Number != 1 || attempt.MaxRetry != 4 { + t.Fatalf("unexpected delivery attempt: %+v, present=%t", attempt, ok) + } + return busruntime.Uncommitted(errors.New("store unavailable")) + }, + }, + client: stub, + queueURL: "https://example.local/queue/default", + } + body, err := json.Marshal(sqsMessage{ + Type: "job:uncommitted", + Queue: "default", + Attempt: 1, + MaxRetry: 4, + BackoffMillis: 1_000, + }) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + w.process(context.Background(), sqstypes.Message{Body: aws.String(string(body)), ReceiptHandle: aws.String("rh-uncommitted")}) + + if len(stub.sendInputs) != 0 || len(stub.deleteInputs) != 0 { + t.Fatalf("uncommitted failure must await SQS redelivery, got sends=%d deletes=%d", len(stub.sendInputs), len(stub.deleteInputs)) + } + }) +} + func TestSQSWorker_NewRegisterAndShutdown(t *testing.T) { backend := newSQSWorker(sqsWorkerConfig{}) if backend.cfg.DefaultQueue != "default" { @@ -363,6 +617,34 @@ func TestSQSWorker_StartWorkersFastPaths(t *testing.T) { } } +// TestSQSWorkerShutdownHonorsDeadline verifies a stuck in-flight handler cannot block the caller forever. +func TestSQSWorkerShutdownHonorsDeadline(t *testing.T) { + w := newSQSWorker(sqsWorkerConfig{}) + w.started = true + w.cancel = func() {} + release := make(chan struct{}) + w.wg.Add(1) + go func() { + defer w.wg.Done() + <-release + }() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Millisecond) + defer cancel() + if err := w.Shutdown(ctx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("shutdown error = %v, want deadline exceeded", err) + } + if !w.started { + t.Fatal("timed-out shutdown exposed the worker as restartable while work remained") + } + close(release) + if err := w.Shutdown(context.Background()); err != nil { + t.Fatalf("complete shutdown: %v", err) + } + if w.started { + t.Fatal("completed shutdown retained started state") + } +} + func TestSQSWorker_StartWorkersInvalidEndpoint(t *testing.T) { backend := newSQSWorker(sqsWorkerConfig{ DefaultQueue: "default", @@ -382,14 +664,42 @@ func TestSQSWorker_StartWorkersInvalidEndpoint(t *testing.T) { } } -func TestSQSWorker_DeleteIgnoresNilReceiptHandle(t *testing.T) { +// TestSQSWorkerDeleteRejectsNilReceiptHandle verifies missing settlement identity cannot commit handler success. +func TestSQSWorkerDeleteRejectsNilReceiptHandle(t *testing.T) { stub := &sqsWorkerClientStub{} w := &sqsWorker{ client: stub, queueURL: "https://example.local/queue/default", } - w.delete(context.Background(), sqstypes.Message{}) + if err := w.delete(sqstypes.Message{}); err == nil { + t.Fatal("delete without receipt unexpectedly committed") + } if len(stub.deleteInputs) != 0 { t.Fatalf("expected no delete call for nil receipt handle, got %d", len(stub.deleteInputs)) } } + +// TestSQSWorkerMalformedMessagesAreDeleted verifies poison deliveries without a +// usable envelope are settled without entering application handlers. +func TestSQSWorkerMalformedMessagesAreDeleted(t *testing.T) { + stub := &sqsWorkerClientStub{} + w := &sqsWorker{client: stub, queueURL: "https://example.local/queue/default"} + + w.process(context.Background(), sqstypes.Message{ReceiptHandle: aws.String("rh-empty")}) + w.process(context.Background(), sqstypes.Message{ + Body: aws.String("not-json"), + ReceiptHandle: aws.String("rh-invalid"), + }) + if len(stub.deleteInputs) != 2 { + t.Fatalf("poison-message deletes = %d, want 2", len(stub.deleteInputs)) + } +} + +// TestSQSWorkerNilShutdownNormalizesContext verifies a stopped worker accepts +// the optional lifecycle context without starting network work. +func TestSQSWorkerNilShutdownNormalizesContext(t *testing.T) { + w := &sqsWorker{} + if err := w.Shutdown(nil); err != nil { + t.Fatalf("nil-context shutdown: %v", err) + } +} diff --git a/driver_runtime.go b/driver_runtime.go index 77aeda7..d772681 100644 --- a/driver_runtime.go +++ b/driver_runtime.go @@ -3,6 +3,8 @@ package queue import ( "context" "fmt" + + "github.com/goforj/queue/busruntime" ) type driverQueueBackend interface { @@ -15,6 +17,7 @@ type driverRuntimeQueueBackend interface { driverQueueBackend Register(jobType string, handler Handler) StartWorkers(ctx context.Context) error + DrainWorkers(ctx context.Context) error } type driverWorkerBackend interface { @@ -30,6 +33,7 @@ func newQueueFromDriver(cfg Config, backend driverQueueBackend, workerFactory dr return nil, fmt.Errorf("driver backend is nil") } cfg = cfg.normalize() + cfg.Observer = ensureObserverSink(cfg.Observer) var q queueBackend var runtime runtimeQueueBackend @@ -47,14 +51,20 @@ func newQueueFromDriver(cfg Config, backend driverQueueBackend, workerFactory dr } if runtime != nil { return &nativeQueueRuntime{ - common: common, - runtime: runtime, - registered: make(map[string]Handler), + common: common, + runtime: runtime, + nativeQueueRuntimeState: &nativeQueueRuntimeState{ + registered: make(map[string]Handler), + continuation: busruntime.NewContinuationScope(), + }, }, nil } return &externalQueueRuntime{ - common: common, - registered: make(map[string]Handler), - newWorker: workerFactory, + common: common, + newWorker: workerFactory, + externalQueueRuntimeState: &externalQueueRuntimeState{ + registered: make(map[string]Handler), + continuation: busruntime.NewContinuationScope(), + }, }, nil } diff --git a/error_contract_test.go b/error_contract_test.go index bb679d8..7c39d89 100644 --- a/error_contract_test.go +++ b/error_contract_test.go @@ -22,8 +22,10 @@ func TestQueueErrorContract_DispatchCancellation(t *testing.T) { cfg: cfg, driver: cfg.Driver, }, - runtime: backend, - registered: make(map[string]Handler), + runtime: backend, + nativeQueueRuntimeState: &nativeQueueRuntimeState{ + registered: make(map[string]Handler), + }, } q, err := newQueueFromRuntime(rt) if err != nil { diff --git a/examples/fakequeue-batchrecords/main.go b/examples/fakequeue-batchrecords/main.go new file mode 100644 index 0000000..2bd0cc9 --- /dev/null +++ b/examples/fakequeue-batchrecords/main.go @@ -0,0 +1,26 @@ +//go:build ignore +// +build ignore + +// examplegen:generated + +package main + +import ( + "context" + "fmt" + "github.com/goforj/queue" +) + +func main() { + // BatchRecords returns isolated creation records for accepted fake batches. + + // Example: inspect a fake batch + fake := queue.NewFake() + _, _ = fake.Batch( + queue.NewJob("emails:first"), + queue.NewJob("emails:second"), + ).Name("nightly").AllowFailures().Dispatch(context.Background()) + record := fake.BatchRecords()[0] + fmt.Println(record.Name, len(record.Jobs), record.AllowFailed) + // Output: nightly 2 true +} diff --git a/examples/fakequeue-chainrecords/main.go b/examples/fakequeue-chainrecords/main.go new file mode 100644 index 0000000..c94ec71 --- /dev/null +++ b/examples/fakequeue-chainrecords/main.go @@ -0,0 +1,26 @@ +//go:build ignore +// +build ignore + +// examplegen:generated + +package main + +import ( + "context" + "fmt" + "github.com/goforj/queue" +) + +func main() { + // ChainRecords returns isolated creation records for accepted fake chains. + + // Example: inspect a fake chain + fake := queue.NewFake() + _, _ = fake.Chain( + queue.NewJob("reports:build"), + queue.NewJob("reports:publish"), + ).OnQueue("workflow").Dispatch(context.Background()) + record := fake.ChainRecords()[0] + fmt.Println(len(record.Nodes), record.Queue) + // Output: 2 workflow +} diff --git a/examples/fakequeue-ready/main.go b/examples/fakequeue-ready/main.go index 1a0f0ef..8bb4d54 100644 --- a/examples/fakequeue-ready/main.go +++ b/examples/fakequeue-ready/main.go @@ -17,5 +17,5 @@ func main() { // Example: fake ready fake := queue.NewFake() fmt.Println(fake.Ready(context.Background()) == nil) - // true + // Output: true } diff --git a/examples/fakequeue-records/main.go b/examples/fakequeue-records/main.go index c477d21..2644ee1 100644 --- a/examples/fakequeue-records/main.go +++ b/examples/fakequeue-records/main.go @@ -11,7 +11,8 @@ import ( ) func main() { - // Records returns a copy of all dispatch records. + // Records returns isolated records for accepted direct dispatches. + // Chain and batch creation is available through ChainRecords and BatchRecords. // Example: read records fake := queue.NewFake() diff --git a/examples/fakequeue-register/main.go b/examples/fakequeue-register/main.go index a02d533..3c6ddf9 100644 --- a/examples/fakequeue-register/main.go +++ b/examples/fakequeue-register/main.go @@ -11,7 +11,7 @@ import ( ) func main() { - // Register associates a handler with a job type. + // Register is a compatibility no-op because the recording fake never executes handlers. // Example: register no-op on fake fake := queue.NewFake() diff --git a/examples/fakequeue-reset/main.go b/examples/fakequeue-reset/main.go index c7fdc69..b5602c3 100644 --- a/examples/fakequeue-reset/main.go +++ b/examples/fakequeue-reset/main.go @@ -11,7 +11,7 @@ import ( ) func main() { - // Reset clears all recorded dispatches. + // Reset clears direct dispatches and all workflow records through every fake view. // Example: reset records fake := queue.NewFake() diff --git a/examples/fakequeue-shutdown/main.go b/examples/fakequeue-shutdown/main.go index 6bdb34d..c538187 100644 --- a/examples/fakequeue-shutdown/main.go +++ b/examples/fakequeue-shutdown/main.go @@ -11,7 +11,7 @@ import ( ) func main() { - // Shutdown drains running work and releases resources. + // Shutdown is a compatibility no-op because the recording fake owns no worker resources. // Example: shutdown fake queue fake := queue.NewFake() diff --git a/examples/fakequeue-startworkers/main.go b/examples/fakequeue-startworkers/main.go index d05832e..0664337 100644 --- a/examples/fakequeue-startworkers/main.go +++ b/examples/fakequeue-startworkers/main.go @@ -11,7 +11,7 @@ import ( ) func main() { - // StartWorkers starts worker execution. + // StartWorkers is a compatibility no-op because the recording fake owns no workers. // Example: start fake workers fake := queue.NewFake() diff --git a/examples/fakequeue-workers/main.go b/examples/fakequeue-workers/main.go index 30faeb8..c1ea27d 100644 --- a/examples/fakequeue-workers/main.go +++ b/examples/fakequeue-workers/main.go @@ -11,7 +11,7 @@ import ( ) func main() { - // Workers sets desired worker concurrency before StartWorkers. + // Workers preserves fluent lifecycle compatibility without creating workers. // Example: set worker count fake := queue.NewFake() diff --git a/examples/go.mod b/examples/go.mod index 02a16fc..817143f 100644 --- a/examples/go.mod +++ b/examples/go.mod @@ -1,6 +1,6 @@ module github.com/goforj/queue/examples -go 1.24.4 +go 1.25.0 require ( github.com/goforj/queue v0.0.0 @@ -14,6 +14,47 @@ require ( github.com/goforj/queue/driver/sqsqueue v0.0.0 ) +require ( + github.com/aws/aws-sdk-go-v2 v1.41.1 // indirect + github.com/aws/aws-sdk-go-v2/config v1.32.8 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.8 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 // indirect + github.com/aws/aws-sdk-go-v2/service/sqs v1.42.21 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.14 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 // indirect + github.com/aws/smithy-go v1.24.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hibiken/asynq v0.26.0 // indirect + github.com/klauspost/compress v1.18.0 // indirect + github.com/nats-io/nats.go v1.48.0 // indirect + github.com/nats-io/nkeys v0.4.11 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + github.com/rabbitmq/amqp091-go v1.10.0 // indirect + github.com/redis/go-redis/v9 v9.14.1 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/spf13/cast v1.10.0 // indirect + golang.org/x/crypto v0.37.0 // indirect + golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect + golang.org/x/sys v0.40.0 // indirect + golang.org/x/time v0.14.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect + modernc.org/libc v1.67.6 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect + modernc.org/sqlite v1.46.0 // indirect +) + replace github.com/goforj/queue => .. replace github.com/goforj/queue/driver/redisqueue => ../driver/redisqueue diff --git a/examples/go.sum b/examples/go.sum new file mode 100644 index 0000000..0f0a233 --- /dev/null +++ b/examples/go.sum @@ -0,0 +1,76 @@ +github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU= +github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= +github.com/aws/aws-sdk-go-v2/config v1.32.8 h1:iu+64gwDKEoKnyTQskSku72dAwggKI5sV6rNvgSMpMs= +github.com/aws/aws-sdk-go-v2/config v1.32.8/go.mod h1:MI2XvA+qDi3i9AJxX1E2fu730syEBzp/jnXrjxuHwgI= +github.com/aws/aws-sdk-go-v2/credentials v1.19.8 h1:Jp2JYH1lRT3KhX4mshHPvVYsR5qqRec3hGvEarNYoR0= +github.com/aws/aws-sdk-go-v2/credentials v1.19.8/go.mod h1:fZG9tuvyVfxknv1rKibIz3DobRaFw1Poe8IKtXB3XYY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17 h1:I0GyV8wiYrP8XpA70g1HBcQO1JlQxCMTW9npl5UbDHY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.17/go.mod h1:tyw7BOl5bBe/oqvoIeECFJjMdzXoa/dfVz3QQ5lgHGA= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17/go.mod h1:EhG22vHRrvF8oXSTYStZhJc1aUgKtnJe+aOiFEV90cM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 h1:RuNSMoozM8oXlgLG/n6WLaFGoea7/CddrCfIiSA+xdY= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17/go.mod h1:F2xxQ9TZz5gDWsclCtPQscGpP0VUOc8RqgFM3vDENmU= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.5 h1:VrhDvQib/i0lxvr3zqlUwLwJP4fpmpyD9wYG1vfSu+Y= +github.com/aws/aws-sdk-go-v2/service/signin v1.0.5/go.mod h1:k029+U8SY30/3/ras4G/Fnv/b88N4mAfliNn08Dem4M= +github.com/aws/aws-sdk-go-v2/service/sqs v1.42.21 h1:Oa0IhwDLVrcBHDlNo1aosG4CxO4HyvzDV5xUWqWcBc0= +github.com/aws/aws-sdk-go-v2/service/sqs v1.42.21/go.mod h1:t98Ssq+qtXKXl2SFtaSkuT6X42FSM//fnO6sfq5RqGM= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.9 h1:v6EiMvhEYBoHABfbGB4alOYmCIrcgyPPiBE1wZAEbqk= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.9/go.mod h1:yifAsgBxgJWn3ggx70A3urX2AN49Y5sJTD1UQFlfqBw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.14 h1:0jbJeuEHlwKJ9PfXtpSFc4MF+WIWORdhN1n30ITZGFM= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.14/go.mod h1:sTGThjphYE4Ohw8vJiRStAcu3rbjtXRsdNB0TvZ5wwo= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.6 h1:5fFjR/ToSOzB2OQ/XqWpZBmNvmP/pJ1jOWYlFDJTjRQ= +github.com/aws/aws-sdk-go-v2/service/sts v1.41.6/go.mod h1:qgFDZQSD/Kys7nJnVqYlWKnh0SSdMjAi0uSwON4wgYQ= +github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk= +github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hibiken/asynq v0.26.0 h1:1Zxr92MlDnb1Zt/QR5g2vSCqUS03i95lUfqx5X7/wrw= +github.com/hibiken/asynq v0.26.0/go.mod h1:Qk4e57bTnWDoyJ67VkchuV6VzSM9IQW2nPvAGuDyw58= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/nats-io/nats.go v1.48.0 h1:pSFyXApG+yWU/TgbKCjmm5K4wrHu86231/w84qRVR+U= +github.com/nats-io/nats.go v1.48.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g= +github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0= +github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= +github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/redis/go-redis/v9 v9.14.1 h1:nDCrEiJmfOWhD76xlaw+HXT0c9hfNWeXgl0vIRYSDvQ= +github.com/redis/go-redis/v9 v9.14.1/go.mod h1:huWgSWd8mW6+m0VPhJjSSQ+d6Nh1VICQ6Q5lHuCH/Iw= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= +github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/spf13/cast v1.10.0 h1:h2x0u2shc1QuLHfxi+cTJvs30+ZAHOGRic8uyGTDWxY= +github.com/spf13/cast v1.10.0/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY= +golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70= +golang.org/x/sys v0.40.0 h1:DBZZqJ2Rkml6QMQsZywtnjnnGvHza6BTfYFWY9kjEWQ= +golang.org/x/sys v0.40.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +modernc.org/libc v1.67.6 h1:eVOQvpModVLKOdT+LvBPjdQqfrZq+pC39BygcT+E7OI= +modernc.org/libc v1.67.6/go.mod h1:JAhxUVlolfYDErnwiqaLvUqc8nfb2r6S6slAgZOnaiE= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.46.0 h1:pCVOLuhnT8Kwd0gjzPwqgQW1KW2XFpXyJB6cCw11jRE= +modernc.org/sqlite v1.46.0/go.mod h1:CzbrU2lSB1DKUusvwGz7rqEKIq+NUd8GWuBBZDs9/nA= diff --git a/examples/newfake/main.go b/examples/newfake/main.go index b776838..277a275 100644 --- a/examples/newfake/main.go +++ b/examples/newfake/main.go @@ -11,7 +11,7 @@ import ( ) func main() { - // NewFake creates a queue fake that records dispatches and provides assertions. + // NewFake creates the canonical fake used directly and by deprecated testing adapters. // Example: fake queue assertions fake := queue.NewFake() diff --git a/examples/observeall/main.go b/examples/observeall/main.go index a6fb09f..4c82490 100644 --- a/examples/observeall/main.go +++ b/examples/observeall/main.go @@ -22,40 +22,27 @@ func main() { var flakyAttempts atomic.Int32 ctx := context.Background() - runtimeObserver := queue.ObserverFunc(func(ctx context.Context, event queue.Event) { - logger.Info("runtime event", + observer := queue.ObserverFunc(func(ctx context.Context, event queue.Event) { + logger.Info("queue event", + "layer", event.Layer, "kind", event.Kind, "driver", event.Driver, "queue", event.Queue, - "job_type", event.JobType, - "attempt", event.Attempt, - "max_retry", event.MaxRetry, - "duration", event.Duration, - "err", event.Err, - ) - }) - - workflowObserver := queue.WorkflowObserverFunc(func(ctx context.Context, event queue.WorkflowEvent) { - logger.Info("workflow event", - "kind", event.Kind, "dispatch_id", event.DispatchID, "job_id", event.JobID, "chain_id", event.ChainID, "batch_id", event.BatchID, "job_type", event.JobType, - "queue", event.Queue, "attempt", event.Attempt, + "max_retry", event.MaxRetry, "duration", event.Duration, "err", event.Err, ) }) q, err := queue.New( - queue.Config{ - Driver: queue.DriverWorkerpool, - Observer: runtimeObserver, - }, - queue.WithObserver(workflowObserver), + queue.Config{Driver: queue.DriverWorkerpool}, + queue.WithObserver(observer), ) if err != nil { panic(err) diff --git a/examples/queue-batch/main.go b/examples/queue-batch/main.go index 9a6e656..9d4939d 100644 --- a/examples/queue-batch/main.go +++ b/examples/queue-batch/main.go @@ -19,6 +19,10 @@ func main() { return } q.Register("emails:send", func(ctx context.Context, m queue.Message) error { return nil }) + if err := q.StartWorkers(context.Background()); err != nil { + return + } + defer q.Shutdown(context.Background()) _, _ = q.Batch( queue.NewJob("emails:send").Payload(map[string]any{"id": 1}), queue.NewJob("emails:send").Payload(map[string]any{"id": 2}), diff --git a/examples/queue-chain/main.go b/examples/queue-chain/main.go index 656783d..8306686 100644 --- a/examples/queue-chain/main.go +++ b/examples/queue-chain/main.go @@ -20,6 +20,10 @@ func main() { } q.Register("first", func(ctx context.Context, m queue.Message) error { return nil }) q.Register("second", func(ctx context.Context, m queue.Message) error { return nil }) + if err := q.StartWorkers(context.Background()); err != nil { + return + } + defer q.Shutdown(context.Background()) _, _ = q.Chain( queue.NewJob("first"), queue.NewJob("second"), diff --git a/examples/queue-dispatch/main.go b/examples/queue-dispatch/main.go index f264d7b..ba613bc 100644 --- a/examples/queue-dispatch/main.go +++ b/examples/queue-dispatch/main.go @@ -11,7 +11,8 @@ import ( ) func main() { - // Dispatch enqueues a high-level job using the queue's bound context. + // Dispatch enqueues a high-level job using its application type and exact + // payload bytes together with the queue's bound context. // Example: dispatch q, err := queue.NewSync() @@ -19,6 +20,10 @@ func main() { return } q.Register("emails:send", func(ctx context.Context, m queue.Message) error { return nil }) + if err := q.StartWorkers(context.Background()); err != nil { + return + } + defer q.Shutdown(context.Background()) job := queue.NewJob("emails:send").Payload(map[string]any{"id": 1}).OnQueue("default") _, _ = q.Dispatch(job) } diff --git a/examples/withobserver/main.go b/examples/withobserver/main.go index 3492658..24bb648 100644 --- a/examples/withobserver/main.go +++ b/examples/withobserver/main.go @@ -11,10 +11,10 @@ import ( ) func main() { - // WithObserver installs a workflow lifecycle observer. + // WithObserver installs one observer for queue, worker, and workflow lifecycle events. - // Example: workflow observer - observer := queue.WorkflowObserverFunc(func(_ context.Context, event queue.WorkflowEvent) { + // Example: observe all queue activity + observer := queue.ObserverFunc(func(_ context.Context, event queue.Event) { _ = event.Kind }) q, err := queue.New(queue.Config{Driver: queue.DriverSync}, queue.WithObserver(observer)) diff --git a/fake_queue.go b/fake_queue.go index 5751ecd..50d6670 100644 --- a/fake_queue.go +++ b/fake_queue.go @@ -2,9 +2,6 @@ package queue import ( "context" - "encoding/json" - "fmt" - "reflect" "sync" "testing" @@ -18,20 +15,26 @@ type DispatchRecord struct { Queue string } -// FakeQueue is an in-memory queue fake for tests. +// FakeQueue is the concurrency-safe queue and workflow fake for tests. // @group Testing type FakeQueue struct { state *fakeQueueState ctx context.Context } +// fakeQueueState owns every mutable projection shared by context-bound and +// compatibility fake handles. type fakeQueueState struct { defaultQueue string - mu sync.RWMutex - records []DispatchRecord + // workflowOps prevents Reset or Prune from splitting one engine dispatch + // across state generations or intermediate terminal transitions. + workflowOps sync.RWMutex + mu sync.RWMutex + records []DispatchRecord + workflow *fakeWorkflowRecorder } -// NewFake creates a queue fake that records dispatches and provides assertions. +// NewFake creates the canonical fake used directly and by deprecated testing adapters. // @group Testing // // Example: fake queue assertions @@ -46,12 +49,14 @@ type fakeQueueState struct { // fmt.Println(len(records), records[0].Queue, records[0].Job.Type) // // Output: 1 critical emails:send func NewFake() *FakeQueue { - return &FakeQueue{ + fake := &FakeQueue{ state: &fakeQueueState{ defaultQueue: "default", records: make([]DispatchRecord, 0), }, } + fake.state.workflow = newFakeWorkflowRecorder(fake) + return fake } // Driver returns the active queue driver. @@ -64,6 +69,15 @@ func NewFake() *FakeQueue { // _ = driver func (f *FakeQueue) Driver() Driver { return DriverNull } +// physicalQueueNameOrDefault keeps fake event labels aligned with its recording queue names. +func (f *FakeQueue) physicalQueueNameOrDefault(queueName string) string { + defaultQueue := "default" + if f != nil && f.state != nil && f.state.defaultQueue != "" { + defaultQueue = f.state.defaultQueue + } + return PhysicalQueueName(defaultQueue, queueName) +} + // WithContext returns a derived fake queue handle bound to ctx. // @group Testing func (f *FakeQueue) WithContext(ctx context.Context) queueRuntime { @@ -75,6 +89,8 @@ func (f *FakeQueue) WithContext(ctx context.Context) queueRuntime { return &clone } +// setHandlerContextDecorator remains inert because the fake records intent and +// never invokes registered handlers. func (f *FakeQueue) setHandlerContextDecorator(func(context.Context) context.Context) {} // Dispatch records a typed job payload in-memory using the fake default queue. @@ -90,29 +106,39 @@ func (f *FakeQueue) Dispatch(job any) error { if f != nil && f.ctx != nil { ctx = f.ctx } - if ctx != nil { - if err := ctx.Err(); err != nil { - return err - } - } - dispatchJob, err := f.jobFromAny(job) + return f.dispatch(ctx, job) +} + +// dispatch validates and freezes one accepted job before publishing it to all +// fake views that share this state. +func (f *FakeQueue) dispatch(ctx context.Context, job any) error { + dispatchJob, err := normalizeDispatchJob(job, f.state.defaultQueue) if err != nil { return err } + if err := dispatchJob.validate(); err != nil { + return err + } + if ctx == nil { + ctx = context.Background() + } + if err := ctx.Err(); err != nil { + return err + } queueName := dispatchJob.jobOptions().queueName if queueName == "" { queueName = f.state.defaultQueue } f.state.mu.Lock() f.state.records = append(f.state.records, DispatchRecord{ - Job: dispatchJob, + Job: cloneFakeJob(dispatchJob), Queue: queueName, }) f.state.mu.Unlock() return nil } -// Register associates a handler with a job type. +// Register is a compatibility no-op because the recording fake never executes handlers. // @group Testing // // Example: register no-op on fake @@ -121,7 +147,7 @@ func (f *FakeQueue) Dispatch(job any) error { // fake.Register("emails:send", func(context.Context, queue.Job) error { return nil }) func (f *FakeQueue) Register(string, Handler) {} -// StartWorkers starts worker execution. +// StartWorkers is a compatibility no-op because the recording fake owns no workers. // @group Testing // // Example: start fake workers @@ -131,7 +157,7 @@ func (f *FakeQueue) Register(string, Handler) {} // _ = err func (f *FakeQueue) StartWorkers(context.Context) error { return nil } -// Workers sets desired worker concurrency before StartWorkers. +// Workers preserves fluent lifecycle compatibility without creating workers. // @group Testing // // Example: set worker count @@ -142,7 +168,7 @@ func (f *FakeQueue) StartWorkers(context.Context) error { return nil } // // Output: true func (f *FakeQueue) Workers(int) queueRuntime { return f } -// Shutdown drains running work and releases resources. +// Shutdown is a compatibility no-op because the recording fake owns no worker resources. // @group Testing // // Example: shutdown fake queue @@ -159,7 +185,7 @@ func (f *FakeQueue) Shutdown(context.Context) error { return nil } // // fake := queue.NewFake() // fmt.Println(fake.Ready(context.Background()) == nil) -// // true +// // Output: true func (f *FakeQueue) Ready(ctx context.Context) error { if ctx == nil { return nil @@ -174,6 +200,27 @@ func (f *FakeQueue) BusRegister(string, busruntime.Handler) {} // BusDispatch satisfies the internal orchestration runtime adapter. // @group Testing func (f *FakeQueue) BusDispatch(ctx context.Context, jobType string, payload []byte, opts busruntime.JobOptions) error { + job := fakeBusJob(jobType, payload, opts, true) + if fakeWorkflowDeliverySuppressed(ctx, jobType) { + if err := ctx.Err(); err != nil { + return err + } + return job.validate() + } + return f.dispatch(ctx, job) +} + +// BusDispatchDirect records the application job and its correlation metadata +// without introducing the legacy workflow envelope. +// @group Testing +func (f *FakeQueue) BusDispatchDirect(ctx context.Context, jobType string, payload []byte, metadata busruntime.DeliveryMetadata, opts busruntime.JobOptions) error { + job := DriverWithMetadata(fakeBusJob(jobType, payload, opts, false), metadata) + return f.dispatch(ctx, job) +} + +// fakeBusJob mirrors the production runtime adapter so explicit retry zero and +// workflow identity retain their delivery meaning in tests. +func fakeBusJob(jobType string, payload []byte, opts busruntime.JobOptions, legacyIdentity bool) Job { job := NewJob(jobType).Payload(payload) if opts.Queue != "" { job = job.OnQueue(opts.Queue) @@ -184,19 +231,21 @@ func (f *FakeQueue) BusDispatch(ctx context.Context, jobType string, payload []b if opts.Timeout > 0 { job = job.Timeout(opts.Timeout) } - if opts.Retry > 0 { - job = job.Retry(opts.Retry) - } + job = job.Retry(opts.Retry) if opts.Backoff > 0 { job = job.Backoff(opts.Backoff) } if opts.UniqueFor > 0 { job = job.UniqueFor(opts.UniqueFor) + if legacyIdentity { + logical := resolveLogicalJob(jobType, payload) + job = job.withLogicalIdentity(logical.jobType, logical.payload) + } } - return f.WithContext(ctx).Dispatch(job) + return job } -// Reset clears all recorded dispatches. +// Reset clears direct dispatches and all workflow records through every fake view. // @group Testing // // Example: reset records @@ -210,12 +259,16 @@ func (f *FakeQueue) BusDispatch(ctx context.Context, jobType string, payload []b // // 1 // // 0 func (f *FakeQueue) Reset() { + f.state.workflowOps.Lock() + defer f.state.workflowOps.Unlock() f.state.mu.Lock() - f.state.records = f.state.records[:0] + f.state.records = nil + f.state.workflow.resetLocked() f.state.mu.Unlock() } -// Records returns a copy of all dispatch records. +// Records returns isolated records for accepted direct dispatches. +// Chain and batch creation is available through ChainRecords and BatchRecords. // @group Testing // // Example: read records @@ -229,11 +282,16 @@ func (f *FakeQueue) Records() []DispatchRecord { f.state.mu.RLock() defer f.state.mu.RUnlock() out := make([]DispatchRecord, len(f.state.records)) - copy(out, f.state.records) + for i, record := range f.state.records { + out[i] = DispatchRecord{ + Job: cloneFakeJob(record.Job), + Queue: record.Queue, + } + } return out } -// AssertNothingDispatched fails when any dispatch was recorded. +// AssertNothingDispatched fails when any direct dispatch was recorded. // @group Testing // // Example: assert nothing dispatched @@ -247,7 +305,7 @@ func (f *FakeQueue) AssertNothingDispatched(t testing.TB) { } } -// AssertCount fails when dispatch count is not expected. +// AssertCount fails when the direct dispatch count is not expected. // @group Testing // // Example: assert dispatch count @@ -340,42 +398,22 @@ func (f *FakeQueue) AssertNotDispatched(t testing.TB, jobType string) { } } -func (f *FakeQueue) jobFromAny(job any) (Job, error) { - if job, ok := job.(Job); ok { - if job.Type == "" { - return Job{}, fmt.Errorf("dispatch job type is required") - } - return job, nil - } - if job == nil { - return Job{}, fmt.Errorf("dispatch job is nil") - } - jobType := fakeJobTypeFromValue(job) - if jobType == "" { - return Job{}, fmt.Errorf("dispatch job type could not be inferred") - } - if typed, ok := job.(interface{ JobType() string }); ok { - if t := typed.JobType(); t != "" { - jobType = t - } - } - payload, err := json.Marshal(job) - if err != nil { - return Job{}, fmt.Errorf("marshal dispatch job: %w", err) - } - return NewJob(jobType).Payload(payload).OnQueue(f.state.defaultQueue), nil -} - -func fakeJobTypeFromValue(v any) string { - t := reflect.TypeOf(v) - if t == nil { - return "" +// cloneFakeJob isolates the mutable payload and option pointers exposed by +// driver helpers so inspection cannot rewrite previously accepted evidence. +func cloneFakeJob(job Job) Job { + job.payload = cloneWorkflowPayload(job.payload) + job.options.logicalPayload = cloneWorkflowPayload(job.options.logicalPayload) + if job.options.timeout != nil { + value := *job.options.timeout + job.options.timeout = &value } - if t.Kind() == reflect.Pointer { - t = t.Elem() + if job.options.maxRetry != nil { + value := *job.options.maxRetry + job.options.maxRetry = &value } - if t.Name() == "" { - return "" + if job.options.backoff != nil { + value := *job.options.backoff + job.options.backoff = &value } - return t.Name() + return job } diff --git a/fake_queue_context_test.go b/fake_queue_context_test.go new file mode 100644 index 0000000..6fcff45 --- /dev/null +++ b/fake_queue_context_test.go @@ -0,0 +1,40 @@ +package queue + +import ( + "context" + "errors" + "testing" +) + +// TestFakeQueueDispatchNormalizesNilContext verifies internal runtime adapters may dispatch without pre-normalizing context. +func TestFakeQueueDispatchNormalizesNilContext(t *testing.T) { + fake := NewFake() + if err := fake.dispatch(nil, NewJob("emails:send")); err != nil { + t.Fatalf("dispatch with nil context: %v", err) + } + fake.AssertDispatched(t, "emails:send") +} + +// TestFakeQueueReadinessReflectsContext verifies the fake matches production readiness cancellation semantics. +func TestFakeQueueReadinessReflectsContext(t *testing.T) { + fake := NewFake() + if err := fake.Ready(nil); err != nil { + t.Fatalf("Ready(nil): %v", err) + } + if err := fake.Ready(context.Background()); err != nil { + t.Fatalf("Ready(background): %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := fake.Ready(ctx); !errors.Is(err, context.Canceled) { + t.Fatalf("Ready(canceled) = %v, want %v", err, context.Canceled) + } +} + +// TestFakeQueueNilHandleWithContextPreservesNil verifies derived handles do not turn an absent fake into a usable runtime. +func TestFakeQueueNilHandleWithContextPreservesNil(t *testing.T) { + var fake *FakeQueue + if got := fake.WithContext(context.Background()); got != nil { + t.Fatalf("nil fake WithContext returned %T, want nil", got) + } +} diff --git a/fake_workflow.go b/fake_workflow.go new file mode 100644 index 0000000..a8d9f14 --- /dev/null +++ b/fake_workflow.go @@ -0,0 +1,458 @@ +package queue + +import ( + "context" + "testing" + "time" + + "github.com/goforj/queue/internal/workflow" +) + +// fakeWorkflowRecorder runs fake workflows through the production engine while +// retaining immutable creation records for assertion-friendly inspection. +type fakeWorkflowRecorder struct { + state *fakeQueueState + engine workflow.Engine + store fakeWorkflowStateStore + chains map[string]ChainRecord + batches map[string]BatchRecord + acceptedChainIDs []string + acceptedBatchIDs []string +} + +// fakeWorkflowStateStore narrows the internal memory store to exact rejection +// cleanup without widening the production workflow.Store contract. +type fakeWorkflowStateStore interface { + workflow.Store + // FailChainNode preserves first-writer ownership for duplicate fake deliveries. + FailChainNode(context.Context, string, string, error) (workflow.ChainState, bool, error) + // SettleBatchJob preserves first-writer ownership for duplicate fake deliveries. + SettleBatchJob(context.Context, string, string, workflow.BatchJobOutcome, error) (workflow.BatchState, bool, error) + // DiscardChain removes exactly one rejected chain from recording state. + DiscardChain(string) + // DiscardBatch removes exactly one rejected batch from recording state. + DiscardBatch(string) +} + +// newFakeWorkflowStateStore fails fast if the internal recording store loses +// the exact cleanup capability required by concurrent fake dispatches. +func newFakeWorkflowStateStore() fakeWorkflowStateStore { + store, ok := workflow.NewMemoryStore().(fakeWorkflowStateStore) + if !ok { + panic("workflow memory store does not support exact fake cleanup") + } + return store +} + +// newFakeWorkflowRecorder wires the real workflow engine to the recording +// transport and store owned by one FakeQueue state. +func newFakeWorkflowRecorder(fake *FakeQueue) *fakeWorkflowRecorder { + recorder := &fakeWorkflowRecorder{ + state: fake.state, + store: newFakeWorkflowStateStore(), + chains: make(map[string]ChainRecord), + batches: make(map[string]BatchRecord), + } + engine, err := workflow.NewWithStore(fake, recorder, workflow.WithoutEphemeralCallbacks()) + if err != nil { + panic(err) + } + recorder.engine = engine + return recorder +} + +// resetLocked replaces persisted workflow state while the shared fake mutex +// prevents readers from observing a half-reset projection. +func (r *fakeWorkflowRecorder) resetLocked() { + r.store = newFakeWorkflowStateStore() + r.chains = make(map[string]ChainRecord) + r.batches = make(map[string]BatchRecord) + r.acceptedChainIDs = nil + r.acceptedBatchIDs = nil +} + +// acceptChain publishes a chain to assertions only after its initial delivery +// was accepted; merely creating or rejecting a builder must remain invisible. +func (r *fakeWorkflowRecorder) acceptChain(chainID string) { + r.state.mu.Lock() + defer r.state.mu.Unlock() + if _, exists := r.chains[chainID]; !exists { + return + } + r.acceptedChainIDs = append(r.acceptedChainIDs, chainID) +} + +// acceptBatch publishes a batch to assertions only after every initial member +// delivery was accepted by the recording transport. +func (r *fakeWorkflowRecorder) acceptBatch(batchID string) { + r.state.mu.Lock() + defer r.state.mu.Unlock() + if _, exists := r.batches[batchID]; !exists { + return + } + r.acceptedBatchIDs = append(r.acceptedBatchIDs, batchID) +} + +// rejectChain removes state created before a failed initial delivery so +// rejected fake workflows cannot accumulate hidden terminal records. +func (r *fakeWorkflowRecorder) rejectChain(chainID string) { + r.state.mu.Lock() + defer r.state.mu.Unlock() + r.store.DiscardChain(chainID) + delete(r.chains, chainID) +} + +// rejectBatch removes state created before a failed member delivery so +// rejected fake workflows cannot accumulate hidden terminal records. +func (r *fakeWorkflowRecorder) rejectBatch(batchID string) { + r.state.mu.Lock() + defer r.state.mu.Unlock() + r.store.DiscardBatch(batchID) + delete(r.batches, batchID) +} + +// CreateChain records the exact committed engine model before the first node is +// offered to the fake transport. +func (r *fakeWorkflowRecorder) CreateChain(ctx context.Context, record workflow.ChainRecord) error { + r.state.mu.Lock() + defer r.state.mu.Unlock() + if err := r.store.CreateChain(ctx, record); err != nil { + return err + } + r.chains[record.ChainID] = chainRecordFromWorkflow(record) + return nil +} + +// AdvanceChain delegates the retry-safe transition while serializing Reset +// against the same in-memory store generation. +func (r *fakeWorkflowRecorder) AdvanceChain(ctx context.Context, chainID string, completedNode string) (*workflow.ChainNode, bool, error) { + r.state.mu.Lock() + defer r.state.mu.Unlock() + next, done, err := r.store.AdvanceChain(ctx, chainID, completedNode) + if next == nil { + return nil, done, err + } + cloned := chainNodeToWorkflow(chainNodeFromWorkflow(*next)) + return &cloned, done, err +} + +// FailChain delegates terminal failure within the active fake state generation. +func (r *fakeWorkflowRecorder) FailChain(ctx context.Context, chainID string, cause error) error { + r.state.mu.Lock() + defer r.state.mu.Unlock() + return r.store.FailChain(ctx, chainID, cause) +} + +// FailChainNode delegates atomic node failure within the active fake state generation. +func (r *fakeWorkflowRecorder) FailChainNode(ctx context.Context, chainID, nodeID string, cause error) (workflow.ChainState, bool, error) { + r.state.mu.Lock() + defer r.state.mu.Unlock() + return r.store.FailChainNode(ctx, chainID, nodeID, cause) +} + +// GetChain returns an isolated engine state so callers cannot mutate recorded +// payload bytes through a lookup result. +func (r *fakeWorkflowRecorder) GetChain(ctx context.Context, chainID string) (workflow.ChainState, error) { + r.state.mu.RLock() + defer r.state.mu.RUnlock() + state, err := r.store.GetChain(ctx, chainID) + return chainStateToWorkflow(chainStateFromWorkflow(state)), err +} + +// CreateBatch records the exact committed engine model before member delivery +// begins, then acceptance decides whether assertions may observe it. +func (r *fakeWorkflowRecorder) CreateBatch(ctx context.Context, record workflow.BatchRecord) error { + r.state.mu.Lock() + defer r.state.mu.Unlock() + if err := r.store.CreateBatch(ctx, record); err != nil { + return err + } + r.batches[record.BatchID] = batchRecordFromWorkflow(record) + return nil +} + +// MarkBatchJobStarted delegates the transition within the active fake state generation. +func (r *fakeWorkflowRecorder) MarkBatchJobStarted(ctx context.Context, batchID, jobID string) error { + r.state.mu.Lock() + defer r.state.mu.Unlock() + return r.store.MarkBatchJobStarted(ctx, batchID, jobID) +} + +// MarkBatchJobSucceeded delegates aggregate success within the active fake state generation. +func (r *fakeWorkflowRecorder) MarkBatchJobSucceeded(ctx context.Context, batchID, jobID string) (workflow.BatchState, bool, error) { + r.state.mu.Lock() + defer r.state.mu.Unlock() + return r.store.MarkBatchJobSucceeded(ctx, batchID, jobID) +} + +// MarkBatchJobFailed delegates aggregate failure within the active fake state generation. +func (r *fakeWorkflowRecorder) MarkBatchJobFailed(ctx context.Context, batchID, jobID string, cause error) (workflow.BatchState, bool, error) { + r.state.mu.Lock() + defer r.state.mu.Unlock() + return r.store.MarkBatchJobFailed(ctx, batchID, jobID, cause) +} + +// SettleBatchJob delegates atomic member settlement within the active fake state generation. +func (r *fakeWorkflowRecorder) SettleBatchJob(ctx context.Context, batchID, jobID string, outcome workflow.BatchJobOutcome, cause error) (workflow.BatchState, bool, error) { + r.state.mu.Lock() + defer r.state.mu.Unlock() + return r.store.SettleBatchJob(ctx, batchID, jobID, outcome, cause) +} + +// CancelBatch delegates aggregate cancellation within the active fake state generation. +func (r *fakeWorkflowRecorder) CancelBatch(ctx context.Context, batchID string) error { + r.state.mu.Lock() + defer r.state.mu.Unlock() + return r.store.CancelBatch(ctx, batchID) +} + +// GetBatch returns aggregate state from the active fake state generation. +func (r *fakeWorkflowRecorder) GetBatch(ctx context.Context, batchID string) (workflow.BatchState, error) { + r.state.mu.RLock() + defer r.state.mu.RUnlock() + return r.store.GetBatch(ctx, batchID) +} + +// MarkCallbackInvoked delegates idempotency claims within the active fake state generation. +func (r *fakeWorkflowRecorder) MarkCallbackInvoked(ctx context.Context, key string) (bool, error) { + r.state.mu.Lock() + defer r.state.mu.Unlock() + return r.store.MarkCallbackInvoked(ctx, key) +} + +// Prune delegates retention without deleting immutable dispatch evidence. +func (r *fakeWorkflowRecorder) Prune(ctx context.Context, before time.Time) error { + r.state.mu.Lock() + defer r.state.mu.Unlock() + return r.store.Prune(ctx, before) +} + +// fakeWorkflowDispatchContextKey distinguishes the canonical fake engine from +// callers using FakeQueue as an explicit raw-runtime transport. +type fakeWorkflowDispatchContextKey struct{} + +// withFakeWorkflowDispatch marks deliveries emitted by the fake's own engine so +// physical protocol envelopes do not pollute application dispatch assertions. +func withFakeWorkflowDispatch(ctx context.Context) context.Context { + if ctx == nil { + ctx = context.Background() + } + return context.WithValue(ctx, fakeWorkflowDispatchContextKey{}, true) +} + +// fakeWorkflowDeliverySuppressed recognizes only owned protocol types emitted +// by the canonical fake engine; direct raw-runtime calls remain observable. +func fakeWorkflowDeliverySuppressed(ctx context.Context, jobType string) bool { + if ctx == nil || !workflow.IsDeliveryType(jobType) { + return false + } + marked, _ := ctx.Value(fakeWorkflowDispatchContextKey{}).(bool) + return marked +} + +// guardFakeWorkflowDispatch keeps the engine's create, initial delivery, and +// acceptance cleanup intact while destructive state operations wait. +func (f *FakeQueue) guardFakeWorkflowDispatch() func() { + f.state.workflowOps.RLock() + return f.state.workflowOps.RUnlock +} + +// Chain creates a fake chain backed by the production workflow builder and +// records it only when Dispatch accepts its initial delivery. Fluent function +// callbacks are accepted for compatibility but are not retained in fake runtime +// state or executed. +// @group Testing +func (f *FakeQueue) Chain(jobs ...Job) ChainBuilder { + converted, err := toWorkflowJobs(jobs) + if err != nil { + return &chainBuilderAdapter{err: err} + } + return &chainBuilderAdapter{ + inner: f.state.workflow.engine.Chain(converted...), + dispatchGuard: f.guardFakeWorkflowDispatch, + dispatchContext: withFakeWorkflowDispatch, + onAccepted: f.state.workflow.acceptChain, + onRejected: f.state.workflow.rejectChain, + } +} + +// Batch creates a fake batch backed by the production workflow builder and +// records it only when Dispatch accepts all initial member deliveries. Fluent +// function callbacks are accepted for compatibility but are not retained in +// fake runtime state or executed. +// @group Testing +func (f *FakeQueue) Batch(jobs ...Job) BatchBuilder { + converted, err := toWorkflowJobs(jobs) + if err != nil { + return &batchBuilderAdapter{err: err} + } + return &batchBuilderAdapter{ + inner: f.state.workflow.engine.Batch(converted...), + dispatchGuard: f.guardFakeWorkflowDispatch, + dispatchContext: withFakeWorkflowDispatch, + onAccepted: f.state.workflow.acceptBatch, + onRejected: f.state.workflow.rejectBatch, + } +} + +// ChainRecords returns isolated creation records for accepted fake chains. +// @group Testing +// +// Example: inspect a fake chain +// +// fake := queue.NewFake() +// _, _ = fake.Chain( +// queue.NewJob("reports:build"), +// queue.NewJob("reports:publish"), +// ).OnQueue("workflow").Dispatch(context.Background()) +// record := fake.ChainRecords()[0] +// fmt.Println(len(record.Nodes), record.Queue) +// // Output: 2 workflow +func (f *FakeQueue) ChainRecords() []ChainRecord { + f.state.mu.RLock() + defer f.state.mu.RUnlock() + records := make([]ChainRecord, 0, len(f.state.workflow.acceptedChainIDs)) + for _, chainID := range f.state.workflow.acceptedChainIDs { + record, exists := f.state.workflow.chains[chainID] + if exists { + records = append(records, cloneFakeChainRecord(record)) + } + } + return records +} + +// BatchRecords returns isolated creation records for accepted fake batches. +// @group Testing +// +// Example: inspect a fake batch +// +// fake := queue.NewFake() +// _, _ = fake.Batch( +// queue.NewJob("emails:first"), +// queue.NewJob("emails:second"), +// ).Name("nightly").AllowFailures().Dispatch(context.Background()) +// record := fake.BatchRecords()[0] +// fmt.Println(record.Name, len(record.Jobs), record.AllowFailed) +// // Output: nightly 2 true +func (f *FakeQueue) BatchRecords() []BatchRecord { + f.state.mu.RLock() + defer f.state.mu.RUnlock() + records := make([]BatchRecord, 0, len(f.state.workflow.acceptedBatchIDs)) + for _, batchID := range f.state.workflow.acceptedBatchIDs { + record, exists := f.state.workflow.batches[batchID] + if exists { + records = append(records, cloneFakeBatchRecord(record)) + } + } + return records +} + +// FindChain returns workflow state created by the fake's production engine. +// @group Testing +func (f *FakeQueue) FindChain(ctx context.Context, chainID string) (ChainState, error) { + state, err := f.state.workflow.engine.FindChain(ctx, chainID) + return chainStateFromWorkflow(state), err +} + +// FindBatch returns workflow state created by the fake's production engine. +// @group Testing +func (f *FakeQueue) FindBatch(ctx context.Context, batchID string) (BatchState, error) { + state, err := f.state.workflow.engine.FindBatch(ctx, batchID) + return batchStateFromWorkflow(state), err +} + +// Prune removes terminal workflow state while retaining fake dispatch records. +// @group Testing +func (f *FakeQueue) Prune(ctx context.Context, before time.Time) error { + f.state.workflowOps.Lock() + defer f.state.workflowOps.Unlock() + return f.state.workflow.engine.Prune(ctx, before) +} + +// AssertChained fails unless an accepted chain has the expected ordered job types. +// @group Testing +// +// Example: assert a fake chain +// +// fake := queue.NewFake() +// _, _ = fake.Chain( +// queue.NewJob("reports:build"), +// queue.NewJob("reports:publish"), +// ).Dispatch(context.Background()) +// fake.AssertChained(t, []string{"reports:build", "reports:publish"}) +func (f *FakeQueue) AssertChained(t testing.TB, expected []string) { + t.Helper() + for _, record := range f.ChainRecords() { + if fakeChainTypesEqual(record, expected) { + return + } + } + t.Fatalf("expected chain %v", expected) +} + +// AssertBatchCount fails unless the accepted batch count equals expected. +// @group Testing +// +// Example: assert fake batch count +// +// fake := queue.NewFake() +// _, _ = fake.Batch(queue.NewJob("emails:send")).Dispatch(context.Background()) +// fake.AssertBatchCount(t, 1) +func (f *FakeQueue) AssertBatchCount(t testing.TB, expected int) { + t.Helper() + if got := len(f.BatchRecords()); got != expected { + t.Fatalf("expected batch count %d, got %d", expected, got) + } +} + +// AssertNothingBatched fails when any accepted batch was recorded. +// @group Testing +func (f *FakeQueue) AssertNothingBatched(t testing.TB) { + t.Helper() + if got := len(f.BatchRecords()); got != 0 { + t.Fatalf("expected no batches, got %d", got) + } +} + +// AssertBatched fails unless an accepted canonical batch matches predicate. +// The predicate runs outside the recorder lock so it may safely inspect the fake. +// @group Testing +// +// Example: assert fake batch policy +// +// fake := queue.NewFake() +// _, _ = fake.Batch(queue.NewJob("emails:send")).Name("nightly").Dispatch(context.Background()) +// fake.AssertBatched(t, func(record queue.BatchRecord) bool { return record.Name == "nightly" }) +func (f *FakeQueue) AssertBatched(t testing.TB, predicate func(BatchRecord) bool) { + t.Helper() + for _, record := range f.BatchRecords() { + if predicate(record) { + return + } + } + t.Fatalf("expected at least one batch to match predicate") +} + +// fakeChainTypesEqual compares the assertion projection without discarding the +// richer canonical record exposed to callers that need payload or policy checks. +func fakeChainTypesEqual(record ChainRecord, expected []string) bool { + if len(record.Nodes) != len(expected) { + return false + } + for i, node := range record.Nodes { + if node.Job.Type != expected[i] { + return false + } + } + return true +} + +// cloneFakeChainRecord isolates nested node payloads from caller mutation. +func cloneFakeChainRecord(record ChainRecord) ChainRecord { + return chainRecordFromWorkflow(chainRecordToWorkflow(record)) +} + +// cloneFakeBatchRecord isolates nested member payloads from caller mutation. +func cloneFakeBatchRecord(record BatchRecord) BatchRecord { + return batchRecordFromWorkflow(batchRecordToWorkflow(record)) +} diff --git a/fake_workflow_lifecycle_test.go b/fake_workflow_lifecycle_test.go new file mode 100644 index 0000000..f362a7d --- /dev/null +++ b/fake_workflow_lifecycle_test.go @@ -0,0 +1,202 @@ +package queue + +import ( + "context" + "encoding/json" + "errors" + "testing" + + "github.com/goforj/queue/internal/workflow" +) + +// TestFakeWorkflowRecorderRejectsInvalidState verifies malformed records cannot +// become visible even when the recorder is used at its storage boundary. +func TestFakeWorkflowRecorderRejectsInvalidState(t *testing.T) { + fake := NewFake() + recorder := fake.state.workflow + ctx := context.Background() + + if err := recorder.CreateChain(ctx, workflow.ChainRecord{}); err == nil { + t.Fatal("CreateChain error = nil, want invalid record rejection") + } + if err := recorder.CreateBatch(ctx, workflow.BatchRecord{}); err == nil { + t.Fatal("CreateBatch error = nil, want invalid record rejection") + } + + recorder.acceptChain("") + recorder.acceptBatch("") + if got := len(fake.ChainRecords()); got != 0 { + t.Fatalf("invalid chain records = %d, want 0", got) + } + if got := len(fake.BatchRecords()); got != 0 { + t.Fatalf("invalid batch records = %d, want 0", got) + } + fake.AssertNothingBatched(t) +} + +// TestFakeWorkflowRecorderChainLifecycle verifies retry-safe advancement, +// isolated successor data, terminal success, and compatibility failure state. +func TestFakeWorkflowRecorderChainLifecycle(t *testing.T) { + fake := NewFake() + ctx := context.Background() + chainID, err := fake.Chain( + NewJob("chain:first").Payload(json.RawMessage(`{"step":1}`)), + NewJob("chain:second").Payload(json.RawMessage(`{"step":2}`)), + ).Dispatch(ctx) + if err != nil { + t.Fatalf("dispatch chain: %v", err) + } + record := fake.ChainRecords()[0] + recorder := fake.state.workflow + + next, done, err := recorder.AdvanceChain(ctx, chainID, record.Nodes[0].NodeID) + if err != nil || done || next == nil || next.NodeID != record.Nodes[1].NodeID { + t.Fatalf("first advance = next:%+v done:%t err:%v", next, done, err) + } + next.Job.Payload[0] = 'x' + state, err := recorder.GetChain(ctx, chainID) + if err != nil || state.NextIndex != 1 || string(state.Nodes[1].Job.Payload) != `{"step":2}` { + t.Fatalf("state after successor mutation = %+v, %v", state, err) + } + + replayed, done, err := recorder.AdvanceChain(ctx, chainID, record.Nodes[0].NodeID) + if err != nil || done || replayed == nil || replayed.NodeID != record.Nodes[1].NodeID { + t.Fatalf("replayed advance = next:%+v done:%t err:%v", replayed, done, err) + } + state, err = recorder.GetChain(ctx, chainID) + if err != nil || state.NextIndex != 1 { + t.Fatalf("state after replayed advance = %+v, %v", state, err) + } + + next, done, err = recorder.AdvanceChain(ctx, chainID, record.Nodes[1].NodeID) + if err != nil || !done || next != nil { + t.Fatalf("terminal advance = next:%+v done:%t err:%v", next, done, err) + } + if err := recorder.FailChain(ctx, chainID, errors.New("late failure")); err != nil { + t.Fatalf("fail completed chain: %v", err) + } + state, err = recorder.GetChain(ctx, chainID) + if err != nil || !state.Completed || state.Failed || state.Failure != "" { + t.Fatalf("completed state after late failure = %+v, %v", state, err) + } + + failedID, err := fake.Chain(NewJob("chain:failed")).Dispatch(ctx) + if err != nil { + t.Fatalf("dispatch failing chain: %v", err) + } + cause := errors.New("handler failed") + if err := recorder.FailChain(ctx, failedID, cause); err != nil { + t.Fatalf("fail active chain: %v", err) + } + failed, err := recorder.GetChain(ctx, failedID) + if err != nil || !failed.Failed || failed.Completed || failed.Failure != cause.Error() { + t.Fatalf("failed chain state = %+v, %v", failed, err) + } + + if next, done, err := recorder.AdvanceChain(ctx, "missing-chain", "missing-node"); !errors.Is(err, ErrWorkflowNotFound) || done || next != nil { + t.Fatalf("missing chain advance = next:%+v done:%t err:%v", next, done, err) + } + if err := recorder.FailChain(ctx, "missing-chain", cause); !errors.Is(err, ErrWorkflowNotFound) { + t.Fatalf("missing chain failure error = %v, want ErrWorkflowNotFound", err) + } +} + +// TestFakeWorkflowRecorderBatchLifecycle verifies started markers, idempotent +// aggregate counters, allowed failures, and first-writer outcome ownership. +func TestFakeWorkflowRecorderBatchLifecycle(t *testing.T) { + fake := NewFake() + ctx := context.Background() + batchID, err := fake.Batch( + NewJob("batch:first"), + NewJob("batch:second"), + NewJob("batch:third"), + ).AllowFailures().Dispatch(ctx) + if err != nil { + t.Fatalf("dispatch batch: %v", err) + } + record := fake.BatchRecords()[0] + recorder := fake.state.workflow + + if err := recorder.MarkBatchJobStarted(ctx, batchID, record.Jobs[0].JobID); err != nil { + t.Fatalf("mark first member started: %v", err) + } + if err := recorder.MarkBatchJobStarted(ctx, batchID, "missing-job"); !errors.Is(err, ErrWorkflowNotFound) { + t.Fatalf("missing member start error = %v, want ErrWorkflowNotFound", err) + } + + state, done, err := recorder.MarkBatchJobSucceeded(ctx, batchID, record.Jobs[0].JobID) + if err != nil || done || state.Processed != 1 || state.Pending != 2 || state.Failed != 0 { + t.Fatalf("first success = state:%+v done:%t err:%v", state, done, err) + } + replayed, done, err := recorder.MarkBatchJobSucceeded(ctx, batchID, record.Jobs[0].JobID) + if err != nil || done || replayed.Processed != 1 || replayed.Pending != 2 || replayed.Failed != 0 { + t.Fatalf("replayed success = state:%+v done:%t err:%v", replayed, done, err) + } + + cause := errors.New("member failed") + state, done, err = recorder.MarkBatchJobFailed(ctx, batchID, record.Jobs[1].JobID, cause) + if err != nil || done || state.Processed != 2 || state.Pending != 1 || state.Failed != 1 || state.Cancelled { + t.Fatalf("allowed failure = state:%+v done:%t err:%v", state, done, err) + } + + state, owned, err := recorder.SettleBatchJob(ctx, batchID, record.Jobs[2].JobID, workflow.BatchJobSucceeded, nil) + if err != nil || !owned || !state.Completed || state.Cancelled || state.Processed != 3 || state.Pending != 0 || state.Failed != 1 { + t.Fatalf("terminal settlement = state:%+v owned:%t err:%v", state, owned, err) + } + state, owned, err = recorder.SettleBatchJob(ctx, batchID, record.Jobs[2].JobID, workflow.BatchJobFailed, cause) + if err != nil || owned || !state.Completed || state.Processed != 3 || state.Pending != 0 || state.Failed != 1 { + t.Fatalf("contradictory settlement = state:%+v owned:%t err:%v", state, owned, err) + } + if _, _, err := recorder.SettleBatchJob(ctx, batchID, record.Jobs[2].JobID, workflow.BatchJobOutcome("unknown"), nil); err == nil { + t.Fatal("unknown batch outcome error = nil") + } + + claimed, err := recorder.MarkCallbackInvoked(ctx, "batch:finally:"+batchID) + if err != nil || !claimed { + t.Fatalf("first callback claim = %t, %v", claimed, err) + } + claimed, err = recorder.MarkCallbackInvoked(ctx, "batch:finally:"+batchID) + if err != nil || claimed { + t.Fatalf("replayed callback claim = %t, %v", claimed, err) + } +} + +// TestFakeWorkflowAssertionsSearchAcceptedRecords verifies assertion helpers +// search all immutable records instead of requiring the first record to match. +func TestFakeWorkflowAssertionsSearchAcceptedRecords(t *testing.T) { + fake := NewFake() + ctx := context.Background() + if _, err := fake.Chain(NewJob("chain:unrelated")).Dispatch(ctx); err != nil { + t.Fatalf("dispatch unrelated chain: %v", err) + } + if _, err := fake.Chain(NewJob("chain:first"), NewJob("chain:second")).Dispatch(ctx); err != nil { + t.Fatalf("dispatch expected chain: %v", err) + } + if _, err := fake.Batch(NewJob("batch:unrelated")).Name("unrelated").Dispatch(ctx); err != nil { + t.Fatalf("dispatch unrelated batch: %v", err) + } + if _, err := fake.Batch(NewJob("batch:expected")).Name("expected").Dispatch(ctx); err != nil { + t.Fatalf("dispatch expected batch: %v", err) + } + + fake.AssertChained(t, []string{"chain:first", "chain:second"}) + fake.AssertBatchCount(t, 2) + fake.AssertBatched(t, func(record BatchRecord) bool { + return record.Name == "expected" + }) + if fakeChainTypesEqual(fake.ChainRecords()[1], []string{"chain:wrong", "chain:second"}) { + t.Fatal("chain type comparison accepted a mismatched member") + } +} + +// TestWithFakeWorkflowDispatchNormalizesNilContext verifies internally emitted +// workflow envelopes remain suppressed when callers omit a context. +func TestWithFakeWorkflowDispatchNormalizesNilContext(t *testing.T) { + ctx := withFakeWorkflowDispatch(nil) + if ctx == nil { + t.Fatal("withFakeWorkflowDispatch(nil) returned nil") + } + if !fakeWorkflowDeliverySuppressed(ctx, workflow.ChainNodeDeliveryType) { + t.Fatal("marked chain delivery was not suppressed") + } +} diff --git a/fake_workflow_test.go b/fake_workflow_test.go new file mode 100644 index 0000000..b42e3b4 --- /dev/null +++ b/fake_workflow_test.go @@ -0,0 +1,512 @@ +package queue + +import ( + "context" + "encoding/json" + "errors" + "sync" + "testing" + "time" + + "github.com/goforj/queue/busruntime" +) + +// fakeWorkflowFailingPayload exercises deferred Job build failures. +type fakeWorkflowFailingPayload struct{} + +// MarshalJSON forces deferred payload validation to fail at the canonical boundary. +func (fakeWorkflowFailingPayload) MarshalJSON() ([]byte, error) { + return nil, errors.New("fake payload failure") +} + +// fakeWorkflowBlockingContext pauses the initial fake delivery after workflow +// state creation so Reset can contend with the complete dispatch operation. +type fakeWorkflowBlockingContext struct { + context.Context + entered chan struct{} + release chan struct{} + once sync.Once +} + +// Err exposes cancellation only after the test releases the initial delivery. +func (c *fakeWorkflowBlockingContext) Err() error { + c.once.Do(func() { close(c.entered) }) + <-c.release + return context.Canceled +} + +// fakeWorkflowCancelAfterContext rejects delivery only after the configured +// number of batch members have passed their acceptance check. +type fakeWorkflowCancelAfterContext struct { + context.Context + accepted int + mu sync.Mutex + checks int +} + +// Err lets the first accepted checks proceed before exposing cancellation. +func (c *fakeWorkflowCancelAfterContext) Err() error { + c.mu.Lock() + defer c.mu.Unlock() + c.checks++ + if c.checks > c.accepted { + return context.Canceled + } + return nil +} + +// fakeWorkflowDispatchResult carries the blocked builder result across the test goroutine. +type fakeWorkflowDispatchResult struct { + id string + err error +} + +// TestFakeQueueWorkflowRecordsUseCanonicalEngine verifies fake assertions expose +// the same stored options, queue precedence, identifiers, and payload encoding as production. +func TestFakeQueueWorkflowRecordsUseCanonicalEngine(t *testing.T) { + fake := NewFake() + chainBuilder := fake.Chain( + NewJob("reports:build").Payload(nil).Delay(time.Second).Timeout(2*time.Second).Retry(0).Backoff(3*time.Second).UniqueFor(4*time.Second), + NewJob("reports:publish").Payload(json.RawMessage(`{"id":2}`)).OnQueue("dedicated"), + ).OnQueue("workflow") + batchBuilder := fake.Batch( + NewJob("emails:first").Payload(map[string]int{"id": 1}), + NewJob("emails:second").Payload(map[string]int{"id": 2}).OnQueue("priority"), + ).Name("nightly").OnQueue("bulk").AllowFailures() + + if got := len(fake.ChainRecords()); got != 0 { + t.Fatalf("chain records before Dispatch = %d, want 0", got) + } + if got := len(fake.BatchRecords()); got != 0 { + t.Fatalf("batch records before Dispatch = %d, want 0", got) + } + + chainID, err := chainBuilder.Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch chain: %v", err) + } + batchID, err := batchBuilder.Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch batch: %v", err) + } + if chainID == "" || batchID == "" || chainID == batchID { + t.Fatalf("workflow IDs = %q/%q, want distinct nonempty values", chainID, batchID) + } + if got := len(fake.Records()); got != 0 { + t.Fatalf("physical workflow envelopes leaked into direct records: %d", got) + } + + chains := fake.ChainRecords() + if len(chains) != 1 { + t.Fatalf("chain records = %d, want 1", len(chains)) + } + chain := chains[0] + if chain.ChainID != chainID || chain.DispatchID == "" || chain.Queue != "workflow" { + t.Fatalf("chain identity = %+v", chain) + } + if len(chain.Nodes) != 2 { + t.Fatalf("chain nodes = %d, want 2", len(chain.Nodes)) + } + if got := string(chain.Nodes[0].Job.Payload); got != "null" { + t.Fatalf("nil workflow payload = %q, want null", got) + } + if chain.Nodes[0].Job.Options.Queue != "workflow" || chain.Nodes[0].Job.Options.Retry != 0 { + t.Fatalf("defaulted first node = %+v", chain.Nodes[0].Job) + } + if options := chain.Nodes[0].Job.Options; options.Delay != time.Second || options.Timeout != 2*time.Second || options.Backoff != 3*time.Second || options.UniqueFor != 4*time.Second { + t.Fatalf("first node delivery policy = %+v", options) + } + if chain.Nodes[1].Job.Options.Queue != "dedicated" || string(chain.Nodes[1].Job.Payload) != `{"id":2}` { + t.Fatalf("explicit second node = %+v", chain.Nodes[1].Job) + } + + batches := fake.BatchRecords() + if len(batches) != 1 { + t.Fatalf("batch records = %d, want 1", len(batches)) + } + batch := batches[0] + if batch.BatchID != batchID || batch.DispatchID == "" || batch.Name != "nightly" || batch.Queue != "bulk" || !batch.AllowFailed { + t.Fatalf("batch identity/options = %+v", batch) + } + if len(batch.Jobs) != 2 || batch.Jobs[0].Job.Options.Queue != "bulk" || batch.Jobs[1].Job.Options.Queue != "priority" { + t.Fatalf("batch queue precedence = %+v", batch.Jobs) + } + + chainState, err := fake.FindChain(context.Background(), chainID) + if err != nil || chainState.DispatchID != chain.DispatchID || len(chainState.Nodes) != 2 { + t.Fatalf("find chain = %+v, %v", chainState, err) + } + batchState, err := fake.FindBatch(context.Background(), batchID) + if err != nil || batchState.DispatchID != batch.DispatchID || batchState.Total != 2 || batchState.Pending != 2 { + t.Fatalf("find batch = %+v, %v", batchState, err) + } + fake.AssertChained(t, []string{"reports:build", "reports:publish"}) + fake.AssertBatchCount(t, 1) + fake.AssertBatched(t, func(record BatchRecord) bool { + fake.Reset() + return record.Name == "nightly" && record.AllowFailed + }) +} + +// TestFakeQueueWorkflowRecordsOnlyAcceptedDispatches verifies abandoned and +// rejected builders cannot satisfy chain or batch assertions. +func TestFakeQueueWorkflowRecordsOnlyAcceptedDispatches(t *testing.T) { + fake := NewFake() + _ = fake.Chain(NewJob("abandoned:chain")) + _ = fake.Batch(NewJob("abandoned:batch")) + + tests := []struct { + name string + dispatch func(context.Context) (string, error) + }{ + {name: "empty chain", dispatch: fake.Chain().Dispatch}, + {name: "empty batch", dispatch: fake.Batch().Dispatch}, + {name: "invalid chain option", dispatch: fake.Chain(NewJob("bad:chain").Retry(-1)).Dispatch}, + {name: "invalid batch option", dispatch: fake.Batch(NewJob("bad:batch").Timeout(-1)).Dispatch}, + {name: "malformed chain payload", dispatch: fake.Chain(NewJob("bad:chain-json").Payload(json.RawMessage(`{`))).Dispatch}, + {name: "malformed batch payload", dispatch: fake.Batch(NewJob("bad:batch-json").Payload(json.RawMessage(`{`))).Dispatch}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if _, err := test.dispatch(context.Background()); err == nil { + t.Fatal("Dispatch error = nil, want validation failure") + } + }) + } + + canceled, cancel := context.WithCancel(context.Background()) + cancel() + rejectedChainID, err := fake.Chain(NewJob("cancelled:chain")).Dispatch(canceled) + if !errors.Is(err, context.Canceled) { + t.Fatalf("canceled chain error = %v, want context.Canceled", err) + } + rejectedBatchID, err := fake.Batch(NewJob("cancelled:batch")).Dispatch(canceled) + if !errors.Is(err, context.Canceled) { + t.Fatalf("canceled batch error = %v, want context.Canceled", err) + } + if _, err := fake.FindChain(context.Background(), rejectedChainID); !errors.Is(err, ErrWorkflowNotFound) { + t.Fatalf("rejected chain state error = %v, want ErrWorkflowNotFound", err) + } + if _, err := fake.FindBatch(context.Background(), rejectedBatchID); !errors.Is(err, ErrWorkflowNotFound) { + t.Fatalf("rejected batch state error = %v, want ErrWorkflowNotFound", err) + } + + if got := len(fake.ChainRecords()); got != 0 { + t.Fatalf("rejected chain records = %d, want 0", got) + } + if got := len(fake.BatchRecords()); got != 0 { + t.Fatalf("rejected batch records = %d, want 0", got) + } +} + +// TestFakeQueueBatchRejectsPartialInitialFanout verifies accepting an earlier +// member cannot publish a batch when a later initial delivery is canceled. +func TestFakeQueueBatchRejectsPartialInitialFanout(t *testing.T) { + fake := NewFake() + ctx := &fakeWorkflowCancelAfterContext{ + Context: context.Background(), + accepted: 1, + } + batchID, err := fake.Batch( + NewJob("batch:first"), + NewJob("batch:second"), + ).Dispatch(ctx) + if !errors.Is(err, context.Canceled) || busruntime.IsUncommitted(err) { + t.Fatalf("partial batch error = %v, want committed context cancellation", err) + } + if batchID == "" { + t.Fatal("partial batch returned an empty lookup ID") + } + if got := len(fake.BatchRecords()); got != 0 { + t.Fatalf("partial batch records = %d, want 0", got) + } + if got := len(fake.Records()); got != 0 { + t.Fatalf("partial batch leaked %d protocol deliveries", got) + } + if _, err := fake.FindBatch(context.Background(), batchID); !errors.Is(err, ErrWorkflowNotFound) { + t.Fatalf("partial batch lookup error = %v, want ErrWorkflowNotFound", err) + } +} + +// TestFakeQueueWorkflowBuilderRedispatchRecordsEachAcceptance verifies reusable +// builders retain production behavior instead of recording only construction. +func TestFakeQueueWorkflowBuilderRedispatchRecordsEachAcceptance(t *testing.T) { + fake := NewFake() + chain := fake.Chain(NewJob("chain:repeat")) + firstChainID, err := chain.Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch first chain: %v", err) + } + secondChainID, err := chain.Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch second chain: %v", err) + } + batch := fake.Batch(NewJob("batch:repeat")) + firstBatchID, err := batch.Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch first batch: %v", err) + } + secondBatchID, err := batch.Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch second batch: %v", err) + } + if firstChainID == secondChainID || firstBatchID == secondBatchID { + t.Fatalf("reused builder IDs collided: chains=%q/%q batches=%q/%q", firstChainID, secondChainID, firstBatchID, secondBatchID) + } + if len(fake.ChainRecords()) != 2 || len(fake.BatchRecords()) != 2 { + t.Fatalf("reused builder records = chains:%d batches:%d", len(fake.ChainRecords()), len(fake.BatchRecords())) + } +} + +// TestFakeQueueDispatchValidationMetadataAndIsolation verifies direct records +// reject deferred errors, preserve direct correlation, and own mutable options. +func TestFakeQueueDispatchValidationMetadataAndIsolation(t *testing.T) { + fake := NewFake() + if err := fake.Dispatch(NewJob("invalid:retry").Retry(-1)); err == nil { + t.Fatal("negative retry dispatch error = nil") + } + if err := fake.Dispatch(NewJob("invalid:payload").Payload(fakeWorkflowFailingPayload{})); err == nil { + t.Fatal("failing payload dispatch error = nil") + } + fake.AssertNothingDispatched(t) + canceled, cancel := context.WithCancel(context.Background()) + cancel() + if err := fake.WithContext(canceled).Dispatch(NewJob("invalid:before-context").Retry(-1)); err == nil || errors.Is(err, context.Canceled) { + t.Fatalf("invalid canceled dispatch error = %v, want validation precedence", err) + } + + job := NewJob("reports:build").Payload([]byte(`{"id":1}`)).Timeout(time.Second).Retry(2).Backoff(time.Millisecond) + if err := fake.Dispatch(job); err != nil { + t.Fatalf("dispatch job: %v", err) + } + originalOptions := DriverOptions(job) + *originalOptions.Timeout = 9 * time.Second + *originalOptions.MaxRetry = 9 + *originalOptions.Backoff = 9 * time.Second + recordedOptions := DriverOptions(fake.Records()[0].Job) + if *recordedOptions.Timeout != time.Second || *recordedOptions.MaxRetry != 2 || *recordedOptions.Backoff != time.Millisecond { + t.Fatalf("input mutation changed record: %+v", recordedOptions) + } + *recordedOptions.Timeout = 7 * time.Second + if got := *DriverOptions(fake.Records()[0].Job).Timeout; got != time.Second { + t.Fatalf("returned record mutation changed stored timeout: %v", got) + } + + metadata := busruntime.DeliveryMetadata{ + SchemaVersion: busruntime.DeliveryMetadataVersion, + DispatchID: "dsp_fake", + JobID: "job_fake", + ChainID: "chn_fake", + BatchID: "bat_fake", + Queue: "critical", + } + payload := []byte{0x00, 0xff, 0x01} + if err := fake.BusDispatchDirect(context.Background(), "binary:job", payload, metadata, busruntime.JobOptions{Queue: "critical", Retry: 0}); err != nil { + t.Fatalf("direct bus dispatch: %v", err) + } + payload[0] = 0x7f + direct := fake.Records()[1].Job + if got := direct.PayloadBytes(); len(got) != 3 || got[0] != 0x00 || got[1] != 0xff { + t.Fatalf("direct payload = %v", got) + } + if got := DriverMetadata(direct); got != metadata { + t.Fatalf("direct metadata = %+v, want %+v", got, metadata) + } + if retry := DriverOptions(direct).MaxRetry; retry == nil || *retry != 0 { + t.Fatalf("direct explicit retry = %v, want pointer to zero", retry) + } +} + +// TestFakeQueueWorkflowSnapshotsAndReset verifies returned nested records are +// isolated and Reset clears every canonical projection and lookup. +func TestFakeQueueWorkflowSnapshotsAndReset(t *testing.T) { + fake := NewFake() + if err := fake.Dispatch(NewJob("direct:job")); err != nil { + t.Fatalf("dispatch direct job: %v", err) + } + chainID, err := fake.Chain(NewJob("chain:job").Payload(json.RawMessage(`{"value":1}`))).Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch chain: %v", err) + } + batchID, err := fake.Batch(NewJob("batch:job").Payload(json.RawMessage(`{"value":2}`))).Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch batch: %v", err) + } + + chains := fake.ChainRecords() + batches := fake.BatchRecords() + chains[0].Nodes[0].Job.Payload[0] = 'x' + batches[0].Jobs[0].Job.Payload[0] = 'x' + if got := string(fake.ChainRecords()[0].Nodes[0].Job.Payload); got != `{"value":1}` { + t.Fatalf("mutated chain snapshot changed stored payload: %q", got) + } + if got := string(fake.BatchRecords()[0].Jobs[0].Job.Payload); got != `{"value":2}` { + t.Fatalf("mutated batch snapshot changed stored payload: %q", got) + } + chainState, err := fake.FindChain(context.Background(), chainID) + if err != nil { + t.Fatalf("find chain for isolation: %v", err) + } + chainState.Nodes[0].Job.Payload[0] = 'x' + chainStateAgain, err := fake.FindChain(context.Background(), chainID) + if err != nil || string(chainStateAgain.Nodes[0].Job.Payload) != `{"value":1}` { + t.Fatalf("mutated lookup changed stored chain: %+v, %v", chainStateAgain, err) + } + + fake.Reset() + if len(fake.Records()) != 0 || len(fake.ChainRecords()) != 0 || len(fake.BatchRecords()) != 0 { + t.Fatalf("Reset retained records: direct=%d chain=%d batch=%d", len(fake.Records()), len(fake.ChainRecords()), len(fake.BatchRecords())) + } + if _, err := fake.FindChain(context.Background(), chainID); !errors.Is(err, ErrWorkflowNotFound) { + t.Fatalf("FindChain after Reset error = %v, want ErrWorkflowNotFound", err) + } + if _, err := fake.FindBatch(context.Background(), batchID); !errors.Is(err, ErrWorkflowNotFound) { + t.Fatalf("FindBatch after Reset error = %v, want ErrWorkflowNotFound", err) + } +} + +// TestFakeQueueResetWaitsForWorkflowDispatch verifies Reset cannot replace the +// store between engine creation and rejection cleanup for chains or batches. +func TestFakeQueueResetWaitsForWorkflowDispatch(t *testing.T) { + tests := []struct { + name string + dispatch func(*FakeQueue, context.Context) (string, error) + find func(*FakeQueue, string) error + }{ + { + name: "chain", + dispatch: func(fake *FakeQueue, ctx context.Context) (string, error) { + return fake.Chain(NewJob("chain:blocked")).Dispatch(ctx) + }, + find: func(fake *FakeQueue, workflowID string) error { + _, err := fake.FindChain(context.Background(), workflowID) + return err + }, + }, + { + name: "batch", + dispatch: func(fake *FakeQueue, ctx context.Context) (string, error) { + return fake.Batch(NewJob("batch:blocked")).Dispatch(ctx) + }, + find: func(fake *FakeQueue, workflowID string) error { + _, err := fake.FindBatch(context.Background(), workflowID) + return err + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fake := NewFake() + blocked := &fakeWorkflowBlockingContext{ + Context: context.Background(), + entered: make(chan struct{}), + release: make(chan struct{}), + } + dispatchDone := make(chan fakeWorkflowDispatchResult, 1) + go func() { + workflowID, err := test.dispatch(fake, blocked) + dispatchDone <- fakeWorkflowDispatchResult{id: workflowID, err: err} + }() + + select { + case <-blocked.entered: + case <-time.After(2 * time.Second): + t.Fatal("initial fake delivery did not block") + } + if fake.state.workflowOps.TryLock() { + fake.state.workflowOps.Unlock() + close(blocked.release) + <-dispatchDone + t.Fatal("workflow dispatch did not hold the Reset operation guard") + } + + resetStarted := make(chan struct{}) + resetDone := make(chan struct{}) + go func() { + close(resetStarted) + fake.Reset() + close(resetDone) + }() + <-resetStarted + close(blocked.release) + + var result fakeWorkflowDispatchResult + select { + case result = <-dispatchDone: + case <-time.After(2 * time.Second): + t.Fatal("blocked fake dispatch did not return") + } + if !errors.Is(result.err, context.Canceled) { + t.Fatalf("dispatch error = %v, want context.Canceled", result.err) + } + select { + case <-resetDone: + case <-time.After(2 * time.Second): + t.Fatal("Reset did not finish after workflow dispatch returned") + } + if err := test.find(fake, result.id); !errors.Is(err, ErrWorkflowNotFound) { + t.Fatalf("workflow lookup after Reset error = %v, want ErrWorkflowNotFound", err) + } + }) + } +} + +// TestFakeQueueConcurrentViews exercises shared recording, workflow dispatch, +// snapshots, and reset under the race detector. +func TestFakeQueueConcurrentViews(t *testing.T) { + fake := NewFake() + var wait sync.WaitGroup + errorsSeen := make(chan error, 12*40) + for worker := 0; worker < 12; worker++ { + worker := worker + wait.Add(1) + go func() { + defer wait.Done() + for iteration := 0; iteration < 40; iteration++ { + switch worker % 5 { + case 0: + errorsSeen <- fake.WithContext(context.Background()).Dispatch(NewJob("direct:concurrent")) + case 1: + chainID, err := fake.Chain(NewJob("chain:concurrent")).Dispatch(context.Background()) + errorsSeen <- err + _, _ = fake.FindChain(context.Background(), chainID) + case 2: + batchID, err := fake.Batch(NewJob("batch:concurrent")).Dispatch(context.Background()) + errorsSeen <- err + _, _ = fake.FindBatch(context.Background(), batchID) + case 3: + canceled, cancel := context.WithCancel(context.Background()) + cancel() + _, err := fake.Chain(NewJob("chain:rejected-concurrent")).Dispatch(canceled) + if errors.Is(err, context.Canceled) && !busruntime.IsUncommitted(err) { + err = nil + } + errorsSeen <- err + case 4: + canceled, cancel := context.WithCancel(context.Background()) + cancel() + _, err := fake.Batch(NewJob("batch:rejected-concurrent")).Dispatch(canceled) + if errors.Is(err, context.Canceled) && !busruntime.IsUncommitted(err) { + err = nil + } + errorsSeen <- err + } + _ = fake.Records() + _ = fake.ChainRecords() + _ = fake.BatchRecords() + _ = fake.Prune(context.Background(), time.Now()) + if iteration%19 == 0 { + fake.Reset() + } + } + }() + } + wait.Wait() + close(errorsSeen) + for err := range errorsSeen { + if err != nil { + t.Errorf("concurrent fake dispatch: %v", err) + } + } +} diff --git a/go.work b/go.work index df50730..199d52e 100644 --- a/go.work +++ b/go.work @@ -1,7 +1,8 @@ -go 1.24.4 +go 1.25.0 use ( . + ./docs ./driver/natsqueue ./driver/rabbitmqqueue ./driver/redisqueue diff --git a/integration/README.md b/integration/README.md index a93d786..30a483c 100644 --- a/integration/README.md +++ b/integration/README.md @@ -44,6 +44,15 @@ Local-only compile/smoke checks (no Docker): INTEGRATION_BACKEND=null,sync,workerpool GOCACHE=/tmp/queue-gocache go test -tags=integration ./integration/... -run '^$' ``` +Coverage uses the repository collector so root and driver code exercised by this +module is included. CI runs these backend profiles in parallel; `all` is the slow +local equivalent and may start every external service container: + +```bash +scripts/coverage-codecov.sh unit +INTEGRATION_BACKEND=all scripts/coverage-codecov.sh integration +``` + ## Environment - `INTEGRATION_BACKEND=` filters enabled backends (comma-separated). diff --git a/integration/all/integration_scenarios_test.go b/integration/all/integration_scenarios_test.go index 47a3203..2283b08 100644 --- a/integration/all/integration_scenarios_test.go +++ b/integration/all/integration_scenarios_test.go @@ -7,9 +7,11 @@ import ( "database/sql" "errors" "fmt" + "io" "math/rand" "net" "os" + "strconv" "strings" "sync" "sync/atomic" @@ -61,7 +63,10 @@ var integrationRabbitMQ struct { url string } -const redisDefaultJobTimeout = 30 * time.Second +const ( + redisApplicationMaxRetryHeader = "goforj-queue-application-max-retry" + redisDefaultJobTimeout = 30 * time.Second +) type runtimeWorkerBackend interface { Register(jobType string, handler Handler) @@ -231,6 +236,205 @@ func integrationBackendEnabled(name string) bool { return testenv.BackendEnabled(os.Getenv("INTEGRATION_BACKEND"), name) } +// stopRedisBrokerForScenario stops Redis and returns an idempotent recovery function while registering failure-safe cleanup first. +func stopRedisBrokerForScenario(t *testing.T) func() { + t.Helper() + recoveryNeeded := true + recoverBroker := func() error { + if !recoveryNeeded { + return nil + } + if !integrationRedis.container.IsRunning() { + if err := integrationRedis.container.Start(context.Background()); err != nil { + return fmt.Errorf("start Redis broker: %w", err) + } + } + if err := refreshRedisAddr(context.Background()); err != nil { + return fmt.Errorf("refresh Redis address: %w", err) + } + if err := waitForTCP(integrationRedis.addr, 10*time.Second); err != nil { + return fmt.Errorf("wait for Redis broker: %w", err) + } + recoveryNeeded = false + return nil + } + t.Cleanup(func() { + if err := recoverBroker(); err != nil { + t.Errorf("restore Redis broker after scenario: %v", err) + } + }) + + stopTimeout := 10 * time.Second + if err := integrationRedis.container.Stop(context.Background(), &stopTimeout); err != nil { + t.Fatalf("stop Redis broker: %v", err) + } + return func() { + t.Helper() + requireScenarioNoErr(t, "restore_redis_broker", recoverBroker()) + } +} + +// TestIntegrationChaos_RedisBrokerDisconnectRedelivery proves a committed side effect survives lost acknowledgement without consuming the application's retry budget. +func TestIntegrationChaos_RedisBrokerDisconnectRedelivery(t *testing.T) { + if os.Getenv("RUN_CHAOS") != "1" { + t.Skip("set RUN_CHAOS=1 to enable broker-disconnect recovery") + } + if !integrationBackendEnabled(testenv.BackendRedis) { + t.Skip("Redis integration backend not selected") + } + + start := time.Now() + queueName := uniqueQueueName("scenario-redis-lost-ack") + jobType := "job:scenario:redis-lost-ack" + cfg := withDefaultQueue(redisCfg(integrationRedis.addr), queueName) + q, err := newQueueRuntime(cfg) + if err != nil { + t.Fatalf("new Redis chaos queue: %v", err) + } + t.Cleanup(func() { + if shutdownErr := q.Shutdown(context.Background()); shutdownErr != nil { + t.Errorf("shutdown Redis chaos queue: %v", shutdownErr) + } + }) + + w := newQueueBackedWorker(t, cfg, 1) + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + if shutdownErr := w.Shutdown(shutdownCtx); shutdownErr != nil { + t.Errorf("shutdown Redis chaos worker: %v", shutdownErr) + } + }) + + var attempts atomic.Int32 + var committed atomic.Int32 + firstEntered := make(chan struct{}, 1) + firstReturned := make(chan struct{}, 1) + redelivered := make(chan struct{}, 1) + handlerFailure := make(chan error, 1) + releaseFirst := make(chan struct{}) + var releaseOnce sync.Once + release := func() { + releaseOnce.Do(func() { close(releaseFirst) }) + } + t.Cleanup(release) + + handler := func(_ context.Context, job Job) error { + var payload scenarioPayload + if err := job.Bind(&payload); err != nil { + handlerFailure <- fmt.Errorf("bind lost-ack payload: %w", err) + return err + } + if payload.ID != 9702 { + err := fmt.Errorf("lost-ack payload ID = %d, want 9702", payload.ID) + handlerFailure <- err + return err + } + invocation := attempts.Add(1) + // The idempotency key prevents the recovered physical delivery from repeating the external effect. + committed.CompareAndSwap(0, 1) + if invocation == 1 { + firstEntered <- struct{}{} + <-releaseFirst + firstReturned <- struct{}{} + return nil + } + redelivered <- struct{}{} + return nil + } + w.Register(jobType, handler) + requireScenarioNoErr(t, "lost_ack_worker_start", w.StartWorkers(context.Background())) + + inspector := newRedisInspector(t) + job := NewJob(jobType). + Payload(scenarioPayload{ID: 9702, Name: "lost-ack"}). + OnQueue(queueName). + Retry(0). + Timeout(2 * time.Minute) + requireScenarioNoErr(t, "lost_ack_dispatch", q.Dispatch(job)) + + select { + case <-firstEntered: + case handlerErr := <-handlerFailure: + t.Fatalf("[lost_ack_handler_entered] handler rejected initial delivery: %v", handlerErr) + case <-time.After(10 * time.Second): + t.Fatal("[lost_ack_handler_entered] handler did not start") + } + activeBeforeFault := waitForRedisActiveJob(t, inspector, queueName, jobType, 5*time.Second) + + restartBroker := stopRedisBrokerForScenario(t) + release() + select { + case <-firstReturned: + case <-time.After(5 * time.Second): + t.Fatal("[lost_ack_handler_returned] handler did not return while Redis was unavailable") + } + shutdownCtx, cancelShutdown := context.WithTimeout(context.Background(), 45*time.Second) + requireScenarioNoErr(t, "lost_ack_original_worker_shutdown", w.Shutdown(shutdownCtx)) + cancelShutdown() + restartBroker() + + recoveryInspector := newRedisInspector(t) + activeAfterFault := waitForRedisActiveJob(t, recoveryInspector, queueName, jobType, 5*time.Second) + if activeAfterFault.ID != activeBeforeFault.ID { + t.Fatalf("[lost_ack_active_identity] active task ID = %q, want %q", activeAfterFault.ID, activeBeforeFault.ID) + } + leaseCtx, cancelLease := context.WithTimeout(context.Background(), 5*time.Second) + requireScenarioNoErr(t, "lost_ack_expire_lease", expireRedisLeaseForRecovery(leaseCtx, queueName, activeBeforeFault.ID)) + cancelLease() + orphaned := waitForRedisActiveJob(t, recoveryInspector, queueName, jobType, 5*time.Second) + if !orphaned.IsOrphaned { + t.Fatal("[lost_ack_orphaned] aged active task was not reported as orphaned") + } + + recoveryCfg := withDefaultQueue(redisCfg(integrationRedis.addr), queueName) + recoveryWorker := newQueueBackedWorker(t, recoveryCfg, 1) + recoveryWorker.Register(jobType, handler) + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + if shutdownErr := recoveryWorker.Shutdown(shutdownCtx); shutdownErr != nil { + t.Errorf("shutdown Redis recovery worker: %v", shutdownErr) + } + }) + requireScenarioNoErr(t, "lost_ack_recovery_worker_start", recoveryWorker.StartWorkers(context.Background())) + + retryTask := waitForRedisRetryJob(t, recoveryInspector, queueName, activeBeforeFault.ID, 5*time.Second) + if retryTask.Retried != 0 { + t.Fatalf("[lost_ack_retry_budget] transport recovery consumed %d application retries, want 0", retryTask.Retried) + } + if retryTask.MaxRetry != 1 { + t.Fatalf("[lost_ack_transport_reserve] transport max retry = %d, want one reserved recovery slot", retryTask.MaxRetry) + } + if !strings.Contains(retryTask.LastErr, asynq.ErrLeaseExpired.Error()) { + t.Fatalf("[lost_ack_recovery_cause] last error = %q, want %q", retryTask.LastErr, asynq.ErrLeaseExpired.Error()) + } + + select { + case <-redelivered: + case <-time.After(15 * time.Second): + t.Fatal("[lost_ack_redelivered] recovered task was not delivered") + } + + settleDeadline := time.Now().Add(10 * time.Second) + settled := false + for time.Now().Before(settleDeadline) { + _, err := recoveryInspector.GetTaskInfo(queueName, activeBeforeFault.ID) + if errors.Is(err, asynq.ErrTaskNotFound) { + settled = true + break + } + if err != nil { + t.Fatalf("[lost_ack_settlement] inspect recovered task: %v", err) + } + time.Sleep(20 * time.Millisecond) + } + requireScenarioTrue(t, "lost_ack_settled", settled, "recovered task remained unsettled") + requireScenarioTrue(t, "lost_ack_attempts", attempts.Load() == 2, "attempts=%d expected=2", attempts.Load()) + requireScenarioTrue(t, "lost_ack_side_effect_once", committed.Load() == 1, "committed=%d expected=1", committed.Load()) + reportScenarioDuration(t, testenv.BackendRedis, "scenario_broker_disconnect_during_handler_redelivery", time.Since(start)) +} + func TestRedisIntegration_DispatchSmoke(t *testing.T) { if !integrationBackendEnabled(testenv.BackendRedis) { t.Skip("redis integration backend not selected") @@ -261,20 +465,48 @@ func TestRedisIntegration_DispatchMapsOptions(t *testing.T) { if !integrationBackendEnabled(testenv.BackendRedis) { t.Skip("redis integration backend not selected") } + const ( + jobType = "job:options" + deliveryJobType = "job:options:delivery" + ) + queueName := uniqueQueueName("redis-options") + attempts := make(chan Event, 1) + observer := ObserverFunc(func(_ context.Context, event Event) { + if event.Kind == EventProcessStarted && event.JobType == deliveryJobType { + select { + case attempts <- event: + default: + } + } + }) inspector := newRedisInspector(t) - q, err := newQueueRuntime(redisCfg(integrationRedis.addr)) + q, err := newQueueRuntime(withObserverAll( + withDefaultQueue(redisCfg(integrationRedis.addr), queueName), + observer, + )) if err != nil { t.Fatalf("new redis queue failed: %v", err) } + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if shutdownErr := q.Shutdown(shutdownCtx); shutdownErr != nil { + t.Errorf("shutdown redis option queue: %v", shutdownErr) + } + }) + q.Register(jobType, func(context.Context, Job) error { return nil }) + q.Register(deliveryJobType, func(context.Context, Job) error { return nil }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start redis option worker: %v", err) + } - queueName := uniqueQueueName("redis-options") delay := 2 * time.Second timeout := 7 * time.Second maxRetry := 4 start := time.Now() err = q.Dispatch( - NewJob("job:options"). + NewJob(jobType). Payload([]byte("opts")). OnQueue(queueName). Delay(delay). @@ -292,12 +524,26 @@ func TestRedisIntegration_DispatchMapsOptions(t *testing.T) { if scheduled.Timeout != timeout { t.Fatalf("expected timeout %s, got %s", timeout, scheduled.Timeout) } - if scheduled.MaxRetry != maxRetry { - t.Fatalf("expected max retry %d, got %d", maxRetry, scheduled.MaxRetry) + if scheduled.MaxRetry != maxRetry+1 { + t.Fatalf("expected transport max retry %d, got %d", maxRetry+1, scheduled.MaxRetry) + } + if got := scheduled.Headers[redisApplicationMaxRetryHeader]; got != fmt.Sprintf("%d", maxRetry) { + t.Fatalf("expected application max retry header %d, got %q", maxRetry, got) } if scheduled.NextProcessAt.Before(start.Add(delay - time.Second)) { t.Fatalf("expected next process time after delay, got %s", scheduled.NextProcessAt) } + if err := q.Dispatch(NewJob(deliveryJobType).OnQueue(queueName).Retry(maxRetry)); err != nil { + t.Fatalf("dispatch immediate Redis option probe: %v", err) + } + select { + case event := <-attempts: + if event.MaxRetry != maxRetry { + t.Fatalf("expected handler-visible max retry %d, got %d", maxRetry, event.MaxRetry) + } + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for Redis option delivery") + } } func TestRedisIntegration_DefaultTimeoutApplied(t *testing.T) { @@ -720,6 +966,84 @@ func waitForScheduledJob(t *testing.T, inspector *asynq.Inspector, queueName str return nil } +// waitForRedisActiveJob returns the matching active task once Asynq has reserved it for a handler. +func waitForRedisActiveJob(t *testing.T, inspector *asynq.Inspector, queueName, jobType string, timeout time.Duration) *asynq.TaskInfo { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + jobs, err := inspector.ListActiveTasks(queueName) + if err != nil { + t.Fatalf("list active jobs failed: %v", err) + } + for _, job := range jobs { + if job.Type == jobType { + return job + } + } + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("active job %q not found for queue %q within %s", jobType, queueName, timeout) + return nil +} + +// waitForRedisRetryJob returns the matching recovered task while it still carries Asynq's lease-expiration evidence. +func waitForRedisRetryJob(t *testing.T, inspector *asynq.Inspector, queueName, jobID string, timeout time.Duration) *asynq.TaskInfo { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + jobs, err := inspector.ListRetryTasks(queueName) + if err != nil { + t.Fatalf("list retry jobs failed: %v", err) + } + for _, job := range jobs { + if job.ID == jobID { + return job + } + } + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("retry job %q not found for queue %q within %s", jobID, queueName, timeout) + return nil +} + +// expireRedisLeaseForRecovery moves an existing Asynq lease behind its recovery skew window so the test exercises recovery without a minute-scale sleep. +func expireRedisLeaseForRecovery(ctx context.Context, queueName, jobID string) error { + leaseKey := fmt.Sprintf("asynq:{%s}:lease", queueName) + leaseScore := strconv.FormatInt(time.Now().Add(-31*time.Second).Unix(), 10) + exitCode, output, err := integrationRedis.container.Exec(ctx, []string{ + "redis-cli", "ZADD", leaseKey, "XX", "CH", leaseScore, jobID, + }) + if err != nil { + return fmt.Errorf("age Redis lease: %w", err) + } + _, err = io.Copy(io.Discard, output) + if err != nil { + return fmt.Errorf("read Redis lease response: %w", err) + } + if exitCode != 0 { + return fmt.Errorf("age Redis lease exited %d", exitCode) + } + return nil +} + +// forceRedisRetryReady advances Asynq's real retry entry so this cross-driver contract measures retry behavior without inheriting randomized production backoff. +func forceRedisRetryReady(t *testing.T, queueName string, timeout time.Duration) { + t.Helper() + inspector := newRedisInspector(t) + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + count, err := inspector.RunAllRetryTasks(queueName) + if err != nil { + t.Fatalf("run Redis retry tasks failed: %v", err) + } + if count > 0 { + return + } + time.Sleep(20 * time.Millisecond) + } + t.Fatalf("Redis retry task not found for queue %q within %s", queueName, timeout) +} + type scenarioFixture struct { name string queueName string @@ -1635,15 +1959,21 @@ func runIntegrationScenariosSuite(t *testing.T, fx scenarioFixture) { idempotencyQ := q idempotencyW := w idempotencyQueueName := fx.queueName - if fx.name == testenv.BackendSQS { - // SQS can retain invisible deliveries from earlier scenarios long enough to - // delay duplicate-processing timing by the queue visibility timeout. Isolate - // this subtest to a dedicated physical queue so it measures idempotency logic. + var idempotencyCfg any + switch fx.name { + case testenv.BackendRedis: + // A dedicated queue lets the test advance exactly its own Asynq retry entry. + idempotencyQueueName = uniqueQueueName("scenario-redis-idempotency") + idempotencyCfg = withDefaultQueue(redisCfg(integrationRedis.addr), idempotencyQueueName) + case testenv.BackendSQS: + // Isolation prevents invisible deliveries from unrelated scenarios from consuming the timing budget. idempotencyQueueName = uniqueQueueName("scenario-sqs-idempotency") - idempotencyCfg := withDefaultQueue( + idempotencyCfg = withDefaultQueue( sqsCfg(integrationSQS.region, integrationSQS.endpoint, integrationSQS.accessKey, integrationSQS.secretKey), idempotencyQueueName, ) + } + if idempotencyCfg != nil { var err error idempotencyQ, err = newQueueRuntime(idempotencyCfg) if err != nil { @@ -1653,11 +1983,11 @@ func runIntegrationScenariosSuite(t *testing.T, fx scenarioFixture) { t.Cleanup(func() { _ = idempotencyQ.Shutdown(context.Background()) }) t.Cleanup(func() { _ = idempotencyW.Shutdown(context.Background()) }) } - requireScenarioNoErr(t, "idempotency_worker_start", (idempotencyW).StartWorkers(context.Background())) jobType := "job:scenario:idempotency:" + fx.name var attempts atomic.Int32 var committed atomic.Int32 + firstAttemptDone := make(chan struct{}, 1) done := make(chan struct{}, 1) var mu sync.Mutex seen := make(map[int]struct{}) @@ -1675,6 +2005,13 @@ func runIntegrationScenariosSuite(t *testing.T, fx scenarioFixture) { committed.Add(1) } mu.Unlock() + if attempt == 1 { + select { + case firstAttemptDone <- struct{}{}: + default: + } + return errors.New("forced retry after committed side effect") + } if attempt >= 2 { select { case done <- struct{}{}: @@ -1683,31 +2020,40 @@ func runIntegrationScenariosSuite(t *testing.T, fx scenarioFixture) { } return nil }) + requireScenarioNoErr(t, "idempotency_worker_start", (idempotencyW).StartWorkers(context.Background())) - first := NewJob(jobType). - Payload(scenarioPayload{ID: 9600, Name: "idempotency"}). - OnQueue(idempotencyQueueName) - second := NewJob(jobType). + job := NewJob(jobType). Payload(scenarioPayload{ID: 9600, Name: "idempotency"}). - OnQueue(idempotencyQueueName) + OnQueue(idempotencyQueueName). + Retry(1) + if fx.supportsBackoff { + job = job.Backoff(20 * time.Millisecond) + } if fx.forceTimeout { - first = first.Timeout(jobTimeout) - second = second.Timeout(jobTimeout) + job = job.Timeout(jobTimeout) + } + requireScenarioNoErr(t, "idempotency_dispatch", idempotencyQ.Dispatch(job)) + + select { + case <-firstAttemptDone: + case <-time.After(15 * time.Second): + t.Fatalf("[idempotency_first_attempt] initial delivery did not complete") + } + if fx.name == testenv.BackendRedis { + forceRedisRetryReady(t, idempotencyQueueName, 10*time.Second) } - requireScenarioNoErr(t, "idempotency_dispatch_first", idempotencyQ.Dispatch(first)) - requireScenarioNoErr(t, "idempotency_dispatch_second", idempotencyQ.Dispatch(second)) select { case <-done: - case <-time.After(45 * time.Second): - t.Fatalf("[idempotency_done] duplicate deliveries did not both complete") + case <-time.After(20 * time.Second): + t.Fatalf("[idempotency_done] forced retry did not complete") } requireScenarioTrue(t, "idempotency_attempts", attempts.Load() >= 2, "attempts=%d expected>=2", attempts.Load()) requireScenarioTrue(t, "idempotency_side_effect_once", committed.Load() == 1, "committed=%d expected=1", committed.Load()) elapsed := time.Since(start) reportScenarioDuration(t, fx.name, "scenario_duplicate_delivery_idempotency", elapsed) - if fx.name == testenv.BackendSQS { - requireScenarioDurationLTE(t, fx.name, "scenario_duplicate_delivery_idempotency", elapsed, 45*time.Second) + if fx.name == testenv.BackendRedis || fx.name == testenv.BackendSQS { + requireScenarioDurationLTE(t, fx.name, "scenario_duplicate_delivery_idempotency", elapsed, 30*time.Second) } }) @@ -1721,8 +2067,7 @@ func runIntegrationScenariosSuite(t *testing.T, fx scenarioFixture) { qFault := fx.newQueue(t) defer func() { _ = qFault.Shutdown(context.Background()) }() - stopTimeout := 10 * time.Second - requireScenarioNoErr(t, "fault_stop_broker", integrationRedis.container.Stop(context.Background(), &stopTimeout)) + restartBroker := stopRedisBrokerForScenario(t) badCtx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() @@ -1736,9 +2081,7 @@ func runIntegrationScenariosSuite(t *testing.T, fx scenarioFixture) { err := qFault.WithContext(badCtx).Dispatch(job) requireScenarioTrue(t, "fault_dispatch_err", err != nil, "expected dispatch error while broker is down") - requireScenarioNoErr(t, "fault_start_broker", integrationRedis.container.Start(context.Background())) - requireScenarioNoErr(t, "fault_refresh_addr", refreshRedisAddr(context.Background())) - requireScenarioNoErr(t, "fault_wait_broker", waitForTCP(integrationRedis.addr, 10*time.Second)) + restartBroker() }) t.Run("scenario_consume_after_broker_recovery", func(t *testing.T) { diff --git a/integration/all/redis_uniqueness_integration_test.go b/integration/all/redis_uniqueness_integration_test.go new file mode 100644 index 0000000..76764a9 --- /dev/null +++ b/integration/all/redis_uniqueness_integration_test.go @@ -0,0 +1,129 @@ +//go:build integration + +package all_test + +import ( + "context" + "errors" + "testing" + "time" + + . "github.com/goforj/queue" + "github.com/goforj/queue/integration/testenv" +) + +// TestRedisIntegration_CanonicalUniqueAcrossClientsAndRestart verifies workflow uniqueness is shared by independent Redis producers and outlives their clients. +func TestRedisIntegration_CanonicalUniqueAcrossClientsAndRestart(t *testing.T) { + if !integrationBackendEnabled(testenv.BackendRedis) { + t.Skip("redis integration backend not selected") + } + + inspector := newRedisInspector(t) + queueName := uniqueQueueName("redis-canonical-unique") + cfg := withDefaultQueue(redisCfg(integrationRedis.addr), queueName) + first, err := newQueue(cfg) + if err != nil { + t.Fatalf("new first redis producer: %v", err) + } + second, err := newQueue(cfg) + if err != nil { + _ = first.Shutdown(context.Background()) + t.Fatalf("new second redis producer: %v", err) + } + t.Cleanup(func() { + _ = first.Shutdown(context.Background()) + _ = second.Shutdown(context.Background()) + }) + + type payload struct { + AccountID string `json:"account_id"` + } + ttl := 2 * time.Second + jobType := uniqueQueueJobType("queue:redis:canonical-unique") + newUniqueJob := func() Job { + return NewJob(jobType). + Payload(payload{AccountID: "account-123"}). + OnQueue(queueName). + UniqueFor(ttl) + } + + start := make(chan struct{}) + results := make(chan error, 2) + for _, producer := range []*Queue{first, second} { + producer := producer + go func() { + <-start + _, dispatchErr := producer.Dispatch(newUniqueJob()) + results <- dispatchErr + }() + } + dispatchStarted := time.Now() + close(start) + + accepted := 0 + duplicates := 0 + for range 2 { + dispatchErr := <-results + switch { + case dispatchErr == nil: + accepted++ + case errors.Is(dispatchErr, ErrDuplicate): + duplicates++ + default: + t.Fatalf("cross-client dispatch returned unexpected error: %v", dispatchErr) + } + } + if accepted != 1 || duplicates != 1 { + t.Fatalf("cross-client accepted/duplicate results = %d/%d, want 1/1", accepted, duplicates) + } + + pending, err := inspector.ListPendingTasks(queueName) + if err != nil { + t.Fatalf("list pending tasks after cross-client dispatch: %v", err) + } + if len(pending) != 1 { + t.Fatalf("pending tasks after cross-client dispatch = %d, want 1", len(pending)) + } + + if err := first.Shutdown(context.Background()); err != nil { + t.Fatalf("shutdown first redis producer: %v", err) + } + if err := second.Shutdown(context.Background()); err != nil { + t.Fatalf("shutdown second redis producer: %v", err) + } + restarted, err := newQueue(cfg) + if err != nil { + t.Fatalf("restart redis producer: %v", err) + } + t.Cleanup(func() { _ = restarted.Shutdown(context.Background()) }) + + if _, err := restarted.Dispatch(newUniqueJob()); !errors.Is(err, ErrDuplicate) { + t.Fatalf("restarted producer dispatch error = %v, want ErrDuplicate", err) + } + + deadline := time.Now().Add(ttl + 3*time.Second) + for { + _, err = restarted.Dispatch(newUniqueJob()) + if err == nil { + break + } + if !errors.Is(err, ErrDuplicate) { + t.Fatalf("dispatch while waiting for uniqueness expiry: %v", err) + } + if time.Now().After(deadline) { + t.Fatalf("canonical uniqueness claim did not expire within %s", ttl+3*time.Second) + } + time.Sleep(25 * time.Millisecond) + } + if elapsed := time.Since(dispatchStarted); elapsed < ttl { + t.Fatalf("canonical uniqueness claim expired after %s, before TTL %s", elapsed, ttl) + } + + pending, err = inspector.ListPendingTasks(queueName) + if err != nil { + t.Fatalf("list pending tasks after uniqueness expiry: %v", err) + } + if len(pending) != 2 { + t.Fatalf("pending tasks after uniqueness expiry = %d, want 2", len(pending)) + } +} diff --git a/integration/all/runtime_integration_test.go b/integration/all/runtime_integration_test.go index f2a45c7..3afc867 100644 --- a/integration/all/runtime_integration_test.go +++ b/integration/all/runtime_integration_test.go @@ -158,6 +158,8 @@ func TestIntegrationQueue_AllBackends(t *testing.T) { _ = q.Shutdown(shutdownCtx) }() + testQueueWorkflowUniqueScenario(t, q, queueName) + if !backend.executes { testQueueWorkflowNullScenario(t, q, queueName) return @@ -171,6 +173,31 @@ func TestIntegrationQueue_AllBackends(t *testing.T) { } } +// testQueueWorkflowUniqueScenario exercises logical identity through the normal public facade. +func testQueueWorkflowUniqueScenario(t *testing.T, q *Queue, queueName string) { + t.Helper() + + type payload struct { + AccountID string `json:"account_id"` + } + + jobType := uniqueQueueJobType("queue:unique") + q.Register(jobType, func(context.Context, Message) error { return nil }) + newUniqueJob := func() Job { + return NewJob(jobType). + Payload(payload{AccountID: "account-123"}). + OnQueue(queueName). + UniqueFor(time.Minute) + } + + if _, err := q.Dispatch(newUniqueJob()); err != nil { + t.Fatalf("unique scenario: initial dispatch failed: %v", err) + } + if _, err := q.Dispatch(newUniqueJob()); !errors.Is(err, ErrDuplicate) { + t.Fatalf("unique scenario: duplicate dispatch error = %v, want %v", err, ErrDuplicate) + } +} + func testQueueWorkflowNullScenario(t *testing.T, q *Queue, queueName string) { t.Helper() @@ -215,7 +242,7 @@ func testQueueWorkflowDispatchScenario(t *testing.T, q *Queue, queueName string) type payload struct { URL string `json:"url"` } - seen := make(chan string, 1) + seen := make(chan Message, 1) jobType := uniqueQueueJobType("queue:dispatch") q.Register(jobType, func(_ context.Context, j Message) error { @@ -224,13 +251,13 @@ func testQueueWorkflowDispatchScenario(t *testing.T, q *Queue, queueName string) return err } select { - case seen <- p.URL: + case seen <- j: default: } return nil }) - _, err := q.Dispatch(NewJob(jobType).Payload(payload{ + result, err := q.Dispatch(NewJob(jobType).Payload(payload{ URL: "https://goforj.dev/health", }).OnQueue(queueName)) if err != nil { @@ -238,9 +265,19 @@ func testQueueWorkflowDispatchScenario(t *testing.T, q *Queue, queueName string) } select { - case got := <-seen: - if got != "https://goforj.dev/health" { - t.Fatalf("dispatch scenario: unexpected url %q", got) + case message := <-seen: + var got payload + if err := message.Bind(&got); err != nil { + t.Fatalf("dispatch scenario: bind delivered payload: %v", err) + } + if got.URL != "https://goforj.dev/health" { + t.Fatalf("dispatch scenario: unexpected url %q", got.URL) + } + if message.SchemaVersion == 0 || message.DispatchID != result.DispatchID || message.JobID == "" { + t.Fatalf("dispatch scenario: incomplete correlation message=%+v result=%+v", message, result) + } + if message.JobType != jobType || message.ChainID != "" || message.BatchID != "" { + t.Fatalf("dispatch scenario: incorrect direct identity %+v", message) } case <-time.After(20 * time.Second): t.Fatal("dispatch scenario: timed out waiting for handler") diff --git a/integration/all/sql_uniqueness_integration_test.go b/integration/all/sql_uniqueness_integration_test.go new file mode 100644 index 0000000..fb60d73 --- /dev/null +++ b/integration/all/sql_uniqueness_integration_test.go @@ -0,0 +1,99 @@ +//go:build integration + +package all_test + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + . "github.com/goforj/queue" + "github.com/goforj/queue/integration/testenv" +) + +// TestSQLiteIntegrationCanonicalUniqueAcrossPublicClients verifies SQL claiming and logical envelope identity in one public composition. +func TestSQLiteIntegrationCanonicalUniqueAcrossPublicClients(t *testing.T) { + if !integrationBackendEnabled(testenv.BackendSQLite) { + t.Skip("sqlite integration backend not selected") + } + + queueName := uniqueQueueName("sqlite-canonical-unique") + cfg := withDefaultQueue(sqliteCfg(fmt.Sprintf("%s/queue-public-unique.db", t.TempDir())), queueName) + first := newStartedUniqueSQLiteQueue(t, cfg, queueName) + second := newStartedUniqueSQLiteQueue(t, cfg, queueName) + + type payload struct { + AccountID string `json:"account_id"` + } + jobType := uniqueQueueJobType("queue:sqlite:canonical-unique") + first.Register(jobType, func(context.Context, Message) error { return nil }) + second.Register(jobType, func(context.Context, Message) error { return nil }) + newUniqueJob := func() Job { + return NewJob(jobType). + Payload(payload{AccountID: "account-123"}). + OnQueue(queueName). + UniqueFor(time.Minute) + } + + start := make(chan struct{}) + results := make(chan error, 2) + for _, producer := range []*Queue{first, second} { + producer := producer + go func() { + <-start + _, dispatchErr := producer.Dispatch(newUniqueJob()) + results <- dispatchErr + }() + } + close(start) + accepted := 0 + duplicates := 0 + for range 2 { + dispatchErr := <-results + switch { + case dispatchErr == nil: + accepted++ + case errors.Is(dispatchErr, ErrDuplicate): + duplicates++ + default: + t.Fatalf("cross-client dispatch returned unexpected error: %v", dispatchErr) + } + } + if accepted != 1 || duplicates != 1 { + t.Fatalf("cross-client accepted/duplicate results = %d/%d, want 1/1", accepted, duplicates) + } + + shutdownQueue(t, first) + shutdownQueue(t, second) + restarted := newStartedUniqueSQLiteQueue(t, cfg, queueName) + restarted.Register(jobType, func(context.Context, Message) error { return nil }) + if _, err := restarted.Dispatch(newUniqueJob()); !errors.Is(err, ErrDuplicate) { + t.Fatalf("restarted public producer dispatch error = %v, want ErrDuplicate", err) + } +} + +// newStartedUniqueSQLiteQueue constructs an independent public producer/worker over one shared SQLite file. +func newStartedUniqueSQLiteQueue(t *testing.T, cfg any, queueName string) *Queue { + t.Helper() + q, err := newQueue(cfg, WithWorkers(1)) + if err != nil { + t.Fatalf("new sqlite public queue: %v", err) + } + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start sqlite public queue %q: %v", queueName, err) + } + t.Cleanup(func() { shutdownQueue(t, q) }) + return q +} + +// shutdownQueue drains one public queue with a bounded test deadline. +func shutdownQueue(t *testing.T, q *Queue) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := q.Shutdown(ctx); err != nil { + t.Fatalf("shutdown queue: %v", err) + } +} diff --git a/integration/go.mod b/integration/go.mod index 38c9d03..36ddfd3 100644 --- a/integration/go.mod +++ b/integration/go.mod @@ -1,6 +1,6 @@ module github.com/goforj/queue/integration -go 1.24.4 +go 1.25.0 replace github.com/goforj/queue => .. @@ -32,7 +32,7 @@ require ( github.com/goforj/queue/driver/sqlitequeue v0.0.0 github.com/goforj/queue/driver/sqsqueue v0.0.0 github.com/hibiken/asynq v0.26.0 - github.com/jackc/pgx/v5 v5.8.0 + github.com/jackc/pgx/v5 v5.9.2 github.com/rabbitmq/amqp091-go v1.10.0 github.com/testcontainers/testcontainers-go v0.39.0 modernc.org/sqlite v1.46.0 diff --git a/integration/go.sum b/integration/go.sum index 6780366..05550d5 100644 --- a/integration/go.sum +++ b/integration/go.sum @@ -107,8 +107,8 @@ github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsI github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo= github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx/v5 v5.8.0 h1:TYPDoleBBme0xGSAX3/+NujXXtpZn9HBONkQC7IEZSo= -github.com/jackc/pgx/v5 v5.8.0/go.mod h1:QVeDInX2m9VyzvNeiCJVjCkNFqzsNb43204HshNSZKw= +github.com/jackc/pgx/v5 v5.9.2 h1:3ZhOzMWnR4yJ+RW1XImIPsD1aNSz4T4fyP7zlQb56hw= +github.com/jackc/pgx/v5 v5.9.2/go.mod h1:mal1tBGAFfLHvZzaYh77YS/eC6IX9OWbRV1QIIM0Jn4= github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo= github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= diff --git a/integration/root/database_integration_test.go b/integration/root/database_integration_test.go index 6bec153..f75a48a 100644 --- a/integration/root/database_integration_test.go +++ b/integration/root/database_integration_test.go @@ -4,43 +4,2125 @@ package root_test import ( "context" + "crypto/sha256" "database/sql" + "encoding/hex" "errors" "fmt" + "math" + "strings" + "sync" "sync/atomic" "testing" "time" _ "github.com/go-sql-driver/mysql" "github.com/goforj/queue" + "github.com/goforj/queue/busruntime" + "github.com/goforj/queue/driver/sqlitequeue" "github.com/goforj/queue/integration/testenv" _ "github.com/jackc/pgx/v5/stdlib" _ "modernc.org/sqlite" ) +type databaseSettlementRecorder struct { + mu sync.Mutex + events []queue.Event + settlement chan struct{} + once sync.Once +} + +// prepareSQLiteIntegrationSchema completes migration and worker cleanup before a fault trigger takes ownership of the database. +func prepareSQLiteIntegrationSchema(t *testing.T, dsn string) { + t.Helper() + bootstrap, err := sqlitequeue.New(dsn) + if err != nil { + t.Fatalf("new SQLite schema bootstrap: %v", err) + } + if err := bootstrap.StartWorkers(context.Background()); err != nil { + t.Fatalf("start SQLite schema bootstrap: %v", err) + } + if err := bootstrap.Shutdown(context.Background()); err != nil { + t.Fatalf("shutdown SQLite schema bootstrap: %v", err) + } +} + +// execSQLiteIntegrationEventually tolerates the worker's short polling lock +// while keeping deterministic fault-fixture schema changes bounded. +func execSQLiteIntegrationEventually(db *sql.DB, query string, args ...any) (sql.Result, error) { + deadline := time.Now().Add(5 * time.Second) + for { + result, err := db.Exec(query, args...) + if err == nil { + return result, nil + } + message := strings.ToLower(err.Error()) + if (!strings.Contains(message, "busy") && !strings.Contains(message, "locked")) || time.Now().After(deadline) { + return nil, err + } + time.Sleep(10 * time.Millisecond) + } +} + +// Observe records SQL delivery events and signals the first finalization failure. +func (r *databaseSettlementRecorder) Observe(_ context.Context, event queue.Event) { + r.mu.Lock() + r.events = append(r.events, event) + r.mu.Unlock() + if event.Kind == queue.EventSettlementFailed { + r.once.Do(func() { close(r.settlement) }) + } +} + +// has reports whether the recorder contains one matching event. +func (r *databaseSettlementRecorder) has(kind queue.EventKind, jobType string) bool { + r.mu.Lock() + defer r.mu.Unlock() + for _, event := range r.events { + if event.Kind == kind && event.JobType == jobType { + return true + } + } + return false +} + +// count returns the number of recorded events matching one kind and logical job type. +func (r *databaseSettlementRecorder) count(kind queue.EventKind, jobType string) int { + r.mu.Lock() + defer r.mu.Unlock() + count := 0 + for _, event := range r.events { + if event.Kind == kind && event.JobType == jobType { + count++ + } + } + return count +} + +// first returns one matching event so integration assertions can verify its +// correlation fields instead of relying only on aggregate counts. +func (r *databaseSettlementRecorder) first(kind queue.EventKind, jobType string) (queue.Event, bool) { + r.mu.Lock() + defer r.mu.Unlock() + for _, event := range r.events { + if event.Kind == kind && event.JobType == jobType { + return event, true + } + } + return queue.Event{}, false +} + +// runSQLiteWorkflowWinnerFactRecovery proves receipt-backed recovery publishes +// the workflow facts already committed by the winning generation without +// executing application code a second time. +func runSQLiteWorkflowWinnerFactRecovery(t *testing.T, workflowKind string) { + t.Helper() + queueDSN := fmt.Sprintf("%s/queue-workflow-recovery-%s-%d.db", t.TempDir(), workflowKind, time.Now().UnixNano()) + workflowDSN := fmt.Sprintf("%s/workflow-recovery-%s-%d.db", t.TempDir(), workflowKind, time.Now().UnixNano()) + prepareSQLiteIntegrationSchema(t, queueDSN) + + workflowDB, err := sql.Open(testenv.BackendSQLite, workflowDSN) + if err != nil { + t.Fatalf("open workflow recovery store: %v", err) + } + t.Cleanup(func() { _ = workflowDB.Close() }) + store, err := queue.NewSQLStore(queue.SQLStoreConfig{ + DB: workflowDB, + DriverName: testenv.BackendSQLite, + AutoMigrate: true, + }) + if err != nil { + t.Fatalf("new workflow recovery store: %v", err) + } + + recorder := &databaseSettlementRecorder{settlement: make(chan struct{})} + queueName := "workflow-recovery-" + workflowKind + runtimeCfg := withDBRecoveryPolicy(withDefaultQueue(sqliteCfg(queueDSN), queueName), 10*time.Millisecond, 30*time.Second) + runtimeCfg.DisableAutoMigrate = true + runtime, err := testenv.NewQueue(runtimeCfg, queue.WithStore(store), queue.WithObserver(recorder), queue.WithWorkers(1)) + if err != nil { + t.Fatalf("new workflow recovery runtime: %v", err) + } + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = runtime.Shutdown(shutdownCtx) + }) + + chainPredecessor := workflowKind == "chain_predecessor" + batchPredecessor := workflowKind == "batch_predecessor" + jobType := "job:db:workflow-recovery:" + workflowKind + successorJobType := "" + if chainPredecessor || batchPredecessor { + jobType += ":first" + successorJobType = "job:db:workflow-recovery:" + workflowKind + ":final" + } + var handlerCalls, successorCalls atomic.Int64 + runtime.Register(jobType, func(context.Context, queue.Message) error { + handlerCalls.Add(1) + return nil + }) + if successorJobType != "" { + runtime.Register(successorJobType, func(context.Context, queue.Message) error { + successorCalls.Add(1) + return nil + }) + } + + queueDB, err := sql.Open(testenv.BackendSQLite, queueDSN) + if err != nil { + t.Fatalf("open workflow recovery queue database: %v", err) + } + defer queueDB.Close() + triggerName := "reject_" + workflowKind + "_workflow_finalization" + trigger := fmt.Sprintf(`CREATE TRIGGER %s +BEFORE DELETE ON queue_jobs +WHEN OLD.queue_name = '%s' AND OLD.id = 1 +BEGIN + SELECT RAISE(ABORT, 'forced workflow finalization failure'); +END`, triggerName, queueName) + if _, err := queueDB.Exec(trigger); err != nil { + t.Fatalf("create workflow finalization trigger: %v", err) + } + if err := runtime.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workflow recovery runtime: %v", err) + } + + var workflowID string + primaryJob := queue.NewJob(jobType).OnQueue(queueName) + switch workflowKind { + case "chain": + workflowID, err = runtime.Chain(primaryJob).Dispatch(context.Background()) + case "chain_predecessor": + workflowID, err = runtime.Chain( + primaryJob, + queue.NewJob(successorJobType).OnQueue(queueName), + ).Dispatch(context.Background()) + case "batch": + workflowID, err = runtime.Batch(primaryJob).OnQueue(queueName).Dispatch(context.Background()) + case "batch_predecessor": + workflowID, err = runtime.Batch(primaryJob, queue.NewJob(successorJobType).OnQueue(queueName)).OnQueue(queueName).Dispatch(context.Background()) + default: + t.Fatalf("unsupported workflow kind %q", workflowKind) + } + if err != nil { + t.Fatalf("dispatch %s recovery workflow: %v", workflowKind, err) + } + + select { + case <-recorder.settlement: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for workflow settlement failure") + } + if handlerCalls.Load() != 1 { + t.Fatalf("handler calls before recovery = %d, want 1", handlerCalls.Load()) + } + if recorder.count(queue.EventJobSucceeded, jobType) != 0 { + t.Fatal("job success published before durable queue settlement") + } + switch workflowKind { + case "chain": + state, stateErr := store.GetChain(context.Background(), workflowID) + if stateErr != nil { + t.Fatalf("get committed chain: %v", stateErr) + } + if !state.Completed || state.Failed || recorder.count(queue.EventChainCompleted, jobType) != 0 { + t.Fatalf("chain before recovery = state:%+v completed facts:%d", state, recorder.count(queue.EventChainCompleted, jobType)) + } + case "chain_predecessor": + waitForObservabilityScenario(t, "sqlite_workflow_successor_settlement", 5*time.Second, func() bool { + state, stateErr := store.GetChain(context.Background(), workflowID) + return stateErr == nil && state.Completed && !state.Failed && successorCalls.Load() == 1 && recorder.count(queue.EventChainCompleted, successorJobType) == 1 + }) + if recorder.count(queue.EventChainAdvanced, jobType) != 0 || recorder.count(queue.EventChainCompleted, jobType) != 0 { + t.Fatalf("predecessor facts before recovery = advanced:%d completed:%d", recorder.count(queue.EventChainAdvanced, jobType), recorder.count(queue.EventChainCompleted, jobType)) + } + case "batch": + state, stateErr := store.GetBatch(context.Background(), workflowID) + if stateErr != nil { + t.Fatalf("get committed batch: %v", stateErr) + } + if !state.Completed || state.Cancelled || state.Failed != 0 || recorder.count(queue.EventBatchProgressed, jobType) != 0 || recorder.count(queue.EventBatchCompleted, jobType) != 0 { + t.Fatalf("batch before recovery = state:%+v progress/completed facts:%d/%d", state, recorder.count(queue.EventBatchProgressed, jobType), recorder.count(queue.EventBatchCompleted, jobType)) + } + case "batch_predecessor": + waitForObservabilityScenario(t, "sqlite_batch_terminal_member_settlement", 5*time.Second, func() bool { + state, stateErr := store.GetBatch(context.Background(), workflowID) + return stateErr == nil && state.Completed && !state.Cancelled && successorCalls.Load() == 1 && recorder.count(queue.EventBatchCompleted, successorJobType) == 1 + }) + if recorder.count(queue.EventBatchProgressed, jobType) != 0 || recorder.count(queue.EventBatchCompleted, jobType) != 0 { + t.Fatalf("stale batch member facts before recovery = progress:%d completed:%d", recorder.count(queue.EventBatchProgressed, jobType), recorder.count(queue.EventBatchCompleted, jobType)) + } + } + + if _, err := execSQLiteIntegrationEventually(queueDB, "DROP TRIGGER "+triggerName); err != nil { + t.Fatalf("drop workflow finalization trigger: %v", err) + } + if _, err := execSQLiteIntegrationEventually(queueDB, `UPDATE queue_jobs SET processing_started_at=1 WHERE queue_name=? AND state='processing'`, queueName); err != nil { + t.Fatalf("age workflow delivery for recovery: %v", err) + } + waitForObservabilityScenario(t, "sqlite_workflow_winner_fact_recovery_"+workflowKind, 5*time.Second, func() bool { + if recorder.count(queue.EventJobSucceeded, jobType) != 1 { + return false + } + if workflowKind == "chain" { + return recorder.count(queue.EventChainCompleted, jobType) == 1 + } + if chainPredecessor { + return recorder.count(queue.EventChainAdvanced, jobType) == 1 && recorder.count(queue.EventChainCompleted, jobType) == 0 + } + if batchPredecessor { + return recorder.count(queue.EventBatchProgressed, jobType) == 1 && recorder.count(queue.EventBatchCompleted, jobType) == 0 && recorder.count(queue.EventBatchCompleted, successorJobType) == 1 + } + return recorder.count(queue.EventBatchProgressed, jobType) == 1 && recorder.count(queue.EventBatchCompleted, jobType) == 1 + }) + if handlerCalls.Load() != 1 { + t.Fatalf("handler calls after receipt-backed recovery = %d, want 1", handlerCalls.Load()) + } + succeeded, ok := recorder.first(queue.EventJobSucceeded, jobType) + if !ok || succeeded.Attempt != 0 || succeeded.EventID == "" { + t.Fatalf("recovered attempt-zero success = %+v present:%t", succeeded, ok) + } + if (chainPredecessor || batchPredecessor) && successorCalls.Load() != 1 { + t.Fatalf("successor calls after predecessor recovery = %d, want 1", successorCalls.Load()) + } + var remaining int + if err := queueDB.QueryRow(`SELECT COUNT(*) FROM queue_jobs WHERE queue_name=?`, queueName).Scan(&remaining); err != nil { + t.Fatalf("count recovered workflow deliveries: %v", err) + } + if remaining != 0 { + t.Fatalf("recovered workflow deliveries = %d, want 0", remaining) + } + if recorder.count(queue.EventJobFailed, jobType) != 0 || recorder.count(queue.EventChainFailed, jobType) != 0 || recorder.count(queue.EventBatchFailed, jobType) != 0 || recorder.count(queue.EventBatchCancelled, jobType) != 0 { + t.Fatal("contradictory replay published losing workflow facts") + } +} + +// runSQLiteRepeatedWorkflowSettlementRecovery proves multiple recovery +// finalization failures retain the original receipt owner until one later +// generation positively settles the physical row and releases deferred facts. +func runSQLiteRepeatedWorkflowSettlementRecovery(t *testing.T) { + t.Helper() + queueDSN := fmt.Sprintf("%s/queue-repeated-workflow-recovery-%d.db", t.TempDir(), time.Now().UnixNano()) + workflowDSN := fmt.Sprintf("%s/workflow-repeated-recovery-%d.db", t.TempDir(), time.Now().UnixNano()) + prepareSQLiteIntegrationSchema(t, queueDSN) + + workflowDB, err := sql.Open(testenv.BackendSQLite, workflowDSN) + if err != nil { + t.Fatalf("open repeated-recovery workflow store: %v", err) + } + t.Cleanup(func() { _ = workflowDB.Close() }) + store, err := queue.NewSQLStore(queue.SQLStoreConfig{ + DB: workflowDB, + DriverName: testenv.BackendSQLite, + AutoMigrate: true, + }) + if err != nil { + t.Fatalf("new repeated-recovery workflow store: %v", err) + } + + const ( + queueName = "repeated-workflow-recovery" + jobType = "job:db:repeated-workflow-recovery" + ) + recorder := &databaseSettlementRecorder{settlement: make(chan struct{})} + runtimeCfg := withDBRecoveryPolicy(withDefaultQueue(sqliteCfg(queueDSN), queueName), 10*time.Millisecond, 30*time.Second) + runtimeCfg.DisableAutoMigrate = true + firstRuntime, err := testenv.NewQueue(runtimeCfg, queue.WithStore(store), queue.WithObserver(recorder), queue.WithWorkers(1)) + if err != nil { + t.Fatalf("new first repeated-recovery runtime: %v", err) + } + firstStopped := false + t.Cleanup(func() { + if firstStopped { + return + } + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = firstRuntime.Shutdown(shutdownCtx) + }) + + var handlerCalls atomic.Int64 + firstRuntime.Register(jobType, func(context.Context, queue.Message) error { + handlerCalls.Add(1) + return nil + }) + + queueDB, err := sql.Open(testenv.BackendSQLite, queueDSN) + if err != nil { + t.Fatalf("open repeated-recovery queue database: %v", err) + } + defer queueDB.Close() + const triggerName = "reject_repeated_workflow_finalization" + trigger := fmt.Sprintf(`CREATE TRIGGER %s +BEFORE DELETE ON queue_jobs +WHEN OLD.queue_name = '%s' +BEGIN + SELECT RAISE(ABORT, 'forced repeated workflow finalization failure'); +END`, triggerName, queueName) + if _, err := queueDB.Exec(trigger); err != nil { + t.Fatalf("create repeated-recovery finalization trigger: %v", err) + } + triggerInstalled := true + defer func() { + if triggerInstalled { + _, _ = execSQLiteIntegrationEventually(queueDB, "DROP TRIGGER "+triggerName) + } + }() + if err := firstRuntime.StartWorkers(context.Background()); err != nil { + t.Fatalf("start first repeated-recovery runtime: %v", err) + } + chainID, err := firstRuntime.Chain(queue.NewJob(jobType).OnQueue(queueName)).Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch repeated-recovery chain: %v", err) + } + select { + case <-recorder.settlement: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for initial repeated-recovery settlement failure") + } + if recorder.count(queue.EventSettlementFailed, jobType) != 1 { + t.Fatalf("initial settlement failures = %d, want 1 before stale recovery", recorder.count(queue.EventSettlementFailed, jobType)) + } + if handlerCalls.Load() != 1 || recorder.count(queue.EventJobSucceeded, jobType) != 0 || recorder.count(queue.EventChainCompleted, jobType) != 0 || recorder.count(queue.EventProcessSucceeded, jobType) != 0 { + t.Fatalf("initial calls/job/chain/process facts = %d/%d/%d/%d, want 1/0/0/0", handlerCalls.Load(), recorder.count(queue.EventJobSucceeded, jobType), recorder.count(queue.EventChainCompleted, jobType), recorder.count(queue.EventProcessSucceeded, jobType)) + } + + var ( + rowID int64 + rowState string + rowAttempt int + originalGeneration string + ) + if err := queueDB.QueryRow(`SELECT id, state, attempt, processing_token FROM queue_jobs WHERE queue_name=?`, queueName).Scan(&rowID, &rowState, &rowAttempt, &originalGeneration); err != nil { + t.Fatalf("read initial repeated-recovery delivery: %v", err) + } + if rowState != "processing" || rowAttempt != 0 || originalGeneration == "" { + t.Fatalf("initial repeated-recovery delivery = id:%d state:%q attempt:%d generation:%q", rowID, rowState, rowAttempt, originalGeneration) + } + var receiptOwner string + if err := workflowDB.QueryRow(`SELECT owner_delivery_id FROM bus_workflow_transition_receipts WHERE workflow_kind='chain' AND workflow_id=?`, chainID).Scan(&receiptOwner); err != nil { + t.Fatalf("read repeated-recovery receipt owner: %v", err) + } + if receiptOwner != originalGeneration { + t.Fatalf("receipt owner = %q, want initial generation %q", receiptOwner, originalGeneration) + } + + if _, err := execSQLiteIntegrationEventually(queueDB, `UPDATE queue_jobs SET processing_started_at=1 WHERE id=? AND state='processing'`, rowID); err != nil { + t.Fatalf("age repeated-recovery delivery: %v", err) + } + waitForObservabilityScenario(t, "sqlite_repeated_workflow_settlement_failures", 5*time.Second, func() bool { + return recorder.count(queue.EventSettlementFailed, jobType) >= 3 + }) + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + if err := firstRuntime.Shutdown(shutdownCtx); err != nil { + cancel() + t.Fatalf("shutdown repeated-recovery fault runtime: %v", err) + } + cancel() + firstStopped = true + + var ( + pendingState string + pendingAttempt int + pendingGeneration string + processingStarted sql.NullInt64 + ) + if err := queueDB.QueryRow(`SELECT state, attempt, processing_token, processing_started_at FROM queue_jobs WHERE id=?`, rowID).Scan(&pendingState, &pendingAttempt, &pendingGeneration, &processingStarted); err != nil { + t.Fatalf("read repaired repeated-recovery delivery: %v", err) + } + if pendingState != "pending" || pendingAttempt != 0 || pendingGeneration != receiptOwner || processingStarted.Valid { + t.Fatalf("repaired delivery = state:%q attempt:%d generation:%q started:%#v, want pending/0/%q/NULL", pendingState, pendingAttempt, pendingGeneration, processingStarted, receiptOwner) + } + if handlerCalls.Load() != 1 || recorder.count(queue.EventJobSucceeded, jobType) != 0 || recorder.count(queue.EventChainCompleted, jobType) != 0 || recorder.count(queue.EventProcessSucceeded, jobType) != 0 { + t.Fatalf("pre-final calls/job/chain/process facts = %d/%d/%d/%d, want 1/0/0/0", handlerCalls.Load(), recorder.count(queue.EventJobSucceeded, jobType), recorder.count(queue.EventChainCompleted, jobType), recorder.count(queue.EventProcessSucceeded, jobType)) + } + if _, err := execSQLiteIntegrationEventually(queueDB, "DROP TRIGGER "+triggerName); err != nil { + t.Fatalf("drop repeated-recovery finalization trigger: %v", err) + } + triggerInstalled = false + + finalRuntime, err := testenv.NewQueue(runtimeCfg, queue.WithStore(store), queue.WithObserver(recorder), queue.WithWorkers(1)) + if err != nil { + t.Fatalf("new final repeated-recovery runtime: %v", err) + } + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = finalRuntime.Shutdown(shutdownCtx) + }) + finalRuntime.Register(jobType, func(context.Context, queue.Message) error { + handlerCalls.Add(1) + return queue.Permanent(errors.New("receipt recovery re-executed application code")) + }) + if err := finalRuntime.StartWorkers(context.Background()); err != nil { + t.Fatalf("start final repeated-recovery runtime: %v", err) + } + waitForObservabilityScenario(t, "sqlite_repeated_workflow_final_settlement", 5*time.Second, func() bool { + var remaining int + rowErr := queueDB.QueryRow(`SELECT COUNT(*) FROM queue_jobs WHERE id=?`, rowID).Scan(&remaining) + return rowErr == nil && remaining == 0 && recorder.count(queue.EventJobSucceeded, jobType) == 1 && recorder.count(queue.EventChainCompleted, jobType) == 1 && recorder.count(queue.EventProcessSucceeded, jobType) == 1 + }) + if handlerCalls.Load() != 1 { + t.Fatalf("handler calls after repeated recovery = %d, want 1", handlerCalls.Load()) + } + if recorder.count(queue.EventJobSucceeded, jobType) != 1 || recorder.count(queue.EventChainCompleted, jobType) != 1 || recorder.count(queue.EventProcessSucceeded, jobType) != 1 { + t.Fatalf("final job/chain/process facts = %d/%d/%d, want 1/1/1", recorder.count(queue.EventJobSucceeded, jobType), recorder.count(queue.EventChainCompleted, jobType), recorder.count(queue.EventProcessSucceeded, jobType)) + } + if recorder.count(queue.EventJobFailed, jobType) != 0 || recorder.count(queue.EventChainFailed, jobType) != 0 { + t.Fatal("repeated settlement recovery published contradictory failure facts") + } +} + +// runSQLiteTerminalBatchOwnerRecovery proves the terminal member's receipt, +// rather than an earlier settled member, owns recovered aggregate completion. +func runSQLiteTerminalBatchOwnerRecovery(t *testing.T) { + t.Helper() + queueDSN := fmt.Sprintf("%s/queue-terminal-batch-owner-%d.db", t.TempDir(), time.Now().UnixNano()) + workflowDSN := fmt.Sprintf("%s/workflow-terminal-batch-owner-%d.db", t.TempDir(), time.Now().UnixNano()) + prepareSQLiteIntegrationSchema(t, queueDSN) + + workflowDB, err := sql.Open(testenv.BackendSQLite, workflowDSN) + if err != nil { + t.Fatalf("open terminal batch workflow store: %v", err) + } + t.Cleanup(func() { _ = workflowDB.Close() }) + store, err := queue.NewSQLStore(queue.SQLStoreConfig{ + DB: workflowDB, + DriverName: testenv.BackendSQLite, + AutoMigrate: true, + }) + if err != nil { + t.Fatalf("new terminal batch workflow store: %v", err) + } + + const ( + queueName = "terminal-batch-owner" + firstJobType = "job:db:terminal-batch-owner:first" + terminalJobType = "job:db:terminal-batch-owner:terminal" + ) + recorder := &databaseSettlementRecorder{settlement: make(chan struct{})} + runtimeCfg := withDBRecoveryPolicy(withDefaultQueue(sqliteCfg(queueDSN), queueName), 10*time.Millisecond, 30*time.Second) + runtimeCfg.DisableAutoMigrate = true + runtime, err := testenv.NewQueue(runtimeCfg, queue.WithStore(store), queue.WithObserver(recorder), queue.WithWorkers(1)) + if err != nil { + t.Fatalf("new terminal batch recovery runtime: %v", err) + } + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = runtime.Shutdown(shutdownCtx) + }) + + var firstCalls, terminalCalls atomic.Int64 + runtime.Register(firstJobType, func(context.Context, queue.Message) error { + firstCalls.Add(1) + return nil + }) + runtime.Register(terminalJobType, func(context.Context, queue.Message) error { + terminalCalls.Add(1) + return nil + }) + + queueDB, err := sql.Open(testenv.BackendSQLite, queueDSN) + if err != nil { + t.Fatalf("open terminal batch queue database: %v", err) + } + defer queueDB.Close() + const triggerName = "reject_terminal_batch_finalization" + trigger := fmt.Sprintf(`CREATE TRIGGER %s +BEFORE DELETE ON queue_jobs +WHEN OLD.queue_name = '%s' AND OLD.id = 2 +BEGIN + SELECT RAISE(ABORT, 'forced terminal batch finalization failure'); +END`, triggerName, queueName) + if _, err := queueDB.Exec(trigger); err != nil { + t.Fatalf("create terminal batch finalization trigger: %v", err) + } + if err := runtime.StartWorkers(context.Background()); err != nil { + t.Fatalf("start terminal batch recovery runtime: %v", err) + } + + batchID, err := runtime.Batch( + queue.NewJob(firstJobType).OnQueue(queueName), + queue.NewJob(terminalJobType).OnQueue(queueName), + ).OnQueue(queueName).Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch terminal batch recovery workflow: %v", err) + } + select { + case <-recorder.settlement: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for terminal batch settlement failure") + } + + state, err := store.GetBatch(context.Background(), batchID) + if err != nil { + t.Fatalf("get committed terminal batch: %v", err) + } + if !state.Completed || state.Cancelled || state.Failed != 0 || state.Processed != 2 || state.Pending != 0 { + t.Fatalf("terminal batch state before recovery = %+v", state) + } + if firstCalls.Load() != 1 || terminalCalls.Load() != 1 { + t.Fatalf("first/terminal handler calls before recovery = %d/%d, want 1/1", firstCalls.Load(), terminalCalls.Load()) + } + if recorder.count(queue.EventJobSucceeded, firstJobType) != 1 || recorder.count(queue.EventBatchProgressed, firstJobType) != 1 { + t.Fatalf("settled first-member job/progress facts = %d/%d, want 1/1", recorder.count(queue.EventJobSucceeded, firstJobType), recorder.count(queue.EventBatchProgressed, firstJobType)) + } + if recorder.count(queue.EventJobSucceeded, terminalJobType) != 0 || recorder.count(queue.EventBatchProgressed, terminalJobType) != 0 || recorder.count(queue.EventBatchCompleted, terminalJobType) != 0 { + t.Fatalf("terminal facts before recovery = success/progress/completed %d/%d/%d, want 0/0/0", recorder.count(queue.EventJobSucceeded, terminalJobType), recorder.count(queue.EventBatchProgressed, terminalJobType), recorder.count(queue.EventBatchCompleted, terminalJobType)) + } + if recorder.count(queue.EventBatchCompleted, firstJobType) != 0 { + t.Fatal("earlier member was incorrectly credited with terminal batch completion") + } + var remainingID int64 + var remainingState string + var processingToken string + if err := queueDB.QueryRow(`SELECT id, state, processing_token FROM queue_jobs WHERE queue_name=?`, queueName).Scan(&remainingID, &remainingState, &processingToken); err != nil { + t.Fatalf("read terminal delivery before recovery: %v", err) + } + if remainingID != 2 || remainingState != "processing" || processingToken == "" { + t.Fatalf("remaining terminal delivery = id:%d state:%q token:%q, want id:2 state:processing with token", remainingID, remainingState, processingToken) + } + var receiptOwner, receiptJobID string + var receiptCompleted int + if err := workflowDB.QueryRow(`SELECT owner_delivery_id, job_id, aggregate_completed + FROM bus_workflow_transition_receipts + WHERE workflow_kind='batch' AND workflow_id=? AND member_id=''`, batchID).Scan(&receiptOwner, &receiptJobID, &receiptCompleted); err != nil { + t.Fatalf("read terminal batch aggregate receipt: %v", err) + } + if receiptOwner != processingToken || receiptJobID == "" || receiptCompleted != 1 { + t.Fatalf("terminal aggregate receipt = owner:%q job:%q completed:%d, want owner:%q with job and completion", receiptOwner, receiptJobID, receiptCompleted, processingToken) + } + var terminalMemberReceipts int + if err := workflowDB.QueryRow(`SELECT COUNT(*) FROM bus_workflow_transition_receipts + WHERE workflow_kind='batch' AND workflow_id=? AND member_id=? AND owner_delivery_id=? AND outcome='succeeded'`, batchID, receiptJobID, receiptOwner).Scan(&terminalMemberReceipts); err != nil { + t.Fatalf("read terminal batch member receipt: %v", err) + } + if terminalMemberReceipts != 1 { + t.Fatalf("terminal member receipts = %d, want 1", terminalMemberReceipts) + } + + if _, err := execSQLiteIntegrationEventually(queueDB, "DROP TRIGGER "+triggerName); err != nil { + t.Fatalf("drop terminal batch finalization trigger: %v", err) + } + if _, err := execSQLiteIntegrationEventually(queueDB, `UPDATE queue_jobs SET processing_started_at=1 WHERE id=? AND queue_name=? AND state='processing'`, remainingID, queueName); err != nil { + t.Fatalf("age terminal batch delivery for recovery: %v", err) + } + waitForObservabilityScenario(t, "sqlite_terminal_batch_owner_recovery", 5*time.Second, func() bool { + return recorder.count(queue.EventJobSucceeded, terminalJobType) == 1 && + recorder.count(queue.EventBatchProgressed, terminalJobType) == 1 && + recorder.count(queue.EventBatchCompleted, terminalJobType) == 1 + }) + if firstCalls.Load() != 1 || terminalCalls.Load() != 1 { + t.Fatalf("first/terminal handler calls after recovery = %d/%d, want 1/1", firstCalls.Load(), terminalCalls.Load()) + } + if recorder.count(queue.EventJobSucceeded, firstJobType) != 1 || recorder.count(queue.EventBatchProgressed, firstJobType) != 1 || recorder.count(queue.EventBatchCompleted, firstJobType) != 0 { + t.Fatalf("first-member facts after recovery = success/progress/completed %d/%d/%d, want 1/1/0", recorder.count(queue.EventJobSucceeded, firstJobType), recorder.count(queue.EventBatchProgressed, firstJobType), recorder.count(queue.EventBatchCompleted, firstJobType)) + } + firstProgressed, firstOK := recorder.first(queue.EventBatchProgressed, firstJobType) + terminalSucceeded, successOK := recorder.first(queue.EventJobSucceeded, terminalJobType) + terminalCompleted, completedOK := recorder.first(queue.EventBatchCompleted, terminalJobType) + if !firstOK || !successOK || !completedOK || firstProgressed.JobID == "" || firstProgressed.JobID == receiptJobID || terminalSucceeded.JobID != receiptJobID || terminalCompleted.JobID != receiptJobID || terminalCompleted.BatchID != batchID || terminalCompleted.DispatchID != state.DispatchID { + t.Fatalf("first progress/terminal success/terminal completion ownership = %+v present:%t / %+v present:%t / %+v present:%t", firstProgressed, firstOK, terminalSucceeded, successOK, terminalCompleted, completedOK) + } + var remaining int + if err := queueDB.QueryRow(`SELECT COUNT(*) FROM queue_jobs WHERE queue_name=?`, queueName).Scan(&remaining); err != nil { + t.Fatalf("count terminal batch deliveries after recovery: %v", err) + } + if remaining != 0 { + t.Fatalf("terminal batch deliveries after recovery = %d, want 0", remaining) + } + if recorder.count(queue.EventJobFailed, terminalJobType) != 0 || recorder.count(queue.EventBatchFailed, terminalJobType) != 0 || recorder.count(queue.EventBatchCancelled, terminalJobType) != 0 { + t.Fatal("terminal batch recovery published contradictory failure facts") + } +} + +// runSQLiteFailedChainSettlementRecovery proves a terminal failure receipt +// survives repeated archive faults without replaying its application occurrence. +func runSQLiteFailedChainSettlementRecovery(t *testing.T) { + t.Helper() + queueDSN := fmt.Sprintf("%s/queue-failed-chain-recovery-%d.db", t.TempDir(), time.Now().UnixNano()) + workflowDSN := fmt.Sprintf("%s/workflow-failed-chain-recovery-%d.db", t.TempDir(), time.Now().UnixNano()) + prepareSQLiteIntegrationSchema(t, queueDSN) + + workflowDB, err := sql.Open(testenv.BackendSQLite, workflowDSN) + if err != nil { + t.Fatalf("open failed chain workflow store: %v", err) + } + t.Cleanup(func() { _ = workflowDB.Close() }) + store, err := queue.NewSQLStore(queue.SQLStoreConfig{DB: workflowDB, DriverName: testenv.BackendSQLite, AutoMigrate: true}) + if err != nil { + t.Fatalf("new failed chain workflow store: %v", err) + } + + const ( + queueName = "failed-chain-recovery" + jobType = "job:db:failed-chain-recovery" + ) + recorder := &databaseSettlementRecorder{settlement: make(chan struct{})} + runtimeCfg := withDBRecoveryPolicy(withDefaultQueue(sqliteCfg(queueDSN), queueName), 10*time.Millisecond, 30*time.Second) + runtimeCfg.DisableAutoMigrate = true + runtime, err := testenv.NewQueue(runtimeCfg, queue.WithStore(store), queue.WithObserver(recorder), queue.WithWorkers(1)) + if err != nil { + t.Fatalf("new failed chain recovery runtime: %v", err) + } + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = runtime.Shutdown(shutdownCtx) + }) + + originalCause := errors.New("original failed chain application cause") + var handlerCalls atomic.Int64 + runtime.Register(jobType, func(context.Context, queue.Message) error { + handlerCalls.Add(1) + return queue.Permanent(originalCause) + }) + + queueDB, err := sql.Open(testenv.BackendSQLite, queueDSN) + if err != nil { + t.Fatalf("open failed chain queue database: %v", err) + } + defer queueDB.Close() + const triggerName = "reject_failed_chain_terminal_settlement" + trigger := fmt.Sprintf(`CREATE TRIGGER %s +BEFORE UPDATE OF state ON queue_jobs +WHEN OLD.queue_name = '%s' AND NEW.state = 'dead' +BEGIN + SELECT RAISE(ABORT, 'forced failed chain finalization failure'); +END`, triggerName, queueName) + if _, err := queueDB.Exec(trigger); err != nil { + t.Fatalf("create failed chain finalization trigger: %v", err) + } + triggerInstalled := true + defer func() { + if triggerInstalled { + _, _ = execSQLiteIntegrationEventually(queueDB, "DROP TRIGGER "+triggerName) + } + }() + if err := runtime.StartWorkers(context.Background()); err != nil { + t.Fatalf("start failed chain recovery runtime: %v", err) + } + chainID, err := runtime.Chain(queue.NewJob(jobType).OnQueue(queueName).Retry(3)).Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch failed chain recovery workflow: %v", err) + } + select { + case <-recorder.settlement: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for failed chain settlement failure") + } + + state, err := store.GetChain(context.Background(), chainID) + if err != nil || !state.Failed || state.Completed || state.Failure != originalCause.Error() { + t.Fatalf("failed chain before recovery = %+v err:%v", state, err) + } + if handlerCalls.Load() != 1 || recorder.count(queue.EventJobStarted, jobType) != 1 || recorder.count(queue.EventJobFailed, jobType) != 1 || recorder.count(queue.EventChainFailed, jobType) != 1 { + t.Fatalf("initial chain calls/started/job-failed/chain-failed = %d/%d/%d/%d, want 1/1/1/1", handlerCalls.Load(), recorder.count(queue.EventJobStarted, jobType), recorder.count(queue.EventJobFailed, jobType), recorder.count(queue.EventChainFailed, jobType)) + } + + var ( + rowID int64 + rowState string + rowAttempt int + processingToken string + ) + if err := queueDB.QueryRow(`SELECT id, state, attempt, processing_token FROM queue_jobs WHERE queue_name=?`, queueName).Scan(&rowID, &rowState, &rowAttempt, &processingToken); err != nil { + t.Fatalf("read failed chain delivery: %v", err) + } + if rowState != "processing" || rowAttempt != 0 || processingToken == "" { + t.Fatalf("failed chain delivery = id:%d state:%q attempt:%d token:%q, want processing attempt 0", rowID, rowState, rowAttempt, processingToken) + } + var ( + receiptOwner, receiptDispatch, receiptJobID, receiptOutcome string + receiptAttempt int + receiptCompleted, receiptCancelled int + ) + if err := workflowDB.QueryRow(`SELECT owner_delivery_id, owner_attempt, job_dispatch_id, job_id, outcome, aggregate_completed, aggregate_cancelled + FROM bus_workflow_transition_receipts WHERE workflow_kind='chain' AND workflow_id=?`, chainID).Scan( + &receiptOwner, &receiptAttempt, &receiptDispatch, &receiptJobID, &receiptOutcome, &receiptCompleted, &receiptCancelled, + ); err != nil { + t.Fatalf("read failed chain receipt: %v", err) + } + if receiptOwner != processingToken || receiptAttempt != 0 || receiptDispatch != state.DispatchID || receiptJobID == "" || receiptOutcome != "failed" || receiptCompleted != 0 || receiptCancelled != 0 { + t.Fatalf("failed chain receipt = owner:%q attempt:%d dispatch:%q job:%q outcome:%q completed:%d cancelled:%d", receiptOwner, receiptAttempt, receiptDispatch, receiptJobID, receiptOutcome, receiptCompleted, receiptCancelled) + } + + if _, err := execSQLiteIntegrationEventually(queueDB, `UPDATE queue_jobs SET processing_started_at=1 WHERE id=? AND state='processing'`, rowID); err != nil { + t.Fatalf("age failed chain delivery: %v", err) + } + waitForObservabilityScenario(t, "sqlite_repeated_failed_chain_settlement_failures", 5*time.Second, func() bool { + return recorder.count(queue.EventSettlementFailed, jobType) >= 3 + }) + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + if err := runtime.Shutdown(shutdownCtx); err != nil { + cancel() + t.Fatalf("shutdown failed-chain fault runtime: %v", err) + } + cancel() + + var ( + repairedState string + repairedAttempt int + repairedToken string + repairedStarted sql.NullInt64 + ) + if err := queueDB.QueryRow(`SELECT state, attempt, processing_token, processing_started_at FROM queue_jobs WHERE id=?`, rowID).Scan(&repairedState, &repairedAttempt, &repairedToken, &repairedStarted); err != nil { + t.Fatalf("read repaired failed chain delivery: %v", err) + } + if repairedState != "pending" || repairedAttempt != 0 || repairedToken != receiptOwner || repairedStarted.Valid { + t.Fatalf("repaired failed chain delivery = state:%q attempt:%d token:%q started:%#v, want pending/0/%q/NULL", repairedState, repairedAttempt, repairedToken, repairedStarted, receiptOwner) + } + state, err = store.GetChain(context.Background(), chainID) + if err != nil || state.Failure != originalCause.Error() { + t.Fatalf("failed chain cause after repeated recovery = %+v err:%v", state, err) + } + if handlerCalls.Load() != 1 || recorder.count(queue.EventJobStarted, jobType) != 1 || recorder.count(queue.EventJobFailed, jobType) != 1 || recorder.count(queue.EventChainFailed, jobType) != 1 { + t.Fatalf("pre-archive chain calls/started/job-failed/chain-failed = %d/%d/%d/%d, want 1/1/1/1", handlerCalls.Load(), recorder.count(queue.EventJobStarted, jobType), recorder.count(queue.EventJobFailed, jobType), recorder.count(queue.EventChainFailed, jobType)) + } + if _, err := execSQLiteIntegrationEventually(queueDB, "DROP TRIGGER "+triggerName); err != nil { + t.Fatalf("drop failed chain finalization trigger: %v", err) + } + triggerInstalled = false + + finalRuntime, err := testenv.NewQueue(runtimeCfg, queue.WithStore(store), queue.WithObserver(recorder), queue.WithWorkers(1)) + if err != nil { + t.Fatalf("new final failed chain recovery runtime: %v", err) + } + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = finalRuntime.Shutdown(shutdownCtx) + }) + finalRuntime.Register(jobType, func(context.Context, queue.Message) error { + handlerCalls.Add(1) + return queue.Permanent(errors.New("failed-chain receipt recovery re-executed application code")) + }) + if err := finalRuntime.StartWorkers(context.Background()); err != nil { + t.Fatalf("start final failed chain recovery runtime: %v", err) + } + waitForObservabilityScenario(t, "sqlite_failed_chain_terminal_settlement_recovery", 5*time.Second, func() bool { + var archived string + return queueDB.QueryRow(`SELECT state FROM queue_jobs WHERE id=?`, rowID).Scan(&archived) == nil && archived == "dead" + }) + if handlerCalls.Load() != 1 || recorder.count(queue.EventJobStarted, jobType) != 1 || recorder.count(queue.EventJobFailed, jobType) != 1 || recorder.count(queue.EventChainFailed, jobType) != 1 { + t.Fatalf("archived chain calls/started/job-failed/chain-failed = %d/%d/%d/%d, want 1/1/1/1", handlerCalls.Load(), recorder.count(queue.EventJobStarted, jobType), recorder.count(queue.EventJobFailed, jobType), recorder.count(queue.EventChainFailed, jobType)) + } + var ( + archivedState string + archivedAttempt int + archivedToken sql.NullString + archivedError sql.NullString + ) + if err := queueDB.QueryRow(`SELECT state, attempt, processing_token, last_error FROM queue_jobs WHERE id=?`, rowID).Scan(&archivedState, &archivedAttempt, &archivedToken, &archivedError); err != nil { + t.Fatalf("read archived failed chain delivery: %v", err) + } + if archivedState != "dead" || archivedAttempt != 1 || archivedToken.Valid || !archivedError.Valid || archivedError.String != originalCause.Error() { + t.Fatalf("archived failed chain delivery = state:%q attempt:%d token:%#v error:%q, want dead attempt 1 with persisted cause", archivedState, archivedAttempt, archivedToken, archivedError.String) + } +} + +// runSQLiteFailedBatchSettlementRecovery proves a receipt-backed failed member +// remains archived after finalization recovery without executing its handler or +// fabricating the application cause omitted from durable receipt state. +func runSQLiteFailedBatchSettlementRecovery(t *testing.T) { + t.Helper() + queueDSN := fmt.Sprintf("%s/queue-failed-batch-recovery-%d.db", t.TempDir(), time.Now().UnixNano()) + workflowDSN := fmt.Sprintf("%s/workflow-failed-batch-recovery-%d.db", t.TempDir(), time.Now().UnixNano()) + prepareSQLiteIntegrationSchema(t, queueDSN) + + workflowDB, err := sql.Open(testenv.BackendSQLite, workflowDSN) + if err != nil { + t.Fatalf("open failed batch workflow store: %v", err) + } + t.Cleanup(func() { _ = workflowDB.Close() }) + store, err := queue.NewSQLStore(queue.SQLStoreConfig{ + DB: workflowDB, + DriverName: testenv.BackendSQLite, + AutoMigrate: true, + }) + if err != nil { + t.Fatalf("new failed batch workflow store: %v", err) + } + + const ( + queueName = "failed-batch-recovery" + firstJobType = "job:db:failed-batch-recovery:first" + failedJobType = "job:db:failed-batch-recovery:failed" + ) + recorder := &databaseSettlementRecorder{settlement: make(chan struct{})} + runtimeCfg := withDBRecoveryPolicy(withDefaultQueue(sqliteCfg(queueDSN), queueName), 10*time.Millisecond, 30*time.Second) + runtimeCfg.DisableAutoMigrate = true + runtime, err := testenv.NewQueue(runtimeCfg, queue.WithStore(store), queue.WithObserver(recorder), queue.WithWorkers(1)) + if err != nil { + t.Fatalf("new failed batch recovery runtime: %v", err) + } + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = runtime.Shutdown(shutdownCtx) + }) + + originalCause := errors.New("original failed batch application cause") + var firstCalls, failedCalls atomic.Int64 + runtime.Register(firstJobType, func(context.Context, queue.Message) error { + firstCalls.Add(1) + return nil + }) + runtime.Register(failedJobType, func(context.Context, queue.Message) error { + failedCalls.Add(1) + return queue.Permanent(originalCause) + }) + + queueDB, err := sql.Open(testenv.BackendSQLite, queueDSN) + if err != nil { + t.Fatalf("open failed batch queue database: %v", err) + } + defer queueDB.Close() + const triggerName = "reject_failed_batch_terminal_settlement" + trigger := fmt.Sprintf(`CREATE TRIGGER %s +BEFORE UPDATE OF state ON queue_jobs +WHEN OLD.queue_name = '%s' AND NEW.state = 'dead' +BEGIN + SELECT RAISE(ABORT, 'forced failed batch finalization failure'); +END`, triggerName, queueName) + if _, err := queueDB.Exec(trigger); err != nil { + t.Fatalf("create failed batch finalization trigger: %v", err) + } + triggerInstalled := true + defer func() { + if triggerInstalled { + _, _ = execSQLiteIntegrationEventually(queueDB, "DROP TRIGGER "+triggerName) + } + }() + if err := runtime.StartWorkers(context.Background()); err != nil { + t.Fatalf("start failed batch recovery runtime: %v", err) + } + + batchID, err := runtime.Batch( + queue.NewJob(firstJobType).OnQueue(queueName), + queue.NewJob(failedJobType).OnQueue(queueName).Retry(3), + ).AllowFailures().OnQueue(queueName).Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch failed batch recovery workflow: %v", err) + } + select { + case <-recorder.settlement: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for failed batch settlement failure") + } + + state, err := store.GetBatch(context.Background(), batchID) + if err != nil { + t.Fatalf("get committed failed batch: %v", err) + } + if !state.Completed || state.Cancelled || state.Failed != 1 || state.Processed != 2 || state.Pending != 0 || !state.AllowFailed { + t.Fatalf("failed batch state before recovery = %+v", state) + } + if firstCalls.Load() != 1 || failedCalls.Load() != 1 { + t.Fatalf("first/failed handler calls before recovery = %d/%d, want 1/1", firstCalls.Load(), failedCalls.Load()) + } + if recorder.count(queue.EventJobSucceeded, firstJobType) != 1 || recorder.count(queue.EventBatchProgressed, firstJobType) != 1 { + t.Fatalf("first-member success/progress facts = %d/%d, want 1/1", recorder.count(queue.EventJobSucceeded, firstJobType), recorder.count(queue.EventBatchProgressed, firstJobType)) + } + if recorder.count(queue.EventJobFailed, failedJobType) != 1 || recorder.count(queue.EventBatchProgressed, failedJobType) != 0 || recorder.count(queue.EventBatchCompleted, failedJobType) != 0 { + t.Fatalf("failed-member facts before recovery = failed/progress/completed %d/%d/%d, want 1/0/0", recorder.count(queue.EventJobFailed, failedJobType), recorder.count(queue.EventBatchProgressed, failedJobType), recorder.count(queue.EventBatchCompleted, failedJobType)) + } + + var ( + rowID int64 + rowState string + rowAttempt int + processingToken string + ) + if err := queueDB.QueryRow(`SELECT id, state, attempt, processing_token FROM queue_jobs WHERE queue_name=?`, queueName).Scan(&rowID, &rowState, &rowAttempt, &processingToken); err != nil { + t.Fatalf("read failed delivery before recovery: %v", err) + } + if rowState != "processing" || rowAttempt != 0 || processingToken == "" { + t.Fatalf("failed delivery before recovery = id:%d state:%q attempt:%d token:%q, want retained processing attempt 0", rowID, rowState, rowAttempt, processingToken) + } + var receiptOwner, receiptJobID, receiptOutcome string + var receiptCompleted int + if err := workflowDB.QueryRow(`SELECT owner_delivery_id, job_id, outcome, aggregate_completed + FROM bus_workflow_transition_receipts + WHERE workflow_kind='batch' AND workflow_id=? AND member_id=''`, batchID).Scan(&receiptOwner, &receiptJobID, &receiptOutcome, &receiptCompleted); err != nil { + t.Fatalf("read failed batch aggregate receipt: %v", err) + } + if receiptOwner != processingToken || receiptJobID == "" || receiptOutcome != "failed" || receiptCompleted != 1 { + t.Fatalf("failed aggregate receipt = owner:%q job:%q outcome:%q completed:%d, want owner:%q failed completion", receiptOwner, receiptJobID, receiptOutcome, receiptCompleted, processingToken) + } + + if _, err := execSQLiteIntegrationEventually(queueDB, `UPDATE queue_jobs SET processing_started_at=1 WHERE id=? AND state='processing'`, rowID); err != nil { + t.Fatalf("age failed batch delivery for recovery: %v", err) + } + waitForObservabilityScenario(t, "sqlite_repeated_failed_batch_settlement_failures", 5*time.Second, func() bool { + return recorder.count(queue.EventSettlementFailed, failedJobType) >= 3 + }) + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + if err := runtime.Shutdown(shutdownCtx); err != nil { + cancel() + t.Fatalf("shutdown repeated failed-batch fault runtime: %v", err) + } + cancel() + var ( + repairedState string + repairedAttempt int + repairedToken string + repairedStarted sql.NullInt64 + ) + if err := queueDB.QueryRow(`SELECT state, attempt, processing_token, processing_started_at FROM queue_jobs WHERE id=?`, rowID).Scan(&repairedState, &repairedAttempt, &repairedToken, &repairedStarted); err != nil { + t.Fatalf("read repeatedly repaired failed delivery: %v", err) + } + if repairedState != "pending" || repairedAttempt != 0 || repairedToken != receiptOwner || repairedStarted.Valid { + t.Fatalf("repaired failed delivery = state:%q attempt:%d token:%q started:%#v, want pending/0/%q/NULL", repairedState, repairedAttempt, repairedToken, repairedStarted, receiptOwner) + } + if firstCalls.Load() != 1 || failedCalls.Load() != 1 || recorder.count(queue.EventBatchCompleted, failedJobType) != 0 { + t.Fatalf("pre-archive calls/completion = %d/%d/%d, want 1/1/0", firstCalls.Load(), failedCalls.Load(), recorder.count(queue.EventBatchCompleted, failedJobType)) + } + if _, err := execSQLiteIntegrationEventually(queueDB, "DROP TRIGGER "+triggerName); err != nil { + t.Fatalf("drop failed batch finalization trigger: %v", err) + } + triggerInstalled = false + + finalRuntime, err := testenv.NewQueue(runtimeCfg, queue.WithStore(store), queue.WithObserver(recorder), queue.WithWorkers(1)) + if err != nil { + t.Fatalf("new final failed-batch recovery runtime: %v", err) + } + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = finalRuntime.Shutdown(shutdownCtx) + }) + finalRuntime.Register(firstJobType, func(context.Context, queue.Message) error { + firstCalls.Add(1) + return queue.Permanent(errors.New("failed-batch recovery re-executed the first member")) + }) + finalRuntime.Register(failedJobType, func(context.Context, queue.Message) error { + failedCalls.Add(1) + return queue.Permanent(errors.New("failed-batch receipt recovery re-executed application code")) + }) + if err := finalRuntime.StartWorkers(context.Background()); err != nil { + t.Fatalf("start final failed-batch recovery runtime: %v", err) + } + waitForObservabilityScenario(t, "sqlite_failed_batch_terminal_settlement_recovery", 5*time.Second, func() bool { + var state string + stateErr := queueDB.QueryRow(`SELECT state FROM queue_jobs WHERE id=?`, rowID).Scan(&state) + return stateErr == nil && state == "dead" && recorder.count(queue.EventBatchCompleted, failedJobType) == 1 + }) + if firstCalls.Load() != 1 || failedCalls.Load() != 1 { + t.Fatalf("first/failed handler calls after recovery = %d/%d, want 1/1", firstCalls.Load(), failedCalls.Load()) + } + var ( + archivedState string + archivedAttempt int + archivedToken sql.NullString + archivedError sql.NullString + ) + if err := queueDB.QueryRow(`SELECT state, attempt, processing_token, last_error FROM queue_jobs WHERE id=?`, rowID).Scan(&archivedState, &archivedAttempt, &archivedToken, &archivedError); err != nil { + t.Fatalf("read archived failed delivery: %v", err) + } + if archivedState != "dead" || archivedAttempt != 1 || archivedToken.Valid || !archivedError.Valid || !strings.Contains(archivedError.String, "original cause was not persisted") || strings.Contains(archivedError.String, originalCause.Error()) { + t.Fatalf("archived failed delivery = state:%q attempt:%d token:%#v error:%q, want dead attempt 1 with generic recovered cause", archivedState, archivedAttempt, archivedToken, archivedError.String) + } + if recorder.count(queue.EventJobFailed, failedJobType) != 1 || recorder.count(queue.EventBatchProgressed, failedJobType) != 0 || recorder.count(queue.EventBatchCompleted, failedJobType) != 1 { + t.Fatalf("failed-member facts after recovery = failed/progress/completed %d/%d/%d, want 1/0/1", recorder.count(queue.EventJobFailed, failedJobType), recorder.count(queue.EventBatchProgressed, failedJobType), recorder.count(queue.EventBatchCompleted, failedJobType)) + } + completed, completedOK := recorder.first(queue.EventBatchCompleted, failedJobType) + if !completedOK || completed.BatchID != batchID || completed.JobID != receiptJobID || completed.DispatchID != state.DispatchID { + t.Fatalf("failed terminal completion = %+v present:%t, want batch:%q job:%q dispatch:%q", completed, completedOK, batchID, receiptJobID, state.DispatchID) + } +} + +type sqliteWorkflowHandlerObservation struct { + call int64 + attempt int + provenance busruntime.DeliveryProvenance + present bool +} + +// runSQLiteLaterWorkflowWinnerRecovery proves recovery follows the generation +// that actually committed the workflow transition after an earlier generation +// was reclaimed and retried. +func runSQLiteLaterWorkflowWinnerRecovery(t *testing.T) { + t.Helper() + queueDSN := fmt.Sprintf("%s/queue-later-workflow-winner-%d.db", t.TempDir(), time.Now().UnixNano()) + workflowDSN := fmt.Sprintf("%s/later-workflow-winner-%d.db", t.TempDir(), time.Now().UnixNano()) + prepareSQLiteIntegrationSchema(t, queueDSN) + + workflowDB, err := sql.Open(testenv.BackendSQLite, workflowDSN) + if err != nil { + t.Fatalf("open later-winner workflow store: %v", err) + } + t.Cleanup(func() { _ = workflowDB.Close() }) + store, err := queue.NewSQLStore(queue.SQLStoreConfig{ + DB: workflowDB, + DriverName: testenv.BackendSQLite, + AutoMigrate: true, + }) + if err != nil { + t.Fatalf("new later-winner workflow store: %v", err) + } + + const ( + queueName = "later-workflow-winner" + jobType = "job:db:later-workflow-winner" + ) + recorder := &databaseSettlementRecorder{settlement: make(chan struct{})} + runtimeCfg := withDBRecoveryPolicy(withDefaultQueue(sqliteCfg(queueDSN), queueName), 10*time.Millisecond, 30*time.Second) + runtimeCfg.DisableAutoMigrate = true + firstRuntime, err := testenv.NewQueue(runtimeCfg, queue.WithStore(store), queue.WithObserver(recorder), queue.WithWorkers(1)) + if err != nil { + t.Fatalf("new first later-winner recovery runtime: %v", err) + } + secondRuntime, err := testenv.NewQueue(runtimeCfg, queue.WithStore(store), queue.WithObserver(recorder), queue.WithWorkers(1)) + if err != nil { + t.Fatalf("new second later-winner recovery runtime: %v", err) + } + + firstStarted := make(chan sqliteWorkflowHandlerObservation, 1) + recoveredAttemptZero := make(chan sqliteWorkflowHandlerObservation, 1) + attemptOneWinner := make(chan sqliteWorkflowHandlerObservation, 1) + firstReturned := make(chan struct{}) + releaseFirst := make(chan struct{}) + unexpectedCall := make(chan sqliteWorkflowHandlerObservation, 1) + var ( + handlerCalls atomic.Int64 + releaseFirstOnce sync.Once + ) + t.Cleanup(func() { + releaseFirstOnce.Do(func() { close(releaseFirst) }) + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = firstRuntime.Shutdown(shutdownCtx) + _ = secondRuntime.Shutdown(shutdownCtx) + }) + + handler := func(ctx context.Context, message queue.Message) error { + provenance, present := busruntime.DeliveryProvenanceFromContext(ctx) + observation := sqliteWorkflowHandlerObservation{ + call: handlerCalls.Add(1), + attempt: message.Attempt, + provenance: provenance, + present: present, + } + switch observation.call { + case 1: + firstStarted <- observation + <-releaseFirst + close(firstReturned) + return nil + case 2: + recoveredAttemptZero <- observation + return errors.New("transient error from reclaimed attempt zero") + case 3: + attemptOneWinner <- observation + return nil + default: + select { + case unexpectedCall <- observation: + default: + } + return queue.Permanent(errors.New("receipt recovery re-executed application code")) + } + } + firstRuntime.Register(jobType, handler) + secondRuntime.Register(jobType, handler) + + queueDB, err := sql.Open(testenv.BackendSQLite, queueDSN) + if err != nil { + t.Fatalf("open later-winner queue database: %v", err) + } + defer queueDB.Close() + const triggerName = "reject_later_workflow_winner_finalization" + const trigger = `CREATE TRIGGER reject_later_workflow_winner_finalization +BEFORE DELETE ON queue_jobs +WHEN OLD.queue_name = 'later-workflow-winner' +BEGIN + SELECT RAISE(ABORT, 'forced later-winner finalization failure'); +END` + if _, err := queueDB.Exec(trigger); err != nil { + t.Fatalf("create later-winner finalization trigger: %v", err) + } + if err := firstRuntime.StartWorkers(context.Background()); err != nil { + t.Fatalf("start first later-winner recovery runtime: %v", err) + } + if err := secondRuntime.StartWorkers(context.Background()); err != nil { + t.Fatalf("start second later-winner recovery runtime: %v", err) + } + workflowID, err := firstRuntime.Chain(queue.NewJob(jobType).OnQueue(queueName).Retry(1)).Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch later-winner chain: %v", err) + } + + var initial sqliteWorkflowHandlerObservation + select { + case initial = <-firstStarted: + case <-time.After(5 * time.Second): + t.Fatal("initial workflow generation did not start") + } + if !initial.present || initial.attempt != 0 || initial.provenance.GenerationID == "" || initial.provenance.Recovered || initial.provenance.RecoveredGenerationID != "" { + t.Fatalf("initial generation observation = %+v, want ordinary attempt-zero provenance", initial) + } + result, err := execSQLiteIntegrationEventually(queueDB, `UPDATE queue_jobs SET processing_started_at=1 WHERE queue_name=? AND state='processing' AND attempt=0`, queueName) + if err != nil { + t.Fatalf("age initial workflow generation: %v", err) + } + if rows, rowsErr := result.RowsAffected(); rowsErr != nil || rows != 1 { + t.Fatalf("aged initial workflow rows = %d, error %v; want 1", rows, rowsErr) + } + + var reclaimed sqliteWorkflowHandlerObservation + select { + case reclaimed = <-recoveredAttemptZero: + case observation := <-unexpectedCall: + t.Fatalf("unexpected workflow handler call before attempt-zero recovery: %+v", observation) + case <-time.After(5 * time.Second): + t.Fatal("stale attempt zero was not reclaimed") + } + if !reclaimed.present || reclaimed.attempt != 0 || !reclaimed.provenance.Recovered || reclaimed.provenance.GenerationID == "" || reclaimed.provenance.GenerationID == initial.provenance.GenerationID || reclaimed.provenance.RecoveredGenerationID != initial.provenance.GenerationID { + t.Fatalf("reclaimed generation observation = %+v, initial = %+v", reclaimed, initial) + } + + var winner sqliteWorkflowHandlerObservation + select { + case winner = <-attemptOneWinner: + case observation := <-unexpectedCall: + t.Fatalf("unexpected workflow handler call before attempt-one winner: %+v", observation) + case <-time.After(5 * time.Second): + t.Fatal("application retry did not reach attempt-one winner") + } + if !winner.present || winner.attempt != 1 || winner.provenance.Recovered || winner.provenance.RecoveredGenerationID != "" || winner.provenance.GenerationID == "" || winner.provenance.GenerationID == reclaimed.provenance.GenerationID { + t.Fatalf("attempt-one winner observation = %+v, reclaimed = %+v", winner, reclaimed) + } + select { + case <-recorder.settlement: + case observation := <-unexpectedCall: + t.Fatalf("unexpected workflow handler call before winner settlement failure: %+v", observation) + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for attempt-one finalization failure") + } + if handlerCalls.Load() != 3 { + t.Fatalf("handler calls before winner recovery = %d, want 3", handlerCalls.Load()) + } + state, err := store.GetChain(context.Background(), workflowID) + if err != nil { + t.Fatalf("get later-winner chain before recovery: %v", err) + } + if !state.Completed || state.Failed { + t.Fatalf("later-winner chain before recovery = %+v, want completed success", state) + } + if recorder.count(queue.EventJobSucceeded, jobType) != 0 || recorder.count(queue.EventChainCompleted, jobType) != 0 { + t.Fatal("winner facts published before durable queue finalization") + } + var ( + queueState string + processingToken sql.NullString + attempt int + ) + if err := queueDB.QueryRow(`SELECT state, processing_token, attempt FROM queue_jobs WHERE queue_name=?`, queueName).Scan(&queueState, &processingToken, &attempt); err != nil { + t.Fatalf("query attempt-one winner row: %v", err) + } + if queueState != "processing" || !processingToken.Valid || processingToken.String != winner.provenance.GenerationID || attempt != 1 { + t.Fatalf("attempt-one winner row = state:%q token:%q valid:%t attempt:%d, want processing generation %q at attempt 1", queueState, processingToken.String, processingToken.Valid, attempt, winner.provenance.GenerationID) + } + + if _, err := execSQLiteIntegrationEventually(queueDB, "DROP TRIGGER "+triggerName); err != nil { + t.Fatalf("drop later-winner finalization trigger: %v", err) + } + result, err = execSQLiteIntegrationEventually(queueDB, `UPDATE queue_jobs SET processing_started_at=1 WHERE queue_name=? AND state='processing' AND attempt=1`, queueName) + if err != nil { + t.Fatalf("age attempt-one winner for recovery: %v", err) + } + if rows, rowsErr := result.RowsAffected(); rowsErr != nil || rows != 1 { + t.Fatalf("aged attempt-one winner rows = %d, error %v; want 1", rows, rowsErr) + } + waitForObservabilityScenario(t, "sqlite_later_workflow_winner_receipt_recovery", 5*time.Second, func() bool { + return recorder.count(queue.EventJobSucceeded, jobType) == 1 && recorder.count(queue.EventChainCompleted, jobType) == 1 + }) + if handlerCalls.Load() != 3 { + t.Fatalf("handler calls after receipt-backed winner recovery = %d, want 3", handlerCalls.Load()) + } + select { + case observation := <-unexpectedCall: + t.Fatalf("receipt-backed winner recovery executed application code: %+v", observation) + default: + } + succeeded, ok := recorder.first(queue.EventJobSucceeded, jobType) + if !ok || succeeded.Attempt != 1 || succeeded.EventID == "" { + t.Fatalf("recovered attempt-one success = %+v present:%t", succeeded, ok) + } + var remaining int + if err := queueDB.QueryRow(`SELECT COUNT(*) FROM queue_jobs WHERE queue_name=?`, queueName).Scan(&remaining); err != nil { + t.Fatalf("count recovered later-winner rows: %v", err) + } + if remaining != 0 { + t.Fatalf("recovered later-winner rows = %d, want 0", remaining) + } + if recorder.count(queue.EventJobFailed, jobType) != 0 || recorder.count(queue.EventChainFailed, jobType) != 0 { + t.Fatal("later-winner recovery published contradictory failure facts") + } + + releaseFirstOnce.Do(func() { close(releaseFirst) }) + select { + case <-firstReturned: + case <-time.After(5 * time.Second): + t.Fatal("superseded initial handler did not return") + } + waitForObservabilityScenario(t, "sqlite_superseded_initial_workflow_settlement", 5*time.Second, func() bool { + return recorder.count(queue.EventSettlementFailed, jobType) == 2 + }) + if handlerCalls.Load() != 3 || recorder.count(queue.EventJobSucceeded, jobType) != 1 || recorder.count(queue.EventChainCompleted, jobType) != 1 || recorder.count(queue.EventJobFailed, jobType) != 0 || recorder.count(queue.EventChainFailed, jobType) != 0 { + t.Fatalf("facts changed after superseded generation returned: calls:%d succeeded:%d completed:%d job_failed:%d chain_failed:%d", handlerCalls.Load(), recorder.count(queue.EventJobSucceeded, jobType), recorder.count(queue.EventChainCompleted, jobType), recorder.count(queue.EventJobFailed, jobType), recorder.count(queue.EventChainFailed, jobType)) + } +} + +// installDatabaseFinalizationFailure installs a queue-scoped delete fault and +// returns an idempotent cleanup closure for the selected SQL dialect. +func installDatabaseFinalizationFailure(t *testing.T, backend string, db *sql.DB, queueName string) func() error { + t.Helper() + suffix := fmt.Sprintf("%d", time.Now().UnixNano()) + triggerName := "queue_receipt_delete_" + suffix + functionName := "queue_receipt_delete_fn_" + suffix + switch backend { + case testenv.BackendMySQL: + blockerTable := "queue_receipt_block_" + suffix + constraintName := blockerTable + "_fk" + statement := fmt.Sprintf(`CREATE TABLE %s ( + queue_job_id BIGINT NOT NULL PRIMARY KEY, + CONSTRAINT %s FOREIGN KEY (queue_job_id) REFERENCES queue_jobs(id) +) ENGINE=InnoDB`, blockerTable, constraintName) + if _, err := db.Exec(statement); err != nil { + t.Fatalf("create MySQL workflow receipt blocker: %v", err) + } + result, err := db.Exec(fmt.Sprintf(`INSERT INTO %s (queue_job_id) +SELECT id FROM queue_jobs WHERE queue_name=?`, blockerTable), queueName) + if err != nil { + _, _ = db.Exec("DROP TABLE IF EXISTS " + blockerTable) + t.Fatalf("attach MySQL workflow receipt blocker: %v", err) + } + if rows, rowsErr := result.RowsAffected(); rowsErr != nil || rows != 1 { + _, _ = db.Exec("DROP TABLE IF EXISTS " + blockerTable) + t.Fatalf("attached MySQL workflow receipt blockers = %d, error %v; want 1", rows, rowsErr) + } + return func() error { + _, err := db.Exec("DROP TABLE IF EXISTS " + blockerTable) + return err + } + case testenv.BackendPostgres: + function := fmt.Sprintf(`CREATE FUNCTION %s() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + IF OLD.queue_name = '%s' THEN + RAISE EXCEPTION 'forced workflow receipt finalization failure'; + END IF; + RETURN OLD; +END; +$$`, functionName, queueName) + if _, err := db.Exec(function); err != nil { + t.Fatalf("create PostgreSQL workflow receipt trigger function: %v", err) + } + trigger := fmt.Sprintf(`CREATE TRIGGER %s BEFORE DELETE ON queue_jobs FOR EACH ROW EXECUTE FUNCTION %s()`, triggerName, functionName) + if _, err := db.Exec(trigger); err != nil { + _, _ = db.Exec("DROP FUNCTION IF EXISTS " + functionName + "()") + t.Fatalf("create PostgreSQL workflow receipt trigger: %v", err) + } + return func() error { + if _, err := db.Exec("DROP TRIGGER IF EXISTS " + triggerName + " ON queue_jobs"); err != nil { + return err + } + _, err := db.Exec("DROP FUNCTION IF EXISTS " + functionName + "()") + return err + } + default: + t.Fatalf("unsupported workflow receipt fault backend %q", backend) + return func() error { return nil } + } +} + +// runDatabaseWorkflowReceiptRecovery proves MySQL and PostgreSQL preserve the +// same generation-to-transition receipt contract already exercised on SQLite. +func runDatabaseWorkflowReceiptRecovery[T any](t *testing.T, backend, driverName, dsn string, runtimeCfg T) { + t.Helper() + db, err := sql.Open(driverName, dsn) + if err != nil { + t.Fatalf("open %s workflow receipt database: %v", backend, err) + } + t.Cleanup(func() { _ = db.Close() }) + store, err := queue.NewSQLStore(queue.SQLStoreConfig{DB: db, DriverName: driverName, AutoMigrate: true}) + if err != nil { + t.Fatalf("new %s workflow receipt store: %v", backend, err) + } + + queueName := fmt.Sprintf("workflow_receipt_%s_%d", backend, time.Now().UnixNano()) + jobType := "job:db:workflow-receipt:" + backend + recorder := &databaseSettlementRecorder{settlement: make(chan struct{})} + runtimeCfg = withDefaultQueue(withDBRecoveryPolicy(runtimeCfg, 10*time.Millisecond, 30*time.Second), queueName) + runtime, err := testenv.NewQueue(runtimeCfg, queue.WithStore(store), queue.WithObserver(recorder), queue.WithWorkers(1)) + if err != nil { + t.Fatalf("new %s workflow receipt runtime: %v", backend, err) + } + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = runtime.Shutdown(shutdownCtx) + }) + + var ( + handlerCalls atomic.Int64 + handlerStartedOnce sync.Once + releaseHandlerOnce sync.Once + ) + handlerStarted := make(chan struct{}) + releaseHandler := make(chan struct{}) + t.Cleanup(func() { releaseHandlerOnce.Do(func() { close(releaseHandler) }) }) + runtime.Register(jobType, func(context.Context, queue.Message) error { + handlerCalls.Add(1) + handlerStartedOnce.Do(func() { close(handlerStarted) }) + <-releaseHandler + return nil + }) + if err := runtime.StartWorkers(context.Background()); err != nil { + t.Fatalf("start %s workflow receipt runtime: %v", backend, err) + } + workflowID, err := runtime.Chain(queue.NewJob(jobType).OnQueue(queueName)).Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch %s workflow receipt chain: %v", backend, err) + } + select { + case <-handlerStarted: + case <-time.After(10 * time.Second): + t.Fatalf("timed out waiting for %s workflow handler before installing finalization fault", backend) + } + dropFault := installDatabaseFinalizationFailure(t, backend, db, queueName) + t.Cleanup(func() { _ = dropFault() }) + releaseHandlerOnce.Do(func() { close(releaseHandler) }) + select { + case <-recorder.settlement: + case <-time.After(10 * time.Second): + t.Fatalf("timed out waiting for %s workflow finalization failure", backend) + } + state, err := store.GetChain(context.Background(), workflowID) + if err != nil || !state.Completed || state.Failed { + t.Fatalf("%s committed workflow state = %+v, err:%v", backend, state, err) + } + if handlerCalls.Load() != 1 || recorder.count(queue.EventJobSucceeded, jobType) != 0 || recorder.count(queue.EventChainCompleted, jobType) != 0 { + t.Fatalf("%s pre-recovery calls/success/completion = %d/%d/%d, want 1/0/0", backend, handlerCalls.Load(), recorder.count(queue.EventJobSucceeded, jobType), recorder.count(queue.EventChainCompleted, jobType)) + } + if err := dropFault(); err != nil { + t.Fatalf("drop %s workflow receipt fault: %v", backend, err) + } + + placeholder := "?" + if backend == testenv.BackendPostgres { + placeholder = "$1" + } + ageQuery := `UPDATE queue_jobs SET processing_started_at=1 WHERE queue_name=` + placeholder + ` AND state='processing'` + result, err := db.Exec(ageQuery, queueName) + if err != nil { + t.Fatalf("age %s workflow receipt row: %v", backend, err) + } + if rows, rowsErr := result.RowsAffected(); rowsErr != nil || rows != 1 { + t.Fatalf("aged %s workflow rows = %d, error %v; want 1", backend, rows, rowsErr) + } + deadline := time.Now().Add(10 * time.Second) + recovered := false + for time.Now().Before(deadline) { + if recorder.count(queue.EventJobSucceeded, jobType) == 1 && recorder.count(queue.EventChainCompleted, jobType) == 1 { + recovered = true + break + } + time.Sleep(20 * time.Millisecond) + } + if !recovered { + var ( + queueState string + processingToken sql.NullString + attempt int + receiptCount int + receiptOwner sql.NullString + ) + rowQuery := `SELECT state, processing_token, attempt FROM queue_jobs WHERE queue_name=` + placeholder + rowErr := db.QueryRow(rowQuery, queueName).Scan(&queueState, &processingToken, &attempt) + receiptQuery := `SELECT COUNT(*), MAX(owner_delivery_id) FROM bus_workflow_transition_receipts WHERE workflow_kind='chain' AND workflow_id=` + placeholder + receiptErr := db.QueryRow(receiptQuery, workflowID).Scan(&receiptCount, &receiptOwner) + recorder.mu.Lock() + events := append([]queue.Event(nil), recorder.events...) + recorder.mu.Unlock() + t.Fatalf("%s receipt recovery timed out: handler_calls=%d queue_state=%q processing_token=%q attempt=%d row_error=%v receipt_count=%d receipt_owner=%q receipt_error=%v events=%+v", + backend, handlerCalls.Load(), queueState, processingToken.String, attempt, rowErr, receiptCount, receiptOwner.String, receiptErr, events) + } + if handlerCalls.Load() != 1 { + t.Fatalf("%s receipt recovery handler calls = %d, want 1", backend, handlerCalls.Load()) + } + succeeded, ok := recorder.first(queue.EventJobSucceeded, jobType) + if !ok || succeeded.Attempt != 0 || succeeded.EventID == "" { + t.Fatalf("%s recovered success = %+v present:%t", backend, succeeded, ok) + } + var remaining int + countQuery := `SELECT COUNT(*) FROM queue_jobs WHERE queue_name=` + placeholder + if err := db.QueryRow(countQuery, queueName).Scan(&remaining); err != nil { + t.Fatalf("count %s recovered workflow rows: %v", backend, err) + } + if remaining != 0 { + t.Fatalf("%s recovered workflow rows = %d, want 0", backend, remaining) + } +} + +// runDatabaseConcurrentBatchReceiptOwnership races distinct receipt-backed +// members through fail-fast completion so only the parent transition winner can +// own terminal facts across the server SQL dialects. +func runDatabaseConcurrentBatchReceiptOwnership[T any](t *testing.T, backend, driverName, dsn string, runtimeCfg T) { + t.Helper() + db, err := sql.Open(driverName, dsn) + if err != nil { + t.Fatalf("open %s concurrent batch receipt database: %v", backend, err) + } + db.SetMaxOpenConns(32) + t.Cleanup(func() { _ = db.Close() }) + store, err := queue.NewSQLStore(queue.SQLStoreConfig{DB: db, DriverName: driverName, AutoMigrate: true}) + if err != nil { + t.Fatalf("new %s concurrent batch receipt store: %v", backend, err) + } + + const memberCount = 12 + queueName := fmt.Sprintf("batch_receipt_race_%s_%d", backend, time.Now().UnixNano()) + jobType := "job:db:batch-receipt-race:" + backend + recorder := &databaseSettlementRecorder{settlement: make(chan struct{})} + runtimeCfg = withDefaultQueue(runtimeCfg, queueName) + started := make(chan struct{}) + release := make(chan struct{}) + var ( + handlerCalls atomic.Int64 + startedOnce sync.Once + releaseOnce sync.Once + ) + t.Cleanup(func() { releaseOnce.Do(func() { close(release) }) }) + handler := func(context.Context, queue.Message) error { + if handlerCalls.Add(1) == memberCount { + startedOnce.Do(func() { close(started) }) + } + <-release + return queue.Permanent(errors.New("concurrent fail-fast member failure")) + } + runtimes := make([]*queue.Queue, memberCount) + t.Cleanup(func() { + releaseOnce.Do(func() { close(release) }) + shutdownCtx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + for _, runtime := range runtimes { + if runtime != nil { + _ = runtime.Shutdown(shutdownCtx) + } + } + }) + for worker := range memberCount { + runtimes[worker], err = testenv.NewQueue(runtimeCfg, queue.WithStore(store), queue.WithObserver(recorder), queue.WithWorkers(1)) + if err != nil { + t.Fatalf("new %s concurrent batch receipt runtime %d: %v", backend, worker, err) + } + runtimes[worker].Register(jobType, handler) + if err := runtimes[worker].StartWorkers(context.Background()); err != nil { + t.Fatalf("start %s concurrent batch receipt runtime %d: %v", backend, worker, err) + } + } + jobs := make([]queue.Job, memberCount) + for member := range memberCount { + jobs[member] = queue.NewJob(jobType).Payload(map[string]int{"member": member}).OnQueue(queueName) + } + batchID, err := runtimes[0].Batch(jobs...).OnQueue(queueName).Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch %s concurrent receipt batch: %v", backend, err) + } + select { + case <-started: + case <-time.After(20 * time.Second): + t.Fatalf("timed out waiting for %s concurrent batch members; calls=%d", backend, handlerCalls.Load()) + } + releaseOnce.Do(func() { close(release) }) + + deadline := time.Now().Add(20 * time.Second) + var state queue.BatchState + for time.Now().Before(deadline) { + state, err = store.GetBatch(context.Background(), batchID) + if err == nil && state.Pending == 0 && state.Processed == memberCount && recorder.count(queue.EventBatchFailed, jobType) == 1 && recorder.count(queue.EventBatchCancelled, jobType) == 1 { + break + } + time.Sleep(20 * time.Millisecond) + } + if err != nil || state.Pending != 0 || state.Processed != memberCount || state.Failed != memberCount || !state.Completed || !state.Cancelled { + t.Fatalf("%s concurrent receipt batch state = %+v, err:%v", backend, state, err) + } + if handlerCalls.Load() != memberCount { + t.Fatalf("%s concurrent receipt handler calls = %d, want %d without receipt-conflict redelivery", backend, handlerCalls.Load(), memberCount) + } + if failed, cancelled, completed := recorder.count(queue.EventBatchFailed, jobType), recorder.count(queue.EventBatchCancelled, jobType), recorder.count(queue.EventBatchCompleted, jobType); failed != 1 || cancelled != 1 || completed != 0 { + t.Fatalf("%s terminal batch facts = failed:%d cancelled:%d completed:%d, want 1/1/0", backend, failed, cancelled, completed) + } + + placeholder := "?" + if backend == testenv.BackendPostgres { + placeholder = "$1" + } + var receiptCount int + countQuery := `SELECT COUNT(*) FROM bus_workflow_transition_receipts WHERE workflow_kind='batch' AND workflow_id=` + placeholder + if err := db.QueryRow(countQuery, batchID).Scan(&receiptCount); err != nil { + t.Fatalf("count %s concurrent batch receipts: %v", backend, err) + } + if receiptCount != memberCount+1 { + t.Fatalf("%s concurrent batch receipts = %d, want %d member plus aggregate rows", backend, receiptCount, memberCount+1) + } + var aggregateOwner, aggregateJob string + var aggregateCompleted, aggregateCancelled int + aggregateQuery := `SELECT owner_delivery_id, job_id, aggregate_completed, aggregate_cancelled FROM bus_workflow_transition_receipts WHERE workflow_kind='batch' AND member_id='' AND workflow_id=` + placeholder + if err := db.QueryRow(aggregateQuery, batchID).Scan(&aggregateOwner, &aggregateJob, &aggregateCompleted, &aggregateCancelled); err != nil { + t.Fatalf("read %s aggregate batch receipt: %v", backend, err) + } + if aggregateOwner == "" || aggregateJob == "" || aggregateCompleted != 1 || aggregateCancelled != 1 { + t.Fatalf("%s aggregate receipt = owner:%q job:%q completed:%d cancelled:%d", backend, aggregateOwner, aggregateJob, aggregateCompleted, aggregateCancelled) + } + memberPlaceholder := "?" + jobPlaceholder := "?" + if backend == testenv.BackendPostgres { + memberPlaceholder = "$2" + jobPlaceholder = "$3" + } + var matchingMemberReceipts int + memberQuery := `SELECT COUNT(*) FROM bus_workflow_transition_receipts WHERE workflow_kind='batch' AND member_id<>'' AND workflow_id=` + placeholder + ` AND owner_delivery_id=` + memberPlaceholder + ` AND job_id=` + jobPlaceholder + if err := db.QueryRow(memberQuery, batchID, aggregateOwner, aggregateJob).Scan(&matchingMemberReceipts); err != nil { + t.Fatalf("match %s aggregate receipt owner to member: %v", backend, err) + } + if matchingMemberReceipts != 1 { + t.Fatalf("%s member receipts matching aggregate owner = %d, want 1", backend, matchingMemberReceipts) + } +} + func newDatabaseQueueIntegration(t *testing.T, cfg queue.DatabaseConfig) QueueRuntime { t.Helper() var runtimeCfg any switch cfg.DriverName { case testenv.BackendMySQL: - runtimeCfg = withDefaultQueue(withDBHandle(mysqlCfg(cfg.DSN), cfg.DB), cfg.DefaultQueue) + runtimeCfg = withDefaultQueue(withDBHandle(mysqlCfg(cfg.DSN), cfg.DB), cfg.DefaultQueue) + case "pgx", testenv.BackendPostgres: + runtimeCfg = withDefaultQueue(withDBHandle(postgresCfg(cfg.DSN), cfg.DB), cfg.DefaultQueue) + case testenv.BackendSQLite: + runtimeCfg = withDefaultQueue(withDBHandle(sqliteCfg(cfg.DSN), cfg.DB), cfg.DefaultQueue) + default: + t.Fatalf("unsupported database driver %q", cfg.DriverName) + } + q, err := newQueueRuntime(runtimeCfg) + if err != nil { + t.Fatalf("new database queue failed: %v", err) + } + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = q.Shutdown(shutdownCtx) + }) + return q +} + +// historicalDatabaseUniqueKey reproduces the pre-version SQL lock identity so +// integration coverage can seed records exactly as an older producer did. +func historicalDatabaseUniqueKey(job queue.Job, queueName string) string { + digest := sha256.Sum256(append([]byte(queueName+":"+job.Type+":"), job.PayloadBytes()...)) + return hex.EncodeToString(digest[:]) +} + +// rebindHistoricalDatabaseQuery preserves the placeholder behavior used by +// the pre-version SQL producer without importing driver internals into the test. +func rebindHistoricalDatabaseQuery(query, driverName string) string { + if driverName != "pgx" && driverName != testenv.BackendPostgres { + return query + } + var rebound strings.Builder + argument := 1 + for _, char := range query { + if char == '?' { + fmt.Fprintf(&rebound, "$%d", argument) + argument++ + continue + } + rebound.WriteRune(char) + } + return rebound.String() +} + +// historicalDatabaseUniqueConflict matches the constraint classification used +// by the pre-version SQL producer so the integration path is behaviorally exact. +func historicalDatabaseUniqueConflict(err error) bool { + if err == nil { + return false + } + message := strings.ToLower(err.Error()) + return strings.Contains(message, "duplicate") || + strings.Contains(message, "unique constraint") || + strings.Contains(message, "unique violation") +} + +// dispatchHistoricalDatabaseJob reproduces origin/main's legacy claim and +// queue-row writes on a dedicated connection for mixed-version race coverage. +func dispatchHistoricalDatabaseJob(ctx context.Context, conn *sql.Conn, driverName string, job queue.Job) error { + options := queue.DriverOptions(job) + now := time.Now() + expiresAt := now.Add(options.UniqueTTL).UnixMilli() + legacyKey := historicalDatabaseUniqueKey(job, options.QueueName) + insertLock := rebindHistoricalDatabaseQuery( + `INSERT INTO queue_unique_locks(lock_key, expires_at) VALUES(?, ?)`, + driverName, + ) + if _, err := conn.ExecContext(ctx, insertLock, legacyKey, expiresAt); err != nil { + if !historicalDatabaseUniqueConflict(err) { + return err + } + updateLock := rebindHistoricalDatabaseQuery( + `UPDATE queue_unique_locks SET expires_at=? WHERE lock_key=? AND expires_at <= ?`, + driverName, + ) + result, updateErr := conn.ExecContext(ctx, updateLock, expiresAt, legacyKey, now.UnixMilli()) + if updateErr != nil { + return updateErr + } + rows, _ := result.RowsAffected() + if rows != 1 { + return queue.ErrDuplicate + } + } + + payload := job.PayloadBytes() + if payload == nil { + payload = []byte{} + } + availableAt := now.Add(options.Delay) + maxRetry := 0 + if options.MaxRetry != nil { + maxRetry = *options.MaxRetry + } + backoffMillis := int64(0) + if options.Backoff != nil && *options.Backoff > 0 { + backoffMillis = options.Backoff.Milliseconds() + } + var timeoutSeconds any + if options.Timeout != nil { + timeoutSeconds = max(1, int64(math.Ceil(options.Timeout.Seconds()))) + } + insertJob := rebindHistoricalDatabaseQuery( + `INSERT INTO queue_jobs + (queue_name, job_type, payload, timeout_seconds, max_retry, backoff_millis, attempt, available_at, state, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, 0, ?, 'pending', ?, ?)`, + driverName, + ) + _, err := conn.ExecContext( + ctx, + insertJob, + options.QueueName, + job.Type, + payload, + timeoutSeconds, + maxRetry, + backoffMillis, + availableAt.UnixMilli(), + now.UnixMilli(), + now.UnixMilli(), + ) + return err +} + +// runDatabaseUniqueKeyTransitionIntegration proves every SQL dialect honors +// outstanding historical claims and atomically rolls back the companion key +// when the canonical identity collides. +func runDatabaseUniqueKeyTransitionIntegration(t *testing.T, cfg queue.DatabaseConfig) { + t.Helper() + provisionDatabaseIntegrationSchema(t, cfg) + db, err := sql.Open(cfg.DriverName, cfg.DSN) + if err != nil { + t.Fatalf("open %s uniqueness transition database: %v", cfg.DriverName, err) + } + t.Cleanup(func() { _ = db.Close() }) + for _, table := range []string{"queue_jobs", "queue_unique_locks"} { + if _, err := db.Exec("DELETE FROM " + table); err != nil { + t.Fatalf("clear %s before uniqueness transition: %v", table, err) + } + } + t.Cleanup(func() { + for _, table := range []string{"queue_jobs", "queue_unique_locks"} { + _, _ = db.Exec("DELETE FROM " + table) + } + }) + + firstPlaceholder := "?" + secondPlaceholder := "?" + if cfg.DriverName == "pgx" || cfg.DriverName == testenv.BackendPostgres { + firstPlaceholder = "$1" + secondPlaceholder = "$2" + } + insertLock := func(key string, expiresAt int64) { + t.Helper() + query := `INSERT INTO queue_unique_locks(lock_key, expires_at) VALUES (` + firstPlaceholder + `, ` + secondPlaceholder + `)` + if _, err := db.Exec(query, key, expiresAt); err != nil { + t.Fatalf("seed uniqueness transition lock %q: %v", key, err) + } + } + lockCount := func(key string) int { + t.Helper() + var count int + query := `SELECT COUNT(*) FROM queue_unique_locks WHERE lock_key=` + firstPlaceholder + if err := db.QueryRow(query, key).Scan(&count); err != nil { + t.Fatalf("count uniqueness transition lock %q: %v", key, err) + } + return count + } + jobCount := func(jobType string) int { + t.Helper() + var count int + query := `SELECT COUNT(*) FROM queue_jobs WHERE job_type=` + firstPlaceholder + if err := db.QueryRow(query, jobType).Scan(&count); err != nil { + t.Fatalf("count uniqueness transition jobs %q: %v", jobType, err) + } + return count + } + producer := newDatabaseQueueIntegration(t, cfg) + + t.Run("legacy_outstanding", func(t *testing.T) { + job := queue.NewJob("job:db:unique:legacy-outstanding"). + Payload([]byte("same logical work")). + OnQueue("default"). + UniqueFor(time.Minute) + legacyKey := historicalDatabaseUniqueKey(job, "default") + canonicalKey := queue.DriverUniqueKey(job, "default") + insertLock(legacyKey, time.Now().Add(time.Hour).UnixMilli()) + + if err := producer.Dispatch(job); !errors.Is(err, queue.ErrDuplicate) { + t.Fatalf("dispatch with outstanding legacy lock = %v, want ErrDuplicate", err) + } + if lockCount(canonicalKey) != 0 || jobCount(job.Type) != 0 { + t.Fatal("legacy collision committed a canonical lock or queue row") + } + }) + + t.Run("legacy_expired", func(t *testing.T) { + job := queue.NewJob("job:db:unique:legacy-expired"). + Payload([]byte("same logical work")). + OnQueue("default"). + UniqueFor(time.Minute) + legacyKey := historicalDatabaseUniqueKey(job, "default") + canonicalKey := queue.DriverUniqueKey(job, "default") + insertLock(legacyKey, 0) + + if err := producer.Dispatch(job); err != nil { + t.Fatalf("dispatch with expired legacy lock: %v", err) + } + if lockCount(legacyKey) != 1 || lockCount(canonicalKey) != 1 || jobCount(job.Type) != 1 { + t.Fatal("expired legacy lock did not commit both identities with the queue row") + } + var expiresAt int64 + query := `SELECT expires_at FROM queue_unique_locks WHERE lock_key=` + firstPlaceholder + if err := db.QueryRow(query, legacyKey).Scan(&expiresAt); err != nil { + t.Fatalf("read renewed legacy lock: %v", err) + } + if expiresAt <= 0 { + t.Fatalf("renewed legacy lock expiry = %d, want positive database time", expiresAt) + } + }) + + t.Run("canonical_outstanding", func(t *testing.T) { + job := queue.NewJob("job:db:unique:canonical-outstanding"). + Payload([]byte("same logical work")). + OnQueue("default"). + UniqueFor(time.Minute) + legacyKey := historicalDatabaseUniqueKey(job, "default") + canonicalKey := queue.DriverUniqueKey(job, "default") + insertLock(canonicalKey, time.Now().Add(time.Hour).UnixMilli()) + + if err := producer.Dispatch(job); !errors.Is(err, queue.ErrDuplicate) { + t.Fatalf("dispatch with outstanding canonical lock = %v, want ErrDuplicate", err) + } + if lockCount(legacyKey) != 0 || jobCount(job.Type) != 0 { + t.Fatal("canonical collision committed the preceding legacy lock or queue row") + } + }) + + t.Run("mixed_producer_race", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + legacyConn, err := db.Conn(ctx) + if err != nil { + t.Fatalf("open dedicated %s legacy producer connection: %v", cfg.DriverName, err) + } + defer legacyConn.Close() + if cfg.DriverName == testenv.BackendSQLite { + if _, err := legacyConn.ExecContext(ctx, `PRAGMA busy_timeout=5000`); err != nil { + t.Fatalf("configure legacy SQLite producer lock wait: %v", err) + } + } + + const rounds = 8 + for round := range rounds { + job := queue.NewJob(fmt.Sprintf("job:db:unique:mixed-producer:%d", round)). + Payload([]byte("same logical work")). + OnQueue("default"). + UniqueFor(time.Minute) + start := make(chan struct{}) + results := make(chan error, 2) + go func() { + <-start + results <- dispatchHistoricalDatabaseJob(ctx, legacyConn, cfg.DriverName, job) + }() + go func() { + <-start + results <- producer.WithContext(ctx).Dispatch(job) + }() + close(start) + + accepted := 0 + duplicates := 0 + for range 2 { + select { + case dispatchErr := <-results: + switch { + case dispatchErr == nil: + accepted++ + case errors.Is(dispatchErr, queue.ErrDuplicate): + duplicates++ + default: + t.Fatalf("round %d mixed-version dispatch error: %v", round, dispatchErr) + } + case <-ctx.Done(): + t.Fatalf("round %d mixed-version dispatch timed out: %v", round, ctx.Err()) + } + } + if accepted != 1 || duplicates != 1 { + t.Fatalf("round %d mixed-version accepted/duplicate results = %d/%d, want 1/1", round, accepted, duplicates) + } + if rows := jobCount(job.Type); rows != 1 { + t.Fatalf("round %d mixed-version queue rows = %d, want 1", round, rows) + } + } + }) +} + +// managedDatabaseRuntimeConfig preserves each integration suite's physical +// database while making external schema ownership explicit for the runtime +// under test. +func managedDatabaseRuntimeConfig(cfg queue.DatabaseConfig, queueName string) any { + switch cfg.DriverName { + case testenv.BackendMySQL: + runtimeCfg := withDefaultQueue(withDBHandle(mysqlCfg(cfg.DSN), cfg.DB), queueName) + runtimeCfg.DisableAutoMigrate = true + return runtimeCfg case "pgx", testenv.BackendPostgres: - runtimeCfg = withDefaultQueue(withDBHandle(postgresCfg(cfg.DSN), cfg.DB), cfg.DefaultQueue) + runtimeCfg := withDefaultQueue(withDBHandle(postgresCfg(cfg.DSN), cfg.DB), queueName) + runtimeCfg.DisableAutoMigrate = true + return runtimeCfg case testenv.BackendSQLite: - runtimeCfg = withDefaultQueue(withDBHandle(sqliteCfg(cfg.DSN), cfg.DB), cfg.DefaultQueue) + runtimeCfg := withDefaultQueue(withDBHandle(sqliteCfg(cfg.DSN), cfg.DB), queueName) + runtimeCfg.DisableAutoMigrate = true + return runtimeCfg default: - t.Fatalf("unsupported database driver %q", cfg.DriverName) + return nil } - q, err := newQueueRuntime(runtimeCfg) +} + +// provisionDatabaseIntegrationSchema installs the canonical dialect schema +// through a distinct auto-migrating runtime before managed-mode validation. +func provisionDatabaseIntegrationSchema(t *testing.T, cfg queue.DatabaseConfig) { + t.Helper() + bootstrap := newDatabaseQueueIntegration(t, cfg) + if err := bootstrap.StartWorkers(context.Background()); err != nil { + t.Fatalf("provision %s managed schema: %v", cfg.DriverName, err) + } + if err := bootstrap.Shutdown(context.Background()); err != nil { + t.Fatalf("close %s managed schema bootstrap: %v", cfg.DriverName, err) + } +} + +// runDatabaseManagedSchemaIntegration proves a canonical externally +// provisioned schema supports readiness, uniqueness, dispatch, and processing +// through the same managed runtime path on every SQL dialect. +func runDatabaseManagedSchemaIntegration(t *testing.T, name string, cfg queue.DatabaseConfig) { + t.Helper() + provisionDatabaseIntegrationSchema(t, cfg) + resetQueueTables(t, cfg) + queueName := name + "-managed-schema" + runtimeCfg := managedDatabaseRuntimeConfig(cfg, queueName) + if runtimeCfg == nil { + t.Fatalf("unsupported managed database driver %q", cfg.DriverName) + } + runtime, err := testenv.NewQueue(runtimeCfg) if err != nil { - t.Fatalf("new database queue failed: %v", err) + t.Fatalf("new %s managed-schema runtime: %v", name, err) } t.Cleanup(func() { shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - _ = q.Shutdown(shutdownCtx) + _ = runtime.Shutdown(shutdownCtx) }) - return q + + jobType := "job:db:managed-schema:" + name + processed := make(chan queue.Message, 2) + runtime.Register(jobType, func(_ context.Context, message queue.Message) error { + processed <- message + return nil + }) + if err := runtime.Ready(context.Background()); err != nil { + t.Fatalf("%s managed schema readiness: %v", name, err) + } + if err := runtime.StartWorkers(context.Background()); err != nil { + t.Fatalf("start %s managed-schema runtime: %v", name, err) + } + job := queue.NewJob(jobType). + Payload([]byte(`{"managed":true}`)). + OnQueue(queueName). + UniqueFor(time.Minute) + if _, err := runtime.Dispatch(job); err != nil { + t.Fatalf("dispatch through %s managed schema: %v", name, err) + } + if _, err := runtime.Dispatch(job); !errors.Is(err, queue.ErrDuplicate) { + t.Fatalf("duplicate dispatch through %s managed schema = %v, want ErrDuplicate", name, err) + } + select { + case delivered := <-processed: + if delivered.JobType != jobType || string(delivered.PayloadBytes()) != `{"managed":true}` { + t.Fatalf("%s managed-schema delivery = type:%q payload:%q", name, delivered.JobType, delivered.PayloadBytes()) + } + case <-time.After(15 * time.Second): + logDatabaseQueueState(t, cfg, name+" managed-schema timeout") + t.Fatalf("%s managed-schema runtime did not consume the dispatched job", name) + } +} + +// runSQLiteStaleProcessingFence proves a superseded handler cannot settle the row generation now owned by another runtime. +func runSQLiteStaleProcessingFence(t *testing.T, staleResult error) { + t.Helper() + dsn := fmt.Sprintf("%s/queue-processing-fence-%d.db", t.TempDir(), time.Now().UnixNano()) + recorder := &databaseSettlementRecorder{settlement: make(chan struct{})} + runtimeCfg := withDBRecoveryPolicy( + withObserver(withDefaultQueue(sqliteCfg(dsn), "default"), recorder), + 10*time.Millisecond, + time.Minute, + ) + firstRuntime, err := newQueueRuntime(runtimeCfg) + if err != nil { + t.Fatalf("new first fenced settlement runtime: %v", err) + } + secondRuntime, err := newQueueRuntime(runtimeCfg) + if err != nil { + t.Fatalf("new second fenced settlement runtime: %v", err) + } + + firstStarted := make(chan struct{}) + secondStarted := make(chan struct{}) + releaseFirst := make(chan struct{}) + releaseSecond := make(chan struct{}) + unexpectedCall := make(chan int64, 1) + var ( + calls atomic.Int64 + releaseFirstOnce sync.Once + releaseSecondOnce sync.Once + ) + t.Cleanup(func() { + releaseFirstOnce.Do(func() { close(releaseFirst) }) + releaseSecondOnce.Do(func() { close(releaseSecond) }) + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = firstRuntime.Shutdown(shutdownCtx) + _ = secondRuntime.Shutdown(shutdownCtx) + }) + + jobType := "job:db:processing-fence:success" + if staleResult != nil { + jobType = "job:db:processing-fence:failure" + } + handler := func(context.Context, queue.Job) error { + switch call := calls.Add(1); call { + case 1: + close(firstStarted) + <-releaseFirst + return staleResult + case 2: + close(secondStarted) + <-releaseSecond + return nil + default: + select { + case unexpectedCall <- call: + default: + } + return nil + } + } + firstRuntime.Register(jobType, handler) + secondRuntime.Register(jobType, handler) + if err := firstRuntime.StartWorkers(context.Background()); err != nil { + t.Fatalf("start first fenced settlement runtime: %v", err) + } + if err := secondRuntime.StartWorkers(context.Background()); err != nil { + t.Fatalf("start second fenced settlement runtime: %v", err) + } + db, err := sql.Open(testenv.BackendSQLite, dsn) + if err != nil { + t.Fatalf("open fenced settlement database: %v", err) + } + defer db.Close() + if err := firstRuntime.Dispatch(queue.NewJob(jobType).OnQueue("default")); err != nil { + t.Fatalf("dispatch fenced settlement job: %v", err) + } + select { + case <-firstStarted: + case <-time.After(5 * time.Second): + t.Fatal("first processing generation did not start") + } + result, err := execSQLiteIntegrationEventually(db, `UPDATE queue_jobs SET processing_started_at=1 WHERE job_type=? AND state='processing'`, jobType) + if err != nil { + t.Fatalf("age first processing generation: %v", err) + } + if rows, err := result.RowsAffected(); err != nil || rows != 1 { + t.Fatalf("aged first processing rows = %d, error %v; want 1", rows, err) + } + select { + case <-secondStarted: + case call := <-unexpectedCall: + t.Fatalf("unexpected processing generation %d started before reclaim", call) + case <-time.After(5 * time.Second): + t.Fatal("stale processing generation was not recovered and reclaimed") + } + + releaseFirstOnce.Do(func() { close(releaseFirst) }) + select { + case <-recorder.settlement: + case call := <-unexpectedCall: + t.Fatalf("unexpected processing generation %d started during stale settlement", call) + case <-time.After(5 * time.Second): + t.Fatal("stale processing generation did not report settlement failure") + } + if successes := recorder.count(queue.EventProcessSucceeded, jobType); successes != 0 { + t.Fatalf("stale generation committed %d process_succeeded events", successes) + } + + var state string + var processingToken sql.NullString + var attempt int + if err := db.QueryRow(`SELECT state, processing_token, attempt FROM queue_jobs WHERE job_type=?`, jobType).Scan(&state, &processingToken, &attempt); err != nil { + t.Fatalf("reclaimed row was deleted or overwritten by stale handler: %v", err) + } + if state != "processing" || !processingToken.Valid || processingToken.String == "" || attempt != 0 { + t.Fatalf("reclaimed row = state:%q token:%q valid:%t attempt:%d, want fenced processing claim at attempt 0", state, processingToken.String, processingToken.Valid, attempt) + } + + releaseSecondOnce.Do(func() { close(releaseSecond) }) + deadline := time.Now().Add(5 * time.Second) + for recorder.count(queue.EventProcessSucceeded, jobType) == 0 && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if successes := recorder.count(queue.EventProcessSucceeded, jobType); successes != 1 { + t.Fatalf("current generation process_succeeded events = %d, want 1", successes) + } + var rows int + if err := db.QueryRow(`SELECT COUNT(*) FROM queue_jobs WHERE job_type=?`, jobType).Scan(&rows); err != nil { + t.Fatalf("count finalized fenced row: %v", err) + } + if rows != 0 { + t.Fatalf("current processing generation left %d queue rows", rows) + } } func runDatabaseIntegrationSuite(t *testing.T, name string, cfg queue.DatabaseConfig) { @@ -112,6 +2194,58 @@ func runDatabaseIntegrationSuite(t *testing.T, name string, cfg queue.DatabaseCo } }) + t.Run(name+"_unique_rolling_upgrade", func(t *testing.T) { + runDatabaseUniqueKeyTransitionIntegration(t, cfg) + }) + + t.Run(name+"_unique_multi_producer", func(t *testing.T) { + first := newDatabaseQueueIntegration(t, cfg) + second := newDatabaseQueueIntegration(t, cfg) + for _, runtime := range []QueueRuntime{first, second} { + runtime.Register("job:db:unique:concurrent", func(_ context.Context, _ queue.Job) error { return nil }) + if err := runtime.StartWorkers(context.Background()); err != nil { + t.Fatalf("start producer runtime failed: %v", err) + } + } + resetQueueTables(t, cfg) + + start := make(chan struct{}) + results := make(chan error, 2) + var dispatches sync.WaitGroup + for _, runtime := range []QueueRuntime{first, second} { + dispatches.Add(1) + go func(runtime QueueRuntime) { + defer dispatches.Done() + <-start + results <- runtime.Dispatch( + queue.NewJob("job:db:unique:concurrent"). + Payload([]byte("same logical work")). + OnQueue("default"). + UniqueFor(time.Minute), + ) + }(runtime) + } + close(start) + dispatches.Wait() + close(results) + + accepted := 0 + duplicates := 0 + for err := range results { + switch { + case err == nil: + accepted++ + case errors.Is(err, queue.ErrDuplicate): + duplicates++ + default: + t.Fatalf("concurrent unique dispatch failed: %v", err) + } + } + if accepted != 1 || duplicates != 1 { + t.Fatalf("concurrent unique results = accepted:%d duplicate:%d, want 1/1", accepted, duplicates) + } + }) + t.Run(name+"_retry_backoff", func(t *testing.T) { d := newDatabaseQueueIntegration(t, cfg) triggered := make(chan struct{}, 1) @@ -199,6 +2333,479 @@ func TestDatabaseIntegration_SQLite(t *testing.T) { PollInterval: 10 * time.Millisecond, } runDatabaseIntegrationSuite(t, testenv.BackendSQLite, cfg) + t.Run("sqlite_managed_schema_dispatch_and_process", func(t *testing.T) { + runDatabaseManagedSchemaIntegration(t, testenv.BackendSQLite, cfg) + }) + + t.Run("sqlite_caller_owned_database_remains_open", func(t *testing.T) { + dsn := fmt.Sprintf("%s/queue-caller-owned-%d.db", t.TempDir(), time.Now().UnixNano()) + db, err := sql.Open(testenv.BackendSQLite, dsn) + if err != nil { + t.Fatalf("open caller-owned database: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + runtime := newDatabaseQueueIntegration(t, queue.DatabaseConfig{ + DB: db, + DriverName: testenv.BackendSQLite, + DSN: dsn, + Workers: 1, + PollInterval: 10 * time.Millisecond, + DefaultQueue: "default", + AutoMigrate: true, + }) + if err := runtime.Shutdown(context.Background()); err != nil { + t.Fatalf("shutdown caller-owned runtime: %v", err) + } + if err := db.PingContext(context.Background()); err != nil { + t.Fatalf("caller-owned database was closed: %v", err) + } + }) + + t.Run("sqlite_managed_schema_fails_closed_then_recovers_after_provisioning", func(t *testing.T) { + dsn := fmt.Sprintf("%s/queue-no-migrate-%d.db", t.TempDir(), time.Now().UnixNano()) + runtime, err := sqlitequeue.NewWithConfig(sqlitequeue.Config{ + DSN: dsn, + DisableAutoMigrate: true, + }) + if err != nil { + t.Fatalf("new no-migrate runtime: %v", err) + } + t.Cleanup(func() { _ = runtime.Shutdown(context.Background()) }) + processed := make(chan struct{}, 1) + runtime.Register("job:db:managed-schema-retry", func(context.Context, queue.Message) error { + processed <- struct{}{} + return nil + }) + if err := runtime.Ready(context.Background()); err == nil { + t.Fatal("managed runtime reported ready before external schema provisioning") + } + if err := runtime.StartWorkers(context.Background()); err == nil { + t.Fatal("managed runtime started workers before external schema provisioning") + } + + db, err := sql.Open(testenv.BackendSQLite, dsn) + if err != nil { + t.Fatalf("open no-migrate database: %v", err) + } + defer db.Close() + var tables int + if err := db.QueryRow(`SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name IN ('queue_jobs', 'queue_unique_locks')`).Scan(&tables); err != nil { + t.Fatalf("inspect no-migrate schema: %v", err) + } + if tables != 0 { + t.Fatalf("managed readiness or startup created %d queue tables", tables) + } + + prepareSQLiteIntegrationSchema(t, dsn) + if err := runtime.Ready(context.Background()); err != nil { + t.Fatalf("managed runtime readiness after external provisioning: %v", err) + } + if err := runtime.StartWorkers(context.Background()); err != nil { + t.Fatalf("start same managed runtime after external provisioning: %v", err) + } + if _, err := runtime.Dispatch(queue.NewJob("job:db:managed-schema-retry").OnQueue("default")); err != nil { + t.Fatalf("dispatch after external schema provisioning: %v", err) + } + select { + case <-processed: + case <-time.After(5 * time.Second): + t.Fatal("same managed runtime did not consume after external schema provisioning") + } + }) + + t.Run("sqlite_managed_schema_requires_processing_token", func(t *testing.T) { + dsn := fmt.Sprintf("%s/queue-managed-missing-processing-token-%d.db", t.TempDir(), time.Now().UnixNano()) + prepareSQLiteIntegrationSchema(t, dsn) + db, err := sql.Open(testenv.BackendSQLite, dsn) + if err != nil { + t.Fatalf("open managed schema database: %v", err) + } + if _, err := db.Exec(`ALTER TABLE queue_jobs DROP COLUMN processing_token`); err != nil { + _ = db.Close() + t.Fatalf("remove managed processing token column: %v", err) + } + if err := db.Close(); err != nil { + t.Fatalf("close managed schema database: %v", err) + } + runtime, err := sqlitequeue.NewWithConfig(sqlitequeue.Config{ + DSN: dsn, + DisableAutoMigrate: true, + }) + if err != nil { + t.Fatalf("new managed schema runtime: %v", err) + } + t.Cleanup(func() { _ = runtime.Shutdown(context.Background()) }) + err = runtime.StartWorkers(context.Background()) + if err == nil || !strings.Contains(err.Error(), "missing required processing_token column") { + t.Fatalf("managed schema startup error = %v", err) + } + }) + + t.Run("sqlite_start_retries_after_canceled_migration", func(t *testing.T) { + runtime := newDatabaseQueueIntegration(t, queue.DatabaseConfig{ + DriverName: testenv.BackendSQLite, + DSN: fmt.Sprintf("%s/queue-start-retry-%d.db", t.TempDir(), time.Now().UnixNano()), + Workers: 1, + PollInterval: 10 * time.Millisecond, + AutoMigrate: true, + }) + processed := make(chan struct{}, 1) + runtime.Register("job:db:start-retry", func(context.Context, queue.Job) error { + processed <- struct{}{} + return nil + }) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := runtime.StartWorkers(ctx); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled start error = %v, want context.Canceled", err) + } + if err := runtime.StartWorkers(context.Background()); err != nil { + t.Fatalf("retry start after canceled migration: %v", err) + } + if err := runtime.Dispatch(queue.NewJob("job:db:start-retry").OnQueue("default")); err != nil { + t.Fatalf("dispatch after retried start: %v", err) + } + select { + case <-processed: + case <-time.After(5 * time.Second): + t.Fatal("retried start reported success without a running worker") + } + }) + + t.Run("sqlite_start_retries_after_migration_lock", func(t *testing.T) { + dsn := fmt.Sprintf("%s/queue-start-lock-%d.db", t.TempDir(), time.Now().UnixNano()) + runtime := newDatabaseQueueIntegration(t, queue.DatabaseConfig{ + DriverName: testenv.BackendSQLite, + DSN: dsn, + Workers: 1, + PollInterval: 10 * time.Millisecond, + AutoMigrate: true, + }) + processed := make(chan struct{}, 1) + runtime.Register("job:db:start-lock", func(context.Context, queue.Job) error { + processed <- struct{}{} + return nil + }) + + lockDB, err := sql.Open(testenv.BackendSQLite, dsn) + if err != nil { + t.Fatalf("open migration lock database: %v", err) + } + defer lockDB.Close() + lockConn, err := lockDB.Conn(context.Background()) + if err != nil { + t.Fatalf("open migration lock connection: %v", err) + } + defer lockConn.Close() + if _, err := lockConn.ExecContext(context.Background(), `PRAGMA busy_timeout=0`); err != nil { + t.Fatalf("disable migration lock wait: %v", err) + } + if _, err := lockConn.ExecContext(context.Background(), `BEGIN EXCLUSIVE`); err != nil { + t.Fatalf("acquire migration lock: %v", err) + } + locked := true + defer func() { + if locked { + _, _ = lockConn.ExecContext(context.Background(), `ROLLBACK`) + } + }() + + startCtx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + startErr := runtime.StartWorkers(startCtx) + cancel() + if startErr == nil { + t.Fatal("migration unexpectedly succeeded while SQLite schema was exclusively locked") + } + if _, err := lockConn.ExecContext(context.Background(), `ROLLBACK`); err != nil { + t.Fatalf("release migration lock: %v", err) + } + locked = false + if err := runtime.StartWorkers(context.Background()); err != nil { + t.Fatalf("retry start after migration lock: %v", err) + } + if err := runtime.Dispatch(queue.NewJob("job:db:start-lock").OnQueue("default")); err != nil { + t.Fatalf("dispatch after migration retry: %v", err) + } + select { + case <-processed: + case <-time.After(5 * time.Second): + t.Fatal("migration retry reported success without a running worker") + } + }) + + t.Run("sqlite_unique_queue_insert_rollback", func(t *testing.T) { + rollbackCfg := queue.DatabaseConfig{ + DriverName: testenv.BackendSQLite, + DSN: fmt.Sprintf("%s/queue-rollback-%d.db", t.TempDir(), time.Now().UnixNano()), + Workers: 1, + PollInterval: 10 * time.Millisecond, + } + provisionDatabaseIntegrationSchema(t, rollbackCfg) + runtime := newDatabaseQueueIntegration(t, rollbackCfg) + + db, err := sql.Open(testenv.BackendSQLite, rollbackCfg.DSN) + if err != nil { + t.Fatalf("open rollback database: %v", err) + } + defer db.Close() + const trigger = `CREATE TRIGGER reject_unique_queue_insert +BEFORE INSERT ON queue_jobs +WHEN NEW.job_type = 'job:db:unique:rollback' +BEGIN + SELECT RAISE(ABORT, 'forced queue insert failure'); +END` + if _, err := db.Exec(trigger); err != nil { + t.Fatalf("create rollback trigger: %v", err) + } + job := queue.NewJob("job:db:unique:rollback").OnQueue("default").UniqueFor(time.Minute) + if err := runtime.Dispatch(job); err == nil || errors.Is(err, queue.ErrDuplicate) { + t.Fatalf("forced queue insert error = %v, want storage rejection", err) + } + if _, err := db.Exec(`DROP TRIGGER reject_unique_queue_insert`); err != nil { + t.Fatalf("drop rollback trigger: %v", err) + } + if err := runtime.Dispatch(job); err != nil { + t.Fatalf("dispatch after rolled-back claim failed: %v", err) + } + }) + + t.Run("sqlite_processing_token_migrates_existing_rows", func(t *testing.T) { + dsn := fmt.Sprintf("%s/queue-processing-token-migration-%d.db", t.TempDir(), time.Now().UnixNano()) + db, err := sql.Open(testenv.BackendSQLite, dsn) + if err != nil { + t.Fatalf("open legacy schema database: %v", err) + } + defer db.Close() + const legacySchema = `CREATE TABLE queue_jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + queue_name TEXT NOT NULL, + job_type TEXT NOT NULL, + payload BLOB NOT NULL, + timeout_seconds INTEGER NULL, + max_retry INTEGER NOT NULL DEFAULT 0, + backoff_millis INTEGER NOT NULL DEFAULT 0, + attempt INTEGER NOT NULL DEFAULT 0, + available_at INTEGER NOT NULL, + processing_started_at INTEGER NULL, + last_error TEXT NULL, + state TEXT NOT NULL, + created_at INTEGER NOT NULL, + updated_at INTEGER NOT NULL + )` + if _, err := db.Exec(legacySchema); err != nil { + t.Fatalf("create legacy queue schema: %v", err) + } + now := time.Now().UnixMilli() + if _, err := db.Exec(`INSERT INTO queue_jobs + (queue_name, job_type, payload, max_retry, backoff_millis, attempt, available_at, state, created_at, updated_at) + VALUES ('default', 'job:db:legacy-row', X'', 0, 0, 0, ?, 'pending', ?, ?)`, now+time.Hour.Milliseconds(), now, now); err != nil { + t.Fatalf("insert legacy queue row: %v", err) + } + + runtime, err := sqlitequeue.New(dsn) + if err != nil { + t.Fatalf("new runtime over legacy schema: %v", err) + } + if err := runtime.StartWorkers(context.Background()); err != nil { + t.Fatalf("migrate legacy processing token column: %v", err) + } + if err := runtime.Shutdown(context.Background()); err != nil { + t.Fatalf("shutdown migrated runtime: %v", err) + } + + var processingToken sql.NullString + if err := db.QueryRow(`SELECT processing_token FROM queue_jobs WHERE job_type='job:db:legacy-row'`).Scan(&processingToken); err != nil { + t.Fatalf("read migrated legacy row: %v", err) + } + if processingToken.Valid { + t.Fatalf("legacy pending row received processing token %q", processingToken.String) + } + }) + + t.Run("sqlite_stale_success_cannot_delete_or_commit_reclaimed_generation", func(t *testing.T) { + runSQLiteStaleProcessingFence(t, nil) + }) + + t.Run("sqlite_stale_failure_cannot_overwrite_reclaimed_generation", func(t *testing.T) { + runSQLiteStaleProcessingFence(t, errors.New("stale application failure")) + }) + + t.Run("sqlite_success_waits_for_durable_finalization", func(t *testing.T) { + dsn := fmt.Sprintf("%s/queue-settlement-%d.db", t.TempDir(), time.Now().UnixNano()) + prepareSQLiteIntegrationSchema(t, dsn) + recorder := &databaseSettlementRecorder{settlement: make(chan struct{})} + runtimeCfg := withObserver(withDefaultQueue(sqliteCfg(dsn), "default"), recorder) + runtimeCfg.DisableAutoMigrate = true + runtime, err := newQueueRuntime(runtimeCfg) + if err != nil { + t.Fatalf("new settlement runtime: %v", err) + } + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = runtime.Shutdown(shutdownCtx) + }) + const jobType = "job:db:settlement" + runtime.Register(jobType, func(context.Context, queue.Job) error { return nil }) + + db, err := sql.Open(testenv.BackendSQLite, dsn) + if err != nil { + t.Fatalf("open settlement database: %v", err) + } + defer db.Close() + const trigger = `CREATE TRIGGER reject_job_finalization +BEFORE DELETE ON queue_jobs +WHEN OLD.job_type = 'job:db:settlement' +BEGIN + SELECT RAISE(ABORT, 'forced finalization failure'); +END` + if _, err := db.Exec(trigger); err != nil { + t.Fatalf("create finalization trigger: %v", err) + } + if err := runtime.StartWorkers(context.Background()); err != nil { + t.Fatalf("start settlement runtime: %v", err) + } + if err := runtime.Dispatch(queue.NewJob(jobType).OnQueue("default")); err != nil { + t.Fatalf("dispatch settlement job: %v", err) + } + select { + case <-recorder.settlement: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for settlement_failed") + } + if recorder.has(queue.EventProcessSucceeded, jobType) { + t.Fatal("process_succeeded emitted before durable row deletion") + } + if !recorder.has(queue.EventSettlementFailed, jobType) { + t.Fatal("missing correlated settlement_failed event") + } + }) + + t.Run("sqlite_workflow_success_facts_recover_after_finalization_failure", func(t *testing.T) { + for _, workflowKind := range []string{ + "chain", + "chain_predecessor", + "batch", + "batch_predecessor", + } { + t.Run(workflowKind, func(t *testing.T) { + runSQLiteWorkflowWinnerFactRecovery(t, workflowKind) + }) + } + }) + + t.Run("sqlite_workflow_receipt_owner_survives_repeated_finalization_failure", func(t *testing.T) { + runSQLiteRepeatedWorkflowSettlementRecovery(t) + }) + + t.Run("sqlite_failed_chain_recovery_archives_without_reexecution", func(t *testing.T) { + runSQLiteFailedChainSettlementRecovery(t) + }) + + t.Run("sqlite_terminal_batch_completion_recovers_from_completing_member", func(t *testing.T) { + runSQLiteTerminalBatchOwnerRecovery(t) + }) + + t.Run("sqlite_failed_batch_recovery_archives_without_reexecution", func(t *testing.T) { + runSQLiteFailedBatchSettlementRecovery(t) + }) + + t.Run("sqlite_later_workflow_attempt_wins_then_recovers_without_reexecution", func(t *testing.T) { + runSQLiteLaterWorkflowWinnerRecovery(t) + }) + + t.Run("sqlite_application_error_cannot_forge_recovery_proof", func(t *testing.T) { + dsn := fmt.Sprintf("%s/queue-recovery-proof-collision-%d.db", t.TempDir(), time.Now().UnixNano()) + prepareSQLiteIntegrationSchema(t, dsn) + const ( + queueName = "recovery-proof-collision" + jobType = "job:db:recovery-proof-collision" + ) + runtimeCfg := withDefaultQueue(sqliteCfg(dsn), queueName) + runtimeCfg.DisableAutoMigrate = true + runtime, err := testenv.NewQueue(runtimeCfg, queue.WithWorkers(1)) + if err != nil { + t.Fatalf("new recovery proof collision runtime: %v", err) + } + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = runtime.Shutdown(shutdownCtx) + }) + var calls atomic.Int64 + var forged atomic.Bool + handlerDone := make(chan struct{}) + var handlerDoneOnce sync.Once + runtime.Register(jobType, func(ctx context.Context, _ queue.Message) error { + if calls.Add(1) == 1 { + return errors.New("queue:internal:stale-processing-recovery:v1") + } + provenance, present := busruntime.DeliveryProvenanceFromContext(ctx) + forged.Store(present && provenance.Recovered) + handlerDoneOnce.Do(func() { close(handlerDone) }) + return nil + }) + if err := runtime.StartWorkers(context.Background()); err != nil { + t.Fatalf("start recovery proof collision runtime: %v", err) + } + if _, err := runtime.Dispatch(queue.NewJob(jobType).OnQueue(queueName).Retry(1)); err != nil { + t.Fatalf("dispatch recovery proof collision job: %v", err) + } + select { + case <-handlerDone: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for collision replay handler") + } + if forged.Load() { + t.Fatal("application error text granted stale-processing recovery authority") + } + }) + + t.Run("sqlite_missing_handler_finalization_failure_is_observed", func(t *testing.T) { + dsn := fmt.Sprintf("%s/queue-missing-handler-settlement-%d.db", t.TempDir(), time.Now().UnixNano()) + prepareSQLiteIntegrationSchema(t, dsn) + recorder := &databaseSettlementRecorder{settlement: make(chan struct{})} + runtimeCfg := withObserver(withDefaultQueue(sqliteCfg(dsn), "default"), recorder) + runtimeCfg.DisableAutoMigrate = true + runtime, err := newQueueRuntime(runtimeCfg) + if err != nil { + t.Fatalf("new missing-handler runtime: %v", err) + } + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = runtime.Shutdown(shutdownCtx) + }) + db, err := sql.Open(testenv.BackendSQLite, dsn) + if err != nil { + t.Fatalf("open missing-handler database: %v", err) + } + defer db.Close() + const jobType = "job:db:missing-handler-settlement" + const trigger = `CREATE TRIGGER reject_missing_handler_finalization +BEFORE UPDATE OF state ON queue_jobs +WHEN OLD.job_type = 'job:db:missing-handler-settlement' AND OLD.state = 'processing' +BEGIN + SELECT RAISE(ABORT, 'forced missing-handler finalization failure'); +END` + if _, err := db.Exec(trigger); err != nil { + t.Fatalf("create missing-handler finalization trigger: %v", err) + } + if err := runtime.StartWorkers(context.Background()); err != nil { + t.Fatalf("start missing-handler runtime: %v", err) + } + if err := runtime.Dispatch(queue.NewJob(jobType).OnQueue("default")); err != nil { + t.Fatalf("dispatch missing-handler job: %v", err) + } + select { + case <-recorder.settlement: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for missing-handler settlement_failed") + } + if !recorder.has(queue.EventSettlementFailed, jobType) { + t.Fatal("missing correlated settlement_failed event for missing handler") + } + }) } func TestDatabaseIntegration_MySQL(t *testing.T) { @@ -213,6 +2820,15 @@ func TestDatabaseIntegration_MySQL(t *testing.T) { PollInterval: 10 * time.Millisecond, } runDatabaseIntegrationSuite(t, testenv.BackendMySQL, cfg) + t.Run("mysql_managed_schema_dispatch_and_process", func(t *testing.T) { + runDatabaseManagedSchemaIntegration(t, testenv.BackendMySQL, cfg) + }) + t.Run("mysql_workflow_receipt_recovery", func(t *testing.T) { + runDatabaseWorkflowReceiptRecovery(t, testenv.BackendMySQL, testenv.BackendMySQL, cfg.DSN, mysqlCfg(cfg.DSN)) + }) + t.Run("mysql_concurrent_batch_receipt_owner", func(t *testing.T) { + runDatabaseConcurrentBatchReceiptOwnership(t, testenv.BackendMySQL, testenv.BackendMySQL, cfg.DSN, mysqlCfg(cfg.DSN)) + }) } func TestDatabaseIntegration_Postgres(t *testing.T) { @@ -227,4 +2843,13 @@ func TestDatabaseIntegration_Postgres(t *testing.T) { PollInterval: 10 * time.Millisecond, } runDatabaseIntegrationSuite(t, testenv.BackendPostgres, cfg) + t.Run("postgres_managed_schema_dispatch_and_process", func(t *testing.T) { + runDatabaseManagedSchemaIntegration(t, testenv.BackendPostgres, cfg) + }) + t.Run("postgres_workflow_receipt_recovery", func(t *testing.T) { + runDatabaseWorkflowReceiptRecovery(t, testenv.BackendPostgres, "pgx", cfg.DSN, postgresCfg(cfg.DSN)) + }) + t.Run("postgres_concurrent_batch_receipt_owner", func(t *testing.T) { + runDatabaseConcurrentBatchReceiptOwnership(t, testenv.BackendPostgres, "pgx", cfg.DSN, postgresCfg(cfg.DSN)) + }) } diff --git a/integration/root/nats_integration_test.go b/integration/root/nats_integration_test.go index 07b6804..032558d 100644 --- a/integration/root/nats_integration_test.go +++ b/integration/root/nats_integration_test.go @@ -101,6 +101,82 @@ func TestNATSIntegration_BindPayloadThroughWorker(t *testing.T) { } } +// TestNATSIntegration_ShutdownDrainsQueuedCallbacks proves the real asynchronous client drain completes admitted callback backlog before closing publication resources. +func TestNATSIntegration_ShutdownDrainsQueuedCallbacks(t *testing.T) { + if !integrationBackendEnabled(testenv.BackendNATS) { + t.Skip("nats integration backend not selected") + } + const jobs = 8 + firstStarted := make(chan struct{}) + releaseFirst := make(chan struct{}) + var handled atomic.Int32 + + q, err := newQueueRuntime(natsCfg(ensureNATS(t))) + if err != nil { + t.Fatalf("new nats queue failed: %v", err) + } + q.Register("job:nats:drain-backlog", func(context.Context, queue.Job) error { + if handled.Add(1) == 1 { + close(firstStarted) + <-releaseFirst + } + return nil + }) + if err := withWorkers(q, 1).StartWorkers(context.Background()); err != nil { + t.Fatalf("nats queue start failed: %v", err) + } + + if err := q.Dispatch(queue.NewJob("job:nats:drain-backlog").OnQueue("default")); err != nil { + t.Fatalf("dispatch first backlog job: %v", err) + } + select { + case <-firstStarted: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for first backlog job") + } + for i := 1; i < jobs; i++ { + if err := q.Dispatch(queue.NewJob("job:nats:drain-backlog").OnQueue("default")); err != nil { + t.Fatalf("dispatch backlog job %d: %v", i, err) + } + } + // The first callback holds the only worker permit, giving the real NATS client time to queue the flushed backlog locally. + time.Sleep(100 * time.Millisecond) + shutdownResult := make(chan error, 1) + go func() { shutdownResult <- q.Shutdown(context.Background()) }() + close(releaseFirst) + if err := <-shutdownResult; err != nil { + t.Fatalf("shutdown nats backlog: %v", err) + } + if got := handled.Load(); got != jobs { + t.Fatalf("handled backlog = %d, want %d before shutdown", got, jobs) + } +} + +// TestNATSIntegration_InvalidSubscriptionSubjectFailsCleanly verifies a +// subscription setup failure is returned without poisoning a later startup. +func TestNATSIntegration_InvalidSubscriptionSubjectFailsCleanly(t *testing.T) { + if !integrationBackendEnabled(testenv.BackendNATS) { + t.Skip("nats integration backend not selected") + } + q, err := newQueueRuntime(withDefaultQueue(natsCfg(ensureNATS(t)), "invalid subject")) + if err != nil { + t.Fatalf("new nats queue failed: %v", err) + } + worker := withWorkers(q, 1) + for attempt := 1; attempt <= 2; attempt++ { + err := worker.StartWorkers(context.Background()) + if err == nil { + t.Fatalf("startup attempt %d accepted an invalid NATS subject", attempt) + } + if errors.Is(err, queue.ErrQueuerShuttingDown) { + t.Fatalf("startup attempt %d retained failed lifecycle state: %v", attempt, err) + } + } + if err := q.Shutdown(context.Background()); err != nil { + t.Fatalf("shutdown after failed startup: %v", err) + } +} + func TestNATSIntegration_OptionBehavior(t *testing.T) { if !integrationBackendEnabled(testenv.BackendNATS) { t.Skip("nats integration backend not selected") diff --git a/integration/root/observability_integration_test.go b/integration/root/observability_integration_test.go index d8b1e28..e0617c4 100644 --- a/integration/root/observability_integration_test.go +++ b/integration/root/observability_integration_test.go @@ -15,6 +15,7 @@ import ( ) func TestObservabilityIntegration_AllBackends(t *testing.T) { + sqsQueueName := uniqueQueueName("obs-sqs") fixtures := []struct { name string queue string @@ -93,14 +94,14 @@ func TestObservabilityIntegration_AllBackends(t *testing.T) { }, { name: testenv.BackendSQS, - queue: "obs_sqs", + queue: sqsQueueName, workers: 2, newQueue: func(t *testing.T, collector *queue.StatsCollector) QueueRuntime { ensureSQS(t) q, err := newQueueRuntime(withObserver( withDefaultQueue( sqsCfg(integrationSQS.region, integrationSQS.endpoint, integrationSQS.accessKey, integrationSQS.secretKey), - "obs_sqs", + sqsQueueName, ), collector, )) @@ -171,13 +172,13 @@ func TestObservabilityIntegration_AllBackends(t *testing.T) { } }) - t.Run("scenario_dispatch_retry_archive", func(t *testing.T) { + t.Run("scenario_dispatch_terminal_failure", func(t *testing.T) { failJob := queue.NewJob(failType). Payload(scenarioPayload{ID: 2, Name: "obs-fail"}). OnQueue(fx.queue). Retry(0) - requireScenarioNoErr(t, "dispatch_retry_archive", q.Dispatch(failJob)) - waitForObservabilityScenario(t, "retry_archive_attempts", 12*time.Second, func() bool { + requireScenarioNoErr(t, "dispatch_terminal_failure", q.Dispatch(failJob)) + waitForObservabilityScenario(t, "terminal_failure_attempts", 12*time.Second, func() bool { return failedCalls.Load() >= 1 }) }) @@ -206,7 +207,7 @@ func TestObservabilityIntegration_AllBackends(t *testing.T) { }) requireScenarioTrue(t, "collector_processed", counters.Processed >= 1, "processed=%d expected>=1", counters.Processed) requireScenarioTrue(t, "collector_failed", counters.Failed >= 1, "failed=%d expected>=1", counters.Failed) - requireScenarioTrue(t, "collector_archived", counters.Archived >= 1, "archived=%d expected>=1", counters.Archived) + // Archived is intentionally not a cross-driver counter until each settlement owner emits a confirmed terminal fact. if fx.name != testenv.BackendRedis { requireScenarioTrue(t, "collector_retried", counters.Retry >= 1, "retry=%d expected>=1", counters.Retry) } @@ -237,14 +238,16 @@ func TestObservabilityIntegration_AllBackends(t *testing.T) { return nativeCounters.Pending == 0 && nativeCounters.Active == 0 }) snapshot := collector.Snapshot() + finalCounters, queueOK := snapshot.Queue(fx.queue) + requireScenarioTrue(t, "collector_final_counters_present", queueOK, "queue=%q not found in final collector snapshot", fx.queue) throughput, ok := snapshot.Throughput(fx.queue) requireScenarioTrue(t, "collector_throughput_present", ok, "throughput missing for queue=%q", fx.queue) requireScenarioTrue(t, "collector_hour_processed", throughput.Hour.Processed >= 1, "hour_processed=%d expected>=1", throughput.Hour.Processed) requireScenarioTrue(t, "collector_hour_failed", throughput.Hour.Failed >= 1, "hour_failed=%d expected>=1", throughput.Hour.Failed) - requireScenarioTrue(t, "collector_getter_processed", snapshot.Processed(fx.queue) == counters.Processed, "getter_processed=%d counters_processed=%d", snapshot.Processed(fx.queue), counters.Processed) - requireScenarioTrue(t, "collector_getter_failed", snapshot.Failed(fx.queue) == counters.Failed, "getter_failed=%d counters_failed=%d", snapshot.Failed(fx.queue), counters.Failed) + requireScenarioTrue(t, "collector_getter_processed", snapshot.Processed(fx.queue) == finalCounters.Processed, "getter_processed=%d counters_processed=%d", snapshot.Processed(fx.queue), finalCounters.Processed) + requireScenarioTrue(t, "collector_getter_failed", snapshot.Failed(fx.queue) == finalCounters.Failed, "getter_failed=%d counters_failed=%d", snapshot.Failed(fx.queue), finalCounters.Failed) if fx.name != testenv.BackendRedis { - requireScenarioTrue(t, "collector_getter_retry", snapshot.RetryCount(fx.queue) == counters.Retry, "getter_retry=%d counters_retry=%d", snapshot.RetryCount(fx.queue), counters.Retry) + requireScenarioTrue(t, "collector_getter_retry", snapshot.RetryCount(fx.queue) == finalCounters.Retry, "getter_retry=%d counters_retry=%d", snapshot.RetryCount(fx.queue), finalCounters.Retry) } }) diff --git a/integration/root/rabbitmq_integration_test.go b/integration/root/rabbitmq_integration_test.go index ad80975..c461a27 100644 --- a/integration/root/rabbitmq_integration_test.go +++ b/integration/root/rabbitmq_integration_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/goforj/queue" + "github.com/goforj/queue/busruntime" "github.com/goforj/queue/integration/testenv" amqp "github.com/rabbitmq/amqp091-go" "github.com/testcontainers/testcontainers-go" @@ -180,6 +181,114 @@ func TestRabbitMQIntegration_OptionBehavior(t *testing.T) { } } +// TestRabbitMQIntegration_ImmediateRetry verifies a retry without backoff advances +// the physical attempt and commits without leaving an immediate duplicate delivery. +func TestRabbitMQIntegration_ImmediateRetry(t *testing.T) { + if !integrationBackendEnabled(testenv.BackendRabbitMQ) { + t.Skip("rabbitmq integration backend not selected") + } + queueName := uniqueQueueName("rabbitmq-immediate-retry") + type deliveryObservation struct { + attempt busruntime.DeliveryAttempt + present bool + deferred bool + } + observations := make(chan deliveryObservation, 3) + committed := make(chan struct{}, 1) + + q, err := newQueueRuntime(newRabbitMQIntegrationConfigForQueue(t, queueName)) + if err != nil { + t.Fatalf("new rabbitmq queue failed: %v", err) + } + q.Register("job:rabbitmq:immediate-retry", func(ctx context.Context, _ queue.Job) error { + attempt, ok := busruntime.DeliveryAttemptFromContext(ctx) + observation := deliveryObservation{attempt: attempt, present: ok} + if !ok { + observations <- observation + return queue.Permanent(errors.New("rabbitmq delivery did not expose attempt metadata")) + } + switch attempt.Number { + case 0: + observations <- observation + return errors.New("retry immediately") + case 1: + observation.deferred = busruntime.DeferUntilDeliveryCommitted(ctx, func() { + committed <- struct{}{} + }) + observations <- observation + if !observation.deferred { + return queue.Permanent(errors.New("rabbitmq delivery did not expose settlement metadata")) + } + return nil + default: + observations <- observation + return nil + } + }) + if err := withWorkers(q, 1).StartWorkers(context.Background()); err != nil { + t.Fatalf("rabbitmq queue start failed: %v", err) + } + defer q.Shutdown(context.Background()) + + job := queue.NewJob("job:rabbitmq:immediate-retry").OnQueue(queueName).Retry(1) + if err := q.Dispatch(job); err != nil { + t.Fatalf("dispatch failed: %v", err) + } + + wantAttempts := []busruntime.DeliveryAttempt{ + {Number: 0, MaxRetry: 1}, + {Number: 1, MaxRetry: 1}, + } + for index, want := range wantAttempts { + select { + case observation := <-observations: + if !observation.present { + t.Fatalf("delivery %d did not expose attempt metadata", index+1) + } + if observation.attempt != want { + t.Fatalf("delivery %d attempt = %+v, want %+v", index+1, observation.attempt, want) + } + if index == 1 && !observation.deferred { + t.Fatal("retry delivery did not expose settlement metadata") + } + case <-time.After(15 * time.Second): + t.Fatalf("timed out waiting for rabbitmq delivery %d", index+1) + } + } + + select { + case <-committed: + case <-time.After(15 * time.Second): + t.Fatal("timed out waiting for retry delivery acknowledgement") + } + if err := q.Shutdown(context.Background()); err != nil { + t.Fatalf("shutdown rabbitmq queue: %v", err) + } + select { + case observation := <-observations: + t.Fatalf("unexpected third rabbitmq delivery during shutdown: %+v", observation.attempt) + default: + } + + inspectionConnection, err := amqp.Dial(integrationRabbitMQ.url) + if err != nil { + t.Fatalf("dial rabbitmq for retry queue inspection: %v", err) + } + defer inspectionConnection.Close() + inspectionChannel, err := inspectionConnection.Channel() + if err != nil { + t.Fatalf("open rabbitmq retry inspection channel: %v", err) + } + defer inspectionChannel.Close() + queueState, err := inspectionChannel.QueueInspect(queueName) + if err != nil { + t.Fatalf("inspect rabbitmq retry queue: %v", err) + } + if queueState.Messages != 0 || queueState.Consumers != 0 { + t.Fatalf("retry queue after shutdown = messages:%d consumers:%d, want empty without consumers", queueState.Messages, queueState.Consumers) + } +} + func TestRabbitMQIntegration_UniqueDuplicate(t *testing.T) { if !integrationBackendEnabled(testenv.BackendRabbitMQ) { t.Skip("rabbitmq integration backend not selected") diff --git a/integration/root/redis_retry_reserve_integration_test.go b/integration/root/redis_retry_reserve_integration_test.go new file mode 100644 index 0000000..3c3a1d9 --- /dev/null +++ b/integration/root/redis_retry_reserve_integration_test.go @@ -0,0 +1,68 @@ +//go:build integration + +package root_test + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/goforj/queue" + "github.com/goforj/queue/busruntime" + "github.com/goforj/queue/integration/testenv" +) + +// TestRedisIntegration_FinalUncommittedRedeliversSameAttempt proves the reserved transport slot against the real Asynq processor. +func TestRedisIntegration_FinalUncommittedRedeliversSameAttempt(t *testing.T) { + if !integrationBackendEnabled(testenv.BackendRedis) { + t.Skip("redis integration backend not selected") + } + ensureRedis(t) + queueName := uniqueQueueName("redis-uncommitted-final") + runtime, err := newQueueRuntime(withDefaultQueue(redisCfg(integrationRedis.addr), queueName)) + if err != nil { + t.Fatalf("new redis runtime: %v", err) + } + t.Cleanup(func() { + shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + _ = runtime.Shutdown(shutdownCtx) + }) + + jobType := queueName + ":job" + done := make(chan struct{}) + var ( + mu sync.Mutex + attempts []int + ) + runtime.Register(jobType, func(_ context.Context, job queue.Job) error { + mu.Lock() + attempts = append(attempts, queue.DriverOptions(job).Attempt) + call := len(attempts) + mu.Unlock() + if call == 1 { + return busruntime.Uncommitted(errors.New("workflow store unavailable")) + } + close(done) + return nil + }) + if err := runtime.StartWorkers(context.Background()); err != nil { + t.Fatalf("start redis workers: %v", err) + } + if err := runtime.Dispatch(queue.NewJob(jobType).OnQueue(queueName).Retry(0)); err != nil { + t.Fatalf("dispatch zero-retry job: %v", err) + } + + select { + case <-done: + case <-time.After(15 * time.Second): + t.Fatal("uncommitted final attempt was not redelivered") + } + mu.Lock() + defer mu.Unlock() + if len(attempts) != 2 || attempts[0] != 0 || attempts[1] != 0 { + t.Fatalf("application attempts = %v, want [0 0]", attempts) + } +} diff --git a/integration/root/workflow_store_integration_test.go b/integration/root/workflow_store_integration_test.go new file mode 100644 index 0000000..2735945 --- /dev/null +++ b/integration/root/workflow_store_integration_test.go @@ -0,0 +1,894 @@ +//go:build integration + +package root_test + +import ( + "context" + "database/sql" + "errors" + "fmt" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/goforj/queue" + "github.com/goforj/queue/integration/testenv" +) + +// newWorkflowStoreIntegration opens a caller-owned handle so each dialect +// contract can exercise multiple real connections without leaking ownership. +func newWorkflowStoreIntegration(t *testing.T, driverName, dsn string) queue.WorkflowStore { + t.Helper() + db, err := sql.Open(driverName, dsn) + if err != nil { + t.Fatalf("open workflow database: %v", err) + } + db.SetMaxOpenConns(32) + t.Cleanup(func() { _ = db.Close() }) + store, err := queue.NewSQLStore(queue.SQLStoreConfig{ + DB: db, + DriverName: driverName, + AutoMigrate: true, + }) + if err != nil { + t.Fatalf("new workflow store: %v", err) + } + return store +} + +// waitWorkflowStoreOperations bounds concurrent database probes so a locking +// regression fails near its source instead of consuming the global test timeout. +func waitWorkflowStoreOperations(t *testing.T, wg *sync.WaitGroup) { + t.Helper() + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + select { + case <-done: + case <-time.After(35 * time.Second): + t.Fatal("timed out waiting for concurrent workflow-store operations") + } +} + +// retryWorkflowStoreConflict models broker redelivery for database-selected +// deadlock victims while preserving every non-transient error verbatim. +func retryWorkflowStoreConflict(ctx context.Context, operation func() error) error { + var lastErr error + for range 10 { + lastErr = operation() + if lastErr == nil { + return nil + } + message := strings.ToLower(lastErr.Error()) + if !strings.Contains(message, "deadlock") { + return lastErr + } + select { + case <-ctx.Done(): + return errors.Join(lastErr, ctx.Err()) + default: + } + } + return lastErr +} + +// settleWorkflowStoreBatchConcurrently creates one batch and races mixed +// outcomes so callers can assert both continuation and fail-fast policies. +func settleWorkflowStoreBatchConcurrently(t *testing.T, ctx context.Context, store queue.WorkflowStore, prefix string, allowFailures bool) queue.BatchState { + t.Helper() + const jobCount = 32 + jobs := make([]queue.BatchJob, jobCount) + for i := range jobs { + jobs[i] = queue.BatchJob{ + JobID: fmt.Sprintf("%s-member-%02d", prefix, i), + Job: queue.StoredJob{Type: "reports:member"}, + } + } + batchID := prefix + "-batch" + if err := store.CreateBatch(ctx, queue.BatchRecord{ + BatchID: batchID, + DispatchID: prefix + "-dispatch", + AllowFailed: allowFailures, + Jobs: jobs, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create concurrent batch: %v", err) + } + start := make(chan struct{}) + errs := make(chan error, jobCount) + var wg sync.WaitGroup + for i, job := range jobs { + wg.Add(1) + go func(index int, member queue.BatchJob) { + defer wg.Done() + <-start + var err error + if index%2 == 0 { + _, _, err = store.MarkBatchJobSucceeded(ctx, batchID, member.JobID) + } else { + _, _, err = store.MarkBatchJobFailed(ctx, batchID, member.JobID, errors.New("member failed")) + } + errs <- err + }(i, job) + } + close(start) + waitWorkflowStoreOperations(t, &wg) + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("settle concurrent batch members: %v", err) + } + } + state, err := store.GetBatch(ctx, batchID) + if err != nil { + t.Fatalf("get concurrent batch: %v", err) + } + return state +} + +// settleWorkflowStoreTerminalMemberConcurrently races contradictory outcomes +// through aggregate completion and optional fail-fast cancellation. +func settleWorkflowStoreTerminalMemberConcurrently(t *testing.T, ctx context.Context, store queue.WorkflowStore, outcomes queue.WorkflowOutcomeStore, prefix string, allowFailures bool) queue.BatchState { + t.Helper() + batchID := prefix + "-batch" + jobID := prefix + "-member" + if err := store.CreateBatch(ctx, queue.BatchRecord{ + BatchID: batchID, + AllowFailed: allowFailures, + Jobs: []queue.BatchJob{{JobID: jobID}}, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create terminal batch: %v", err) + } + start := make(chan struct{}) + errs := make(chan error, 32) + var wg sync.WaitGroup + for delivery := range 32 { + outcome := queue.BatchJobSucceeded + if delivery%2 == 0 { + outcome = queue.BatchJobFailed + } + wg.Add(1) + go func(outcome queue.BatchJobOutcome) { + defer wg.Done() + <-start + errs <- retryWorkflowStoreConflict(ctx, func() error { + _, _, err := outcomes.SettleBatchJob(ctx, batchID, jobID, outcome, errors.New("raced terminal outcome")) + return err + }) + }(outcome) + } + close(start) + waitWorkflowStoreOperations(t, &wg) + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("race terminal batch outcome: %v", err) + } + } + state, err := store.GetBatch(ctx, batchID) + if err != nil { + t.Fatalf("get terminal batch: %v", err) + } + _, successOwned, err := outcomes.SettleBatchJob(ctx, batchID, jobID, queue.BatchJobSucceeded, nil) + if err != nil { + t.Fatalf("replay terminal success: %v", err) + } + _, failureOwned, err := outcomes.SettleBatchJob(ctx, batchID, jobID, queue.BatchJobFailed, errors.New("replayed terminal failure")) + if err != nil { + t.Fatalf("replay terminal failure: %v", err) + } + if successOwned == failureOwned || successOwned != (state.Failed == 0) { + t.Fatalf("terminal outcome ownership = success:%t failure:%t state:%+v", successOwned, failureOwned, state) + } + return state +} + +// runWorkflowStoreConcurrencyContract proves each supported SQL dialect uses +// the same atomic chain, batch, and callback state transitions. +func runWorkflowStoreConcurrencyContract(t *testing.T, driverName, dsn string) { + t.Helper() + store := newWorkflowStoreIntegration(t, driverName, dsn) + outcomes, ok := store.(queue.WorkflowOutcomeStore) + if !ok { + t.Fatalf("built-in SQL store %T does not implement queue.WorkflowOutcomeStore", store) + } + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + prefix := fmt.Sprintf("workflow-store-%d", time.Now().UnixNano()) + + const jobCount = 32 + distinctState := settleWorkflowStoreBatchConcurrently(t, ctx, store, prefix+"-allow-failures", true) + if distinctState.Pending != 0 || distinctState.Processed != jobCount || distinctState.Failed != jobCount/2 || !distinctState.Completed || distinctState.Cancelled { + t.Fatalf("distinct batch state = %+v, want exact aggregate counters", distinctState) + } + failFastState := settleWorkflowStoreBatchConcurrently(t, ctx, store, prefix+"-fail-fast", false) + if failFastState.Pending != 0 || failFastState.Processed != jobCount || failFastState.Failed != jobCount/2 || !failFastState.Completed || !failFastState.Cancelled { + t.Fatalf("fail-fast batch state = %+v, want exact counters and cancellation", failFastState) + } + + duplicateBatchID := prefix + "-duplicate-batch" + duplicateJobID := prefix + "-duplicate-member" + if err := store.CreateBatch(ctx, queue.BatchRecord{ + BatchID: duplicateBatchID, + DispatchID: prefix + "-duplicate-dispatch", + Jobs: []queue.BatchJob{ + {JobID: duplicateJobID, Job: queue.StoredJob{Type: "reports:shared"}}, + {JobID: prefix + "-pending-member", Job: queue.StoredJob{Type: "reports:pending"}}, + }, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create duplicate batch: %v", err) + } + if err := store.MarkBatchJobStarted(ctx, duplicateBatchID, prefix+"-missing-member"); !errors.Is(err, queue.ErrWorkflowNotFound) { + t.Fatalf("start unknown batch member = %v, want ErrWorkflowNotFound", err) + } + if err := store.MarkBatchJobStarted(ctx, duplicateBatchID, duplicateJobID); err != nil { + t.Fatalf("start duplicate batch member: %v", err) + } + if err := store.MarkBatchJobStarted(ctx, duplicateBatchID, duplicateJobID); err != nil { + t.Fatalf("replay duplicate batch member start: %v", err) + } + start := make(chan struct{}) + errs := make(chan error, jobCount) + var wg sync.WaitGroup + for range jobCount { + wg.Add(1) + go func() { + defer wg.Done() + <-start + _, _, err := store.MarkBatchJobSucceeded(ctx, duplicateBatchID, duplicateJobID) + errs <- err + }() + } + close(start) + waitWorkflowStoreOperations(t, &wg) + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("settle duplicate batch member: %v", err) + } + } + duplicateState, owned, err := outcomes.SettleBatchJob(ctx, duplicateBatchID, duplicateJobID, queue.BatchJobFailed, errors.New("inconsistent duplicate")) + if err != nil { + t.Fatalf("reclassify duplicate batch member: %v", err) + } + if owned || duplicateState.Pending != 1 || duplicateState.Processed != 1 || duplicateState.Failed != 0 || duplicateState.Cancelled || duplicateState.Completed { + t.Fatalf("duplicate batch state = %+v owned:%t, want first outcome retained", duplicateState, owned) + } + if _, owned, err := outcomes.SettleBatchJob(ctx, duplicateBatchID, duplicateJobID, queue.BatchJobSucceeded, nil); err != nil || !owned { + t.Fatalf("replay winning batch outcome = owned:%t err:%v", owned, err) + } + + racedBatchID := prefix + "-raced-outcome-batch" + racedJobID := prefix + "-raced-outcome-member" + if err := store.CreateBatch(ctx, queue.BatchRecord{ + BatchID: racedBatchID, + AllowFailed: true, + Jobs: []queue.BatchJob{ + {JobID: racedJobID}, + {JobID: prefix + "-raced-pending-member"}, + }, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create raced batch: %v", err) + } + start = make(chan struct{}) + errs = make(chan error, jobCount) + wg = sync.WaitGroup{} + for delivery := range jobCount { + outcome := queue.BatchJobSucceeded + if delivery%2 == 0 { + outcome = queue.BatchJobFailed + } + wg.Add(1) + go func(outcome queue.BatchJobOutcome) { + defer wg.Done() + <-start + errs <- retryWorkflowStoreConflict(ctx, func() error { + _, _, err := outcomes.SettleBatchJob(ctx, racedBatchID, racedJobID, outcome, errors.New("raced member outcome")) + return err + }) + }(outcome) + } + close(start) + waitWorkflowStoreOperations(t, &wg) + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("race batch outcome: %v", err) + } + } + racedBatchState, err := store.GetBatch(ctx, racedBatchID) + if err != nil { + t.Fatalf("get raced batch: %v", err) + } + if racedBatchState.Pending != 1 || racedBatchState.Processed != 1 || (racedBatchState.Failed != 0 && racedBatchState.Failed != 1) || racedBatchState.Completed { + t.Fatalf("raced batch state = %+v", racedBatchState) + } + _, successOwned, err := outcomes.SettleBatchJob(ctx, racedBatchID, racedJobID, queue.BatchJobSucceeded, nil) + if err != nil { + t.Fatalf("replay raced success: %v", err) + } + _, failureOwned, err := outcomes.SettleBatchJob(ctx, racedBatchID, racedJobID, queue.BatchJobFailed, errors.New("replayed failure")) + if err != nil { + t.Fatalf("replay raced failure: %v", err) + } + if successOwned == failureOwned || successOwned != (racedBatchState.Failed == 0) { + t.Fatalf("raced batch ownership = success:%t failure:%t state:%+v", successOwned, failureOwned, racedBatchState) + } + for _, policy := range []struct { + name string + allowFailures bool + }{ + {name: "allow-failures", allowFailures: true}, + {name: "fail-fast", allowFailures: false}, + } { + state := settleWorkflowStoreTerminalMemberConcurrently(t, ctx, store, outcomes, prefix+"-terminal-"+policy.name, policy.allowFailures) + if state.Pending != 0 || state.Processed != 1 || !state.Completed || (state.Failed != 0 && state.Failed != 1) { + t.Fatalf("%s terminal batch state = %+v", policy.name, state) + } + wantCancelled := state.Failed == 1 && !policy.allowFailures + if state.Cancelled != wantCancelled { + t.Fatalf("%s terminal cancellation = %t, want %t for state %+v", policy.name, state.Cancelled, wantCancelled, state) + } + } + + chainID := prefix + "-chain" + firstNodeID := prefix + "-first-node" + secondNodeID := prefix + "-second-node" + if err := store.CreateChain(ctx, queue.ChainRecord{ + ChainID: chainID, + DispatchID: prefix + "-chain-dispatch", + Nodes: []queue.ChainNode{ + {NodeID: firstNodeID, Job: queue.StoredJob{Type: "reports:first"}}, + {NodeID: secondNodeID, Job: queue.StoredJob{Type: "reports:second"}}, + }, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create chain: %v", err) + } + start = make(chan struct{}) + errs = make(chan error, jobCount) + wg = sync.WaitGroup{} + for range jobCount { + wg.Add(1) + go func() { + defer wg.Done() + <-start + next, done, err := store.AdvanceChain(ctx, chainID, firstNodeID) + if err == nil && (done || next == nil || next.NodeID != secondNodeID) { + err = fmt.Errorf("next = %+v done:%t, want second node", next, done) + } + errs <- err + }() + } + close(start) + waitWorkflowStoreOperations(t, &wg) + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("advance duplicate chain node: %v", err) + } + } + chainState, err := store.GetChain(ctx, chainID) + if err != nil { + t.Fatalf("get chain: %v", err) + } + if chainState.NextIndex != 1 || chainState.Completed || chainState.Failed { + t.Fatalf("chain state = %+v, want one committed node", chainState) + } + chainState, owned, err = outcomes.FailChainNode(ctx, chainID, firstNodeID, errors.New("late node failure")) + if err != nil || owned || chainState.NextIndex != 1 || chainState.Failed { + t.Fatalf("late first-node failure = owned:%t state:%+v err:%v", owned, chainState, err) + } + if next, done, err := store.AdvanceChain(ctx, chainID, secondNodeID); err != nil || !done || next != nil { + t.Fatalf("complete second chain node = next:%+v done:%t err:%v", next, done, err) + } + if err := store.FailChain(ctx, chainID, errors.New("late competing failure")); err != nil { + t.Fatalf("fail completed chain: %v", err) + } + chainState, err = store.GetChain(ctx, chainID) + if err != nil { + t.Fatalf("get completed chain: %v", err) + } + if !chainState.Completed || chainState.Failed || chainState.Failure != "" { + t.Fatalf("late failure changed completed chain: %+v", chainState) + } + + failureFirstChainID := prefix + "-failure-first-chain" + if err := store.CreateChain(ctx, queue.ChainRecord{ + ChainID: failureFirstChainID, + Nodes: []queue.ChainNode{ + {NodeID: prefix + "-failure-first-node"}, + {NodeID: prefix + "-failure-pending-node"}, + }, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create failure-first chain: %v", err) + } + failureNodeID := prefix + "-failure-first-node" + firstCause := errors.New("first chain-node failure") + chainState, owned, err = outcomes.FailChainNode(ctx, failureFirstChainID, failureNodeID, firstCause) + if err != nil || !owned || !chainState.Failed || chainState.NextIndex != 0 { + t.Fatalf("fail current chain node = owned:%t state:%+v err:%v", owned, chainState, err) + } + chainState, owned, err = outcomes.FailChainNode(ctx, failureFirstChainID, failureNodeID, errors.New("replacement cause")) + if err != nil || !owned || chainState.Failure != firstCause.Error() { + t.Fatalf("replay failed chain node = owned:%t state:%+v err:%v", owned, chainState, err) + } + if _, done, err := store.AdvanceChain(ctx, failureFirstChainID, failureNodeID); err != nil || !done { + t.Fatalf("advance failed chain node = done:%t err:%v", done, err) + } + + racedChainID := prefix + "-raced-outcome-chain" + racedNodeID := prefix + "-raced-outcome-node" + if err := store.CreateChain(ctx, queue.ChainRecord{ + ChainID: racedChainID, + Nodes: []queue.ChainNode{{NodeID: racedNodeID}}, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create raced chain: %v", err) + } + start = make(chan struct{}) + errs = make(chan error, jobCount) + wg = sync.WaitGroup{} + for delivery := range jobCount { + wg.Add(1) + go func(fail bool) { + defer wg.Done() + <-start + errs <- retryWorkflowStoreConflict(ctx, func() error { + if fail { + _, _, err := outcomes.FailChainNode(ctx, racedChainID, racedNodeID, errors.New("raced node failure")) + return err + } + _, _, err := store.AdvanceChain(ctx, racedChainID, racedNodeID) + return err + }) + }(delivery%2 == 0) + } + close(start) + waitWorkflowStoreOperations(t, &wg) + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("race chain outcome: %v", err) + } + } + chainState, err = store.GetChain(ctx, racedChainID) + if err != nil { + t.Fatalf("get raced chain: %v", err) + } + successWon := chainState.NextIndex == 1 && !chainState.Failed && chainState.Completed + failureWon := chainState.NextIndex == 0 && chainState.Failed && !chainState.Completed + if !successWon && !failureWon { + t.Fatalf("raced chain state = %+v", chainState) + } + + for _, callbackKey := range []string{prefix + "-Callback", prefix + "-callback"} { + type claimResult struct { + claimed bool + err error + } + start = make(chan struct{}) + results := make(chan claimResult, jobCount) + wg = sync.WaitGroup{} + for range jobCount { + wg.Add(1) + go func() { + defer wg.Done() + <-start + claimed, err := store.MarkCallbackInvoked(ctx, callbackKey) + results <- claimResult{claimed: claimed, err: err} + }() + } + close(start) + waitWorkflowStoreOperations(t, &wg) + close(results) + claims := 0 + for result := range results { + if result.err != nil { + t.Fatalf("claim callback %q: %v", callbackKey, result.err) + } + if result.claimed { + claims++ + } + } + if claims != 1 { + t.Fatalf("callback %q winning claims = %d, want 1", callbackKey, claims) + } + } +} + +// TestWorkflowStoreIntegration_SQLite runs the atomic workflow-store contract +// against multiple real SQLite connections. +func TestWorkflowStoreIntegration_SQLite(t *testing.T) { + if !integrationBackendEnabled(testenv.BackendSQLite) { + t.Skip("sqlite integration backend not selected") + } + dsn := filepath.Join(t.TempDir(), "workflow-store.db") + "?_pragma=busy_timeout%3d10000" + runWorkflowStoreConcurrencyContract(t, testenv.BackendSQLite, dsn) +} + +// TestWorkflowStoreIntegration_MySQL runs the same workflow-store contract on +// MySQL so dialect-specific schema and locking behavior remain executable. +func TestWorkflowStoreIntegration_MySQL(t *testing.T) { + if !integrationBackendEnabled(testenv.BackendMySQL) { + t.Skip("mysql integration backend not selected") + } + ensureMySQLDB(t) + dsn := fmt.Sprintf("queue:queue@tcp(%s)/queue_test?parseTime=true", integrationMySQL.addr) + runWorkflowStoreConcurrencyContract(t, testenv.BackendMySQL, dsn) +} + +// TestWorkflowStoreIntegration_MySQLAutoMigratesMissingReceiptAtLegacyWidths +// proves ordinary startup preserves wide legacy state when introducing receipts. +func TestWorkflowStoreIntegration_MySQLAutoMigratesMissingReceiptAtLegacyWidths(t *testing.T) { + if !integrationBackendEnabled(testenv.BackendMySQL) { + t.Skip("mysql integration backend not selected") + } + ensureMySQLDB(t) + ctx, cancel := context.WithTimeout(context.Background(), 45*time.Second) + defer cancel() + dsn := fmt.Sprintf("queue:queue@tcp(%s)/queue_test?parseTime=true", integrationMySQL.addr) + db, err := sql.Open(testenv.BackendMySQL, dsn) + if err != nil { + t.Fatalf("open MySQL workflow database: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + + bootstrap, err := queue.NewSQLStore(queue.SQLStoreConfig{DB: db, DriverName: testenv.BackendMySQL}) + if err != nil { + t.Fatalf("bootstrap workflow store: %v", err) + } + bootstrapKey := fmt.Sprintf("workflow-receipt-upgrade-bootstrap-%d", time.Now().UnixNano()) + if _, err := bootstrap.MarkCallbackInvoked(ctx, bootstrapKey); err != nil { + t.Fatalf("bootstrap legacy workflow schema: %v", err) + } + if _, err := db.ExecContext(ctx, `DELETE FROM bus_callback_invocations WHERE callback_key=?`, bootstrapKey); err != nil { + t.Fatalf("remove bootstrap callback: %v", err) + } + + chainID := strings.Repeat("c", 320) + nodeID := strings.Repeat("n", 321) + batchID := strings.Repeat("b", 322) + jobID := strings.Repeat("j", 323) + callbackKey := strings.Repeat("k", 700) + t.Cleanup(func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cleanupCancel() + for _, statement := range []struct { + query string + args []any + }{ + {query: `DELETE FROM bus_chain_completed_nodes WHERE chain_id=?`, args: []any{chainID}}, + {query: `DELETE FROM bus_chains WHERE chain_id=?`, args: []any{chainID}}, + {query: `DELETE FROM bus_batch_jobs WHERE batch_id=?`, args: []any{batchID}}, + {query: `DELETE FROM bus_batches WHERE batch_id=?`, args: []any{batchID}}, + {query: `DELETE FROM bus_callback_invocations WHERE callback_key=?`, args: []any{callbackKey}}, + } { + if _, cleanupErr := db.ExecContext(cleanupCtx, statement.query, statement.args...); cleanupErr != nil { + t.Errorf("clean upgraded workflow rows: %v", cleanupErr) + } + } + if _, cleanupErr := db.ExecContext(cleanupCtx, `DROP TABLE IF EXISTS bus_workflow_transition_receipts`); cleanupErr != nil { + t.Errorf("drop derived workflow receipt table: %v", cleanupErr) + } + for _, statement := range []string{ + `ALTER TABLE bus_chains MODIFY chain_id VARBINARY(255) NOT NULL`, + `ALTER TABLE bus_chain_completed_nodes MODIFY chain_id VARBINARY(255) NOT NULL, MODIFY node_id VARBINARY(255) NOT NULL`, + `ALTER TABLE bus_batches MODIFY batch_id VARBINARY(255) NOT NULL`, + `ALTER TABLE bus_batch_jobs MODIFY batch_id VARBINARY(255) NOT NULL, MODIFY job_id VARBINARY(255) NOT NULL`, + `ALTER TABLE bus_callback_invocations MODIFY callback_key VARBINARY(512) NOT NULL`, + } { + if _, cleanupErr := db.ExecContext(cleanupCtx, statement); cleanupErr != nil { + t.Errorf("restore legacy workflow schema width: %v", cleanupErr) + } + } + restored, restoreErr := queue.NewSQLStore(queue.SQLStoreConfig{DB: db, DriverName: testenv.BackendMySQL}) + if restoreErr != nil { + t.Errorf("new workflow store for receipt restoration: %v", restoreErr) + return + } + restoreKey := fmt.Sprintf("workflow-receipt-upgrade-restore-%d", time.Now().UnixNano()) + if _, restoreErr := restored.MarkCallbackInvoked(cleanupCtx, restoreKey); restoreErr != nil { + t.Errorf("restore default workflow receipt table: %v", restoreErr) + return + } + if _, restoreErr := db.ExecContext(cleanupCtx, `DELETE FROM bus_callback_invocations WHERE callback_key=?`, restoreKey); restoreErr != nil { + t.Errorf("remove receipt restoration callback: %v", restoreErr) + } + }) + + if _, err := db.ExecContext(ctx, `DROP TABLE bus_workflow_transition_receipts`); err != nil { + t.Fatalf("remove receipt table from legacy schema: %v", err) + } + for _, statement := range []string{ + `ALTER TABLE bus_chains MODIFY chain_id VARBINARY(512) NOT NULL`, + `ALTER TABLE bus_chain_completed_nodes MODIFY chain_id VARBINARY(512) NOT NULL, MODIFY node_id VARBINARY(512) NOT NULL`, + `ALTER TABLE bus_batches MODIFY batch_id VARBINARY(512) NOT NULL`, + `ALTER TABLE bus_batch_jobs MODIFY batch_id VARBINARY(512) NOT NULL, MODIFY job_id VARBINARY(512) NOT NULL`, + `ALTER TABLE bus_callback_invocations MODIFY callback_key VARBINARY(1024) NOT NULL`, + } { + if _, err := db.ExecContext(ctx, statement); err != nil { + t.Fatalf("widen legacy workflow schema: %v", err) + } + } + + store, err := queue.NewSQLStore(queue.SQLStoreConfig{DB: db, DriverName: testenv.BackendMySQL}) + if err != nil { + t.Fatalf("new auto-migrating store over legacy schema: %v", err) + } + if err := store.CreateChain(ctx, queue.ChainRecord{ + ChainID: chainID, + DispatchID: "legacy-wide-chain-dispatch", + Nodes: []queue.ChainNode{{NodeID: nodeID}}, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create chain through upgraded schema: %v", err) + } + + rows, err := db.QueryContext(ctx, `SELECT column_name, data_type, character_maximum_length + FROM information_schema.columns + WHERE table_schema=DATABASE() AND table_name='bus_workflow_transition_receipts' + AND column_name IN ('workflow_id', 'member_id')`) + if err != nil { + t.Fatalf("read derived receipt widths: %v", err) + } + derivedWidths := make(map[string]int64, 2) + for rows.Next() { + var columnName, dataType string + var width int64 + if err := rows.Scan(&columnName, &dataType, &width); err != nil { + _ = rows.Close() + t.Fatalf("scan derived receipt width: %v", err) + } + if !strings.EqualFold(dataType, "varbinary") { + _ = rows.Close() + t.Fatalf("derived receipt column %s type = %s, want VARBINARY", columnName, dataType) + } + derivedWidths[columnName] = width + } + if err := rows.Err(); err != nil { + _ = rows.Close() + t.Fatalf("iterate derived receipt widths: %v", err) + } + if err := rows.Close(); err != nil { + t.Fatalf("close derived receipt width rows: %v", err) + } + if derivedWidths["workflow_id"] != 512 || derivedWidths["member_id"] != 512 { + t.Fatalf("derived receipt widths = %+v, want workflow_id:512 member_id:512", derivedWidths) + } + + if next, done, err := store.AdvanceChain(ctx, chainID, nodeID); err != nil || !done || next != nil { + t.Fatalf("complete chain through upgraded schema = next:%+v done:%t err:%v", next, done, err) + } + if err := store.CreateBatch(ctx, queue.BatchRecord{ + BatchID: batchID, + DispatchID: "legacy-wide-batch-dispatch", + Jobs: []queue.BatchJob{{JobID: jobID}}, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create batch through upgraded schema: %v", err) + } + outcomes, ok := store.(queue.WorkflowOutcomeStore) + if !ok { + t.Fatalf("upgraded SQL store %T does not implement WorkflowOutcomeStore", store) + } + if state, owned, err := outcomes.SettleBatchJob(ctx, batchID, jobID, queue.BatchJobSucceeded, nil); err != nil || !owned || !state.Completed { + t.Fatalf("settle batch through upgraded schema = state:%+v owned:%t err:%v", state, owned, err) + } + if claimed, err := store.MarkCallbackInvoked(ctx, callbackKey); err != nil || !claimed { + t.Fatalf("claim wide callback through upgraded schema = claimed:%t err:%v", claimed, err) + } + if claimed, err := store.MarkCallbackInvoked(ctx, callbackKey); err != nil || claimed { + t.Fatalf("reclaim wide callback through upgraded schema = claimed:%t err:%v", claimed, err) + } + + receiptWorkflowID := strings.Repeat("r", 324) + receiptMemberID := strings.Repeat("m", 325) + result, err := db.ExecContext(ctx, `INSERT INTO bus_workflow_transition_receipts + (workflow_kind, receipt_version, event_schema_version, workflow_id, member_id, workflow_dispatch_id, + workflow_created_at_ms, outcome, owner_delivery_id, owner_attempt, job_dispatch_id, job_id, + job_fingerprint, aggregate_completed, aggregate_cancelled, created_at_ms) + VALUES ('chain', 1, 1, ?, ?, 'legacy-wide-receipt-dispatch', ?, 'succeeded', + 'legacy-wide-receipt-owner', 1, 'legacy-wide-job-dispatch', 'legacy-wide-job', + 'legacy-wide-job-fingerprint', 0, 0, ?)`, receiptWorkflowID, receiptMemberID, time.Now().UnixMilli(), time.Now().UnixMilli()) + if err != nil { + t.Fatalf("insert wide transition receipt: %v", err) + } + if inserted, err := result.RowsAffected(); err != nil || inserted != 1 { + t.Fatalf("wide transition receipt rows = %d err:%v", inserted, err) + } + result, err = db.ExecContext(ctx, `DELETE FROM bus_workflow_transition_receipts WHERE workflow_kind='chain' AND workflow_id=? AND member_id=?`, receiptWorkflowID, receiptMemberID) + if err != nil { + t.Fatalf("delete wide transition receipt: %v", err) + } + if deleted, err := result.RowsAffected(); err != nil || deleted != 1 { + t.Fatalf("deleted wide transition receipt rows = %d err:%v", deleted, err) + } +} + +// TestWorkflowStoreIntegration_MySQLManagedWideKeys proves validation follows +// an existing wider binary schema instead of imposing fresh-schema defaults. +func TestWorkflowStoreIntegration_MySQLManagedWideKeys(t *testing.T) { + if !integrationBackendEnabled(testenv.BackendMySQL) { + t.Skip("mysql integration backend not selected") + } + ensureMySQLDB(t) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + dsn := fmt.Sprintf("queue:queue@tcp(%s)/queue_test?parseTime=true", integrationMySQL.addr) + db, err := sql.Open(testenv.BackendMySQL, dsn) + if err != nil { + t.Fatalf("open MySQL workflow database: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + + bootstrap, err := queue.NewSQLStore(queue.SQLStoreConfig{DB: db, DriverName: testenv.BackendMySQL, AutoMigrate: true}) + if err != nil { + t.Fatalf("bootstrap workflow store: %v", err) + } + bootstrapKey := fmt.Sprintf("workflow-wide-bootstrap-%d", time.Now().UnixNano()) + if _, err := bootstrap.MarkCallbackInvoked(ctx, bootstrapKey); err != nil { + t.Fatalf("bootstrap workflow schema: %v", err) + } + if _, err := db.ExecContext(ctx, `DELETE FROM bus_callback_invocations WHERE callback_key=?`, bootstrapKey); err != nil { + t.Fatalf("remove bootstrap callback: %v", err) + } + freshOverlongID := strings.Repeat("i", 256) + if err := bootstrap.CreateChain(ctx, queue.ChainRecord{ChainID: freshOverlongID, Nodes: []queue.ChainNode{{NodeID: "fresh-node"}}}); err == nil || !strings.Contains(err.Error(), "255 bytes") { + t.Fatalf("fresh schema overlong chain error = %v", err) + } + if claimed, err := bootstrap.MarkCallbackInvoked(ctx, strings.Repeat("k", 513)); err == nil || claimed || !strings.Contains(err.Error(), "512 bytes") { + t.Fatalf("fresh schema overlong callback = claimed:%t err:%v", claimed, err) + } + + widen := []string{ + `ALTER TABLE bus_chains MODIFY chain_id VARBINARY(512) NOT NULL`, + `ALTER TABLE bus_chain_completed_nodes MODIFY chain_id VARBINARY(512) NOT NULL, MODIFY node_id VARBINARY(512) NOT NULL`, + `ALTER TABLE bus_batches MODIFY batch_id VARBINARY(512) NOT NULL`, + `ALTER TABLE bus_batch_jobs MODIFY batch_id VARBINARY(512) NOT NULL, MODIFY job_id VARBINARY(512) NOT NULL`, + `ALTER TABLE bus_callback_invocations MODIFY callback_key VARBINARY(1024) NOT NULL`, + `ALTER TABLE bus_workflow_transition_receipts MODIFY workflow_id VARBINARY(512) NOT NULL, MODIFY member_id VARBINARY(512) NOT NULL`, + } + chainPrefix := strings.Repeat("chain", 59) + chainIDs := []string{chainPrefix + "A", chainPrefix + "B"} + nodePrefix := strings.Repeat("node", 74) + nodeIDs := []string{nodePrefix + "A", nodePrefix + "B"} + batchID := strings.Repeat("batch", 59) + "A" + jobID := strings.Repeat("member", 49) + "A" + callbackPrefix := strings.Repeat("callback", 87) + callbackKeys := []string{callbackPrefix + "A", callbackPrefix + "B"} + t.Cleanup(func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cleanupCancel() + for _, statement := range []struct { + query string + args []any + }{ + {query: `DELETE FROM bus_chain_completed_nodes WHERE chain_id IN (?, ?)`, args: []any{chainIDs[0], chainIDs[1]}}, + {query: `DELETE FROM bus_chains WHERE chain_id IN (?, ?)`, args: []any{chainIDs[0], chainIDs[1]}}, + {query: `DELETE FROM bus_batch_jobs WHERE batch_id=?`, args: []any{batchID}}, + {query: `DELETE FROM bus_batches WHERE batch_id=?`, args: []any{batchID}}, + {query: `DELETE FROM bus_callback_invocations WHERE callback_key IN (?, ?)`, args: []any{callbackKeys[0], callbackKeys[1]}}, + } { + if _, err := db.ExecContext(cleanupCtx, statement.query, statement.args...); err != nil { + t.Errorf("clean wide workflow rows: %v", err) + } + } + for _, statement := range []string{ + `ALTER TABLE bus_chains MODIFY chain_id VARBINARY(255) NOT NULL`, + `ALTER TABLE bus_chain_completed_nodes MODIFY chain_id VARBINARY(255) NOT NULL, MODIFY node_id VARBINARY(255) NOT NULL`, + `ALTER TABLE bus_batches MODIFY batch_id VARBINARY(255) NOT NULL`, + `ALTER TABLE bus_batch_jobs MODIFY batch_id VARBINARY(255) NOT NULL, MODIFY job_id VARBINARY(255) NOT NULL`, + `ALTER TABLE bus_callback_invocations MODIFY callback_key VARBINARY(512) NOT NULL`, + `ALTER TABLE bus_workflow_transition_receipts MODIFY workflow_id VARBINARY(255) NOT NULL, MODIFY member_id VARBINARY(255) NOT NULL`, + } { + if _, err := db.ExecContext(cleanupCtx, statement); err != nil { + t.Errorf("restore generated workflow schema width: %v", err) + } + } + }) + for _, statement := range widen { + if _, err := db.ExecContext(ctx, statement); err != nil { + t.Fatalf("widen managed workflow schema: %v", err) + } + } + + store, err := queue.NewSQLStoreWithManagedSchema(queue.SQLStoreConfig{DB: db, DriverName: testenv.BackendMySQL}) + if err != nil { + t.Fatalf("new store over managed schema: %v", err) + } + for index := range chainIDs { + if err := store.CreateChain(ctx, queue.ChainRecord{ChainID: chainIDs[index], Nodes: []queue.ChainNode{{NodeID: nodeIDs[index]}}}); err != nil { + t.Fatalf("create wide chain %d: %v", index, err) + } + if next, done, err := store.AdvanceChain(ctx, chainIDs[index], nodeIDs[index]); err != nil || !done || next != nil { + t.Fatalf("complete wide chain %d = next:%+v done:%t err:%v", index, next, done, err) + } + } + if err := store.CreateBatch(ctx, queue.BatchRecord{BatchID: batchID, Jobs: []queue.BatchJob{{JobID: jobID}}}); err != nil { + t.Fatalf("create wide batch: %v", err) + } + outcomes, ok := store.(queue.WorkflowOutcomeStore) + if !ok { + t.Fatalf("managed SQL store %T does not implement WorkflowOutcomeStore", store) + } + if state, owned, err := outcomes.SettleBatchJob(ctx, batchID, jobID, queue.BatchJobSucceeded, nil); err != nil || !owned || !state.Completed { + t.Fatalf("settle wide batch = state:%+v owned:%t err:%v", state, owned, err) + } + for _, key := range callbackKeys { + claimed, err := store.MarkCallbackInvoked(ctx, key) + if err != nil || !claimed { + t.Fatalf("claim wide callback = claimed:%t err:%v", claimed, err) + } + } + if claimed, err := store.MarkCallbackInvoked(ctx, callbackKeys[0]); err != nil || claimed { + t.Fatalf("reclaim wide callback = claimed:%t err:%v", claimed, err) + } +} + +// TestWorkflowStoreIntegration_MySQLRejectsNonVARBINARYKeys proves managed +// schemas cannot silently weaken byte-exact workflow identity comparisons. +func TestWorkflowStoreIntegration_MySQLRejectsNonVARBINARYKeys(t *testing.T) { + if !integrationBackendEnabled(testenv.BackendMySQL) { + t.Skip("mysql integration backend not selected") + } + ensureMySQLDB(t) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + dsn := fmt.Sprintf("queue:queue@tcp(%s)/queue_test?parseTime=true", integrationMySQL.addr) + db, err := sql.Open(testenv.BackendMySQL, dsn) + if err != nil { + t.Fatalf("open MySQL workflow database: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + + bootstrap, err := queue.NewSQLStore(queue.SQLStoreConfig{DB: db, DriverName: testenv.BackendMySQL, AutoMigrate: true}) + if err != nil { + t.Fatalf("bootstrap workflow store: %v", err) + } + bootstrapKey := fmt.Sprintf("workflow-type-bootstrap-%d", time.Now().UnixNano()) + if _, err := bootstrap.MarkCallbackInvoked(ctx, bootstrapKey); err != nil { + t.Fatalf("bootstrap workflow schema: %v", err) + } + if _, err := db.ExecContext(ctx, `DELETE FROM bus_callback_invocations WHERE callback_key=?`, bootstrapKey); err != nil { + t.Fatalf("remove bootstrap callback: %v", err) + } + + if _, err := db.ExecContext(ctx, `ALTER TABLE bus_chain_completed_nodes MODIFY node_id VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL`); err != nil { + t.Fatalf("install comparison-unsafe managed key: %v", err) + } + t.Cleanup(func() { + cleanupCtx, cleanupCancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cleanupCancel() + if _, err := db.ExecContext(cleanupCtx, `ALTER TABLE bus_chain_completed_nodes MODIFY node_id VARBINARY(255) NOT NULL`); err != nil { + t.Errorf("restore workflow node key type: %v", err) + } + }) + + store, err := queue.NewSQLStoreWithManagedSchema(queue.SQLStoreConfig{DB: db, DriverName: testenv.BackendMySQL}) + if err != nil { + t.Fatalf("new store over managed schema: %v", err) + } + if _, err := store.GetChain(ctx, "managed-type-check"); err == nil || !strings.Contains(err.Error(), "bus_chain_completed_nodes.node_id must use VARBINARY") { + t.Fatalf("managed VARCHAR key error = %v", err) + } +} + +// TestWorkflowStoreIntegration_Postgres runs the same workflow-store contract +// on PostgreSQL so its byte and placeholder types cannot silently drift. +func TestWorkflowStoreIntegration_Postgres(t *testing.T) { + if !integrationBackendEnabled(testenv.BackendPostgres) { + t.Skip("postgres integration backend not selected") + } + ensurePostgresDB(t) + dsn := fmt.Sprintf("postgres://queue:queue@%s/queue_test?sslmode=disable", integrationPostgres.addr) + runWorkflowStoreConcurrencyContract(t, "pgx", dsn) +} diff --git a/internal/architecture/comments_test.go b/internal/architecture/comments_test.go new file mode 100644 index 0000000..7c8e663 --- /dev/null +++ b/internal/architecture/comments_test.go @@ -0,0 +1,91 @@ +package architecture_test + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "testing" +) + +// TestWorkflowProductionDeclarationsAreDocumented keeps the extracted engine and compatibility facade aligned with repository comment rules. +func TestWorkflowProductionDeclarationsAreDocumented(t *testing.T) { + repository := repositoryRoot(t) + for _, directory := range []string{filepath.Join(repository, "internal", "workflow"), filepath.Join(repository, "bus")} { + for _, missing := range undocumentedProductionDeclarations(t, directory) { + t.Errorf("missing name-first declaration comment: %s", missing) + } + } +} + +// undocumentedProductionDeclarations reports functions plus exported values and types that lack a name-first declaration comment. +func undocumentedProductionDeclarations(t *testing.T, directory string) []string { + t.Helper() + entries, err := os.ReadDir(directory) + if err != nil { + t.Fatalf("read production declarations in %s: %v", directory, err) + } + var missing []string + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".go") || strings.HasSuffix(entry.Name(), "_test.go") { + continue + } + filename := filepath.Join(directory, entry.Name()) + set := token.NewFileSet() + file, err := parser.ParseFile(set, filename, nil, parser.ParseComments) + if err != nil { + t.Fatalf("parse production declarations for %s: %v", filename, err) + } + for _, declaration := range file.Decls { + switch typed := declaration.(type) { + case *ast.FuncDecl: + if !commentStartsWith(typed.Doc, typed.Name.Name) { + missing = append(missing, declarationLocation(set, filename, typed.Pos(), typed.Name.Name)) + } + case *ast.GenDecl: + for _, specification := range typed.Specs { + switch spec := specification.(type) { + case *ast.TypeSpec: + if ast.IsExported(spec.Name.Name) && !commentStartsWith(firstComment(spec.Doc, typed.Doc), spec.Name.Name) { + missing = append(missing, declarationLocation(set, filename, spec.Pos(), spec.Name.Name)) + } + case *ast.ValueSpec: + for _, name := range spec.Names { + if ast.IsExported(name.Name) && !commentStartsWith(firstComment(spec.Doc, typed.Doc), name.Name) { + missing = append(missing, declarationLocation(set, filename, name.Pos(), name.Name)) + } + } + } + } + } + } + } + sort.Strings(missing) + return missing +} + +// firstComment prefers a declaration-specific comment while accepting a name-first single declaration group comment. +func firstComment(primary, fallback *ast.CommentGroup) *ast.CommentGroup { + if primary != nil { + return primary + } + return fallback +} + +// commentStartsWith applies the repository's name-first rule after removing ordinary comment whitespace. +func commentStartsWith(comment *ast.CommentGroup, name string) bool { + if comment == nil { + return false + } + return strings.HasPrefix(strings.TrimSpace(comment.Text()), name+" ") +} + +// declarationLocation formats one stable diagnostic for a missing declaration comment. +func declarationLocation(set *token.FileSet, filename string, position token.Pos, name string) string { + location := set.Position(position) + return filepath.Base(filename) + ":" + strconv.Itoa(location.Line) + " " + name +} diff --git a/internal/architecture/imports_test.go b/internal/architecture/imports_test.go new file mode 100644 index 0000000..e72510c --- /dev/null +++ b/internal/architecture/imports_test.go @@ -0,0 +1,63 @@ +package architecture_test + +import ( + "go/parser" + "go/token" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" + "testing" +) + +// TestWorkflowDependencyDirection prevents the public compatibility package from becoming an inward engine dependency again. +func TestWorkflowDependencyDirection(t *testing.T) { + repository := repositoryRoot(t) + assertProductionImportsExclude(t, repository, map[string]struct{}{ + "github.com/goforj/queue/bus": {}, + }) + assertProductionImportsExclude(t, filepath.Join(repository, "internal", "workflow"), map[string]struct{}{ + "github.com/goforj/queue": {}, + "github.com/goforj/queue/bus": {}, + "github.com/goforj/queue/queuecore": {}, + }) +} + +// repositoryRoot resolves the module root from this test file instead of relying on the caller's working directory. +func repositoryRoot(t *testing.T) string { + t.Helper() + _, filename, _, ok := runtime.Caller(0) + if !ok { + t.Fatal("resolve architecture test path") + } + return filepath.Clean(filepath.Join(filepath.Dir(filename), "..", "..")) +} + +// assertProductionImportsExclude parses direct production files so tests and outward-facing subpackages remain free to exercise compatibility APIs. +func assertProductionImportsExclude(t *testing.T, directory string, forbidden map[string]struct{}) { + t.Helper() + entries, err := os.ReadDir(directory) + if err != nil { + t.Fatalf("read production package %s: %v", directory, err) + } + for _, entry := range entries { + if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".go") || strings.HasSuffix(entry.Name(), "_test.go") { + continue + } + filename := filepath.Join(directory, entry.Name()) + file, err := parser.ParseFile(token.NewFileSet(), filename, nil, parser.ImportsOnly) + if err != nil { + t.Fatalf("parse production imports for %s: %v", filename, err) + } + for _, spec := range file.Imports { + path, err := strconv.Unquote(spec.Path.Value) + if err != nil { + t.Fatalf("decode import in %s: %v", filename, err) + } + if _, blocked := forbidden[path]; blocked { + t.Errorf("forbidden production dependency: %s imports %s", filename, path) + } + } + } +} diff --git a/internal/driverbridge/bridge.go b/internal/driverbridge/bridge.go index 3bbb65f..86b1dba 100644 --- a/internal/driverbridge/bridge.go +++ b/internal/driverbridge/bridge.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/goforj/queue" + "github.com/goforj/queue/internal/observation" "github.com/goforj/queue/internal/runtimehook" ) @@ -18,6 +19,7 @@ type runtimeQueueBackend interface { queueBackend Register(jobType string, handler queue.Handler) StartWorkers(ctx context.Context) error + DrainWorkers(ctx context.Context) error } type workerBackend interface { @@ -30,6 +32,26 @@ type workerContextDecoratorSetter interface { SetHandlerContextDecorator(func(context.Context) context.Context) } +// NewObserverSink creates the shared observer instance that a driver must pass to its producer, worker, and root configuration. +func NewObserverSink(observers ...queue.Observer) queue.Observer { + if len(observers) == 1 { + if sink, ok := observers[0].(interface { + queue.Observer + Add(func(context.Context, queue.Event)) + HasObservers() bool + }); ok { + return sink + } + } + callbacks := make([]func(context.Context, queue.Event), 0, len(observers)) + for _, observer := range observers { + if observer != nil { + callbacks = append(callbacks, observer.Observe) + } + } + return observation.NewSink(callbacks...) +} + // NewQueueFromDriver builds a high-level *queue.Queue from a driver backend. // // The helper keeps driver modules off the public low-level constructor path while @@ -40,6 +62,7 @@ func NewQueueFromDriver( workerFactory func(workers int) (any, error), opts ...queue.Option, ) (*queue.Queue, error) { + cfg.Observer = NewObserverSink(cfg.Observer) driverBackend, err := adaptQueueBackend(backend) if err != nil { return nil, err @@ -71,6 +94,17 @@ func (a queueBackendAdapter) Dispatch(ctx context.Context, job queue.Job) error return a.inner.Dispatch(ctx, job) } func (a queueBackendAdapter) Shutdown(ctx context.Context) error { return a.inner.Shutdown(ctx) } + +// Ready preserves an optional backend readiness contract through the internal bridge. +func (a queueBackendAdapter) Ready(ctx context.Context) error { + if checker, ok := a.inner.(interface{ Ready(context.Context) error }); ok { + return checker.Ready(ctx) + } + if checker, ok := a.inner.(interface{ Preflight(context.Context) error }); ok { + return checker.Preflight(ctx) + } + return nil +} func (a queueBackendAdapter) Pause(ctx context.Context, queueName string) error { controller, ok := a.inner.(queue.QueueController) if !ok { @@ -104,6 +138,11 @@ func (a runtimeQueueBackendAdapter) Register(jobType string, handler queue.Handl func (a runtimeQueueBackendAdapter) StartWorkers(ctx context.Context) error { return a.inner.StartWorkers(ctx) } + +// DrainWorkers forwards the native backend's worker-drain lifecycle phase. +func (a runtimeQueueBackendAdapter) DrainWorkers(ctx context.Context) error { + return a.inner.DrainWorkers(ctx) +} func (a runtimeQueueBackendAdapter) Pause(ctx context.Context, queueName string) error { return a.queueBackendAdapter.Pause(ctx, queueName) } diff --git a/internal/driverbridge/bridge_test.go b/internal/driverbridge/bridge_test.go index 6cdb849..9e33a77 100644 --- a/internal/driverbridge/bridge_test.go +++ b/internal/driverbridge/bridge_test.go @@ -2,6 +2,7 @@ package driverbridge import ( "context" + "errors" "testing" "github.com/goforj/queue" @@ -86,11 +87,87 @@ func (b *nativeBackendStub) StartWorkers(context.Context) error { b.started = true return nil } + +// DrainWorkers completes the native bridge stub's worker-drain phase. +func (b *nativeBackendStub) DrainWorkers(context.Context) error { + return nil +} func (b *nativeBackendStub) Shutdown(context.Context) error { b.stopped = true return nil } +type preflightBackendStub struct { + externalBackendStub + err error + calls int +} + +// Preflight records one legacy backend readiness check. +func (b *preflightBackendStub) Preflight(context.Context) error { + b.calls++ + return b.err +} + +type readyBackendStub struct { + preflightBackendStub + readyErr error + readyCalls int +} + +// Ready records one canonical backend readiness check. +func (b *readyBackendStub) Ready(context.Context) error { + b.readyCalls++ + return b.readyErr +} + +// TestNewQueueFromDriverPreservesReadiness verifies optional health checks +// survive adaptation and canonical Ready takes precedence over legacy Preflight. +func TestNewQueueFromDriverPreservesReadiness(t *testing.T) { + preflightErr := errors.New("managed schema unavailable") + legacy := &preflightBackendStub{ + externalBackendStub: externalBackendStub{driver: queue.DriverDatabase}, + err: preflightErr, + } + legacyQueue, err := NewQueueFromDriver( + queue.Config{Driver: queue.DriverDatabase}, + legacy, + nil, + ) + if err != nil { + t.Fatalf("new queue from legacy-ready driver: %v", err) + } + if err := legacyQueue.Ready(context.Background()); !errors.Is(err, preflightErr) { + t.Fatalf("legacy readiness = %v, want %v", err, preflightErr) + } + if legacy.calls != 1 { + t.Fatalf("legacy readiness calls = %d, want 1", legacy.calls) + } + + readyErr := errors.New("backend not ready") + canonical := &readyBackendStub{ + preflightBackendStub: preflightBackendStub{ + externalBackendStub: externalBackendStub{driver: queue.DriverDatabase}, + err: errors.New("legacy readiness must not run"), + }, + readyErr: readyErr, + } + canonicalQueue, err := NewQueueFromDriver( + queue.Config{Driver: queue.DriverDatabase}, + canonical, + nil, + ) + if err != nil { + t.Fatalf("new queue from canonical-ready driver: %v", err) + } + if err := canonicalQueue.Ready(context.Background()); !errors.Is(err, readyErr) { + t.Fatalf("canonical readiness = %v, want %v", err, readyErr) + } + if canonical.readyCalls != 1 || canonical.calls != 0 { + t.Fatalf("readiness calls = canonical:%d legacy:%d, want 1/0", canonical.readyCalls, canonical.calls) + } +} + func TestNewQueueFromDriver_ExternalWorkerFactoryAndOptionalCapabilities(t *testing.T) { backend := &externalBackendStub{ driver: queue.DriverNATS, @@ -216,3 +293,38 @@ func TestNewQueueFromDriver_NativeBackendOptionalCapabilities(t *testing.T) { t.Fatal("expected native backend shutdown path to be used") } } + +// TestNewQueueFromDriverSharesObserverSink verifies late root options reach the same sink retained by driver workers. +func TestNewQueueFromDriverSharesObserverSink(t *testing.T) { + var configEvents []queue.Event + var optionEvents []queue.Event + sink := NewObserverSink(queue.ObserverFunc(func(_ context.Context, event queue.Event) { + configEvents = append(configEvents, event) + })) + + q, err := NewQueueFromDriver( + queue.Config{Driver: queue.DriverNATS, DefaultQueue: "default", Observer: sink}, + &externalBackendStub{driver: queue.DriverNATS}, + nil, + queue.WithObserver(queue.ObserverFunc(func(_ context.Context, event queue.Event) { + optionEvents = append(optionEvents, event) + })), + ) + if err != nil { + t.Fatalf("new queue from driver failed: %v", err) + } + if q.Driver() != queue.DriverNATS { + t.Fatalf("driver = %q, want nats", q.Driver()) + } + + queue.SafeObserve(context.Background(), sink, queue.Event{Kind: queue.EventRepublishFailed}) + if len(configEvents) != 1 || len(optionEvents) != 1 { + t.Fatalf("captured sink counts = config:%d option:%d, want 1/1", len(configEvents), len(optionEvents)) + } + if configEvents[0].EventID == "" || configEvents[0].EventID != optionEvents[0].EventID { + t.Fatalf("captured sink event identity differs: config=%+v option=%+v", configEvents[0], optionEvents[0]) + } + if configEvents[0].Layer != queue.EventLayerWorker { + t.Fatalf("republish_failed layer = %q, want worker", configEvents[0].Layer) + } +} diff --git a/internal/jobidentity/identity.go b/internal/jobidentity/identity.go new file mode 100644 index 0000000..41531fe --- /dev/null +++ b/internal/jobidentity/identity.go @@ -0,0 +1,23 @@ +// Package jobidentity centralizes logical payload normalization and telemetry correlation shared by queue and workflow layers. +package jobidentity + +import ( + "bytes" + "crypto/sha1" + "fmt" +) + +// CanonicalPayload normalizes legacy representations of an absent workflow payload. +func CanonicalPayload(payload []byte) []byte { + if len(payload) == 0 || bytes.Equal(payload, []byte("null")) { + return nil + } + return payload +} + +// ObservedKey returns the stable logical type-and-payload correlation used by telemetry. +func ObservedKey(jobType string, payload []byte) string { + payload = CanonicalPayload(payload) + hash := sha1.Sum(append([]byte(jobType+":"), payload...)) + return fmt.Sprintf("%x", hash[:]) +} diff --git a/internal/observation/schema.go b/internal/observation/schema.go new file mode 100644 index 0000000..3f55195 --- /dev/null +++ b/internal/observation/schema.go @@ -0,0 +1,5 @@ +package observation + +// EventSchemaVersion identifies the canonical observer event envelope emitted +// by queue, worker, and workflow layers. +const EventSchemaVersion = 1 diff --git a/internal/observation/sink.go b/internal/observation/sink.go new file mode 100644 index 0000000..9575b20 --- /dev/null +++ b/internal/observation/sink.go @@ -0,0 +1,75 @@ +// Package observation provides internal fan-out plumbing without depending on +// the queue package's public event model. +package observation + +import ( + "context" + "sync" +) + +// Sink fans one typed value out to observers that may be added during runtime construction. +type Sink[T any] struct { + mu sync.RWMutex + observers []func(context.Context, T) +} + +// NewSink creates a mutable typed observer sink. +func NewSink[T any](observers ...func(context.Context, T)) *Sink[T] { + sink := &Sink[T]{} + for _, observer := range observers { + sink.Add(observer) + } + return sink +} + +// Add appends an observer while preserving registration order for each emitted value. +func (s *Sink[T]) Add(observer func(context.Context, T)) { + if s == nil || observer == nil { + return + } + s.mu.Lock() + s.observers = append(s.observers, observer) + s.mu.Unlock() +} + +// HasObservers reports whether emitting through the sink can reach an observer. +func (s *Sink[T]) HasObservers() bool { + if s == nil { + return false + } + s.mu.RLock() + hasObservers := len(s.observers) > 0 + s.mu.RUnlock() + return hasObservers +} + +// Observe invokes a stable observer snapshot so observers may be added without blocking callbacks. +func (s *Sink[T]) Observe(ctx context.Context, value T) { + if s == nil { + return + } + if ctx == nil { + ctx = context.Background() + } + observers := s.snapshot() + for _, observer := range observers { + observe(ctx, observer, value) + } +} + +// snapshot avoids holding the sink lock while application callbacks execute. +func (s *Sink[T]) snapshot() []func(context.Context, T) { + s.mu.RLock() + defer s.mu.RUnlock() + observers := make([]func(context.Context, T), len(s.observers)) + copy(observers, s.observers) + return observers +} + +// observe isolates a panicking callback so later observers still receive the value. +func observe[T any](ctx context.Context, observer func(context.Context, T), value T) { + defer func() { + _ = recover() + }() + observer(ctx, value) +} diff --git a/internal/observation/sink_test.go b/internal/observation/sink_test.go new file mode 100644 index 0000000..6800411 --- /dev/null +++ b/internal/observation/sink_test.go @@ -0,0 +1,50 @@ +package observation + +import ( + "context" + "sync" + "sync/atomic" + "testing" +) + +// TestSinkSupportsLateConcurrentObservers verifies construction-time injection can be extended safely before and during event delivery. +func TestSinkSupportsLateConcurrentObservers(t *testing.T) { + sink := NewSink[int]() + if sink.HasObservers() { + t.Fatal("new sink unexpectedly has observers") + } + sink.Add(nil) + if sink.HasObservers() { + t.Fatal("nil callback unexpectedly enabled the sink") + } + + var calls atomic.Int64 + sink.Add(func(context.Context, int) { panic("isolated") }) + sink.Add(func(ctx context.Context, value int) { + if ctx == nil { + t.Error("observer received a nil context") + } + calls.Add(int64(value)) + }) + if !sink.HasObservers() { + t.Fatal("sink did not report registered observers") + } + sink.Observe(nil, 2) + if calls.Load() != 2 { + t.Fatalf("calls = %d, want 2 after panic-isolated delivery", calls.Load()) + } + + var workers sync.WaitGroup + for range 8 { + workers.Add(2) + go func() { + defer workers.Done() + sink.Add(func(context.Context, int) {}) + }() + go func() { + defer workers.Done() + sink.Observe(context.Background(), 1) + }() + } + workers.Wait() +} diff --git a/internal/readmecheck/readme_manual_snippets_test.go b/internal/readmecheck/readme_manual_snippets_test.go index 6ad12f6..84c8e11 100644 --- a/internal/readmecheck/readme_manual_snippets_test.go +++ b/internal/readmecheck/readme_manual_snippets_test.go @@ -4,7 +4,10 @@ import ( "context" "errors" "log" + "os" "os/signal" + "path/filepath" + "strings" "syscall" "testing" "time" @@ -12,19 +15,47 @@ import ( "github.com/goforj/queue" ) +// TestReadmeManualSnippetsCompile prevents curated README examples from drifting beyond public API signatures. func TestReadmeManualSnippetsCompile(t *testing.T) { - // Compile-check only. The helpers mirror curated manual README snippets that - // have drifted before (Dispatch/WithContext(ctx).Dispatch and handler signatures). + // The helpers mirror curated manual README snippets that have drifted before, + // including Dispatch/WithContext(ctx).Dispatch and handler signatures. _ = []any{ compileQuickStartQueueSnippet, compileQuickStartWorkflowSnippet, compileRunAsWorkerServiceSnippet, compileJobBuilderOptionsSnippet, compileMiddlewareSnippet, + compileObservabilitySnippet, + compileComposeObserversSnippet, compileFakeQueueSnippet, } + assertReadmeObserverSignatures(t) } +// assertReadmeObserverSignatures keeps the manual observer snippets tied to the compiled helpers. +func assertReadmeObserverSignatures(t *testing.T) { + t.Helper() + readmePath := filepath.Join("..", "..", "README.md") + contents, err := os.ReadFile(readmePath) + if err != nil { + t.Fatalf("read %s: %v", readmePath, err) + } + manual, _, found := strings.Cut(string(contents), "") + if !found { + t.Fatal("README is missing the generated API start marker") + } + + for _, signature := range []string{ + "queue.ObserverFunc(func(_ context.Context, event queue.Event)", + "queue.ObserverFunc(func(_ context.Context, e queue.Event)", + } { + if !strings.Contains(manual, signature) { + t.Fatalf("README manual observer snippet is missing compiled signature %q", signature) + } + } +} + +// compileQuickStartQueueSnippet pins the queue quick start to the supported dispatch and handler signatures. func compileQuickStartQueueSnippet(q *queue.Queue) { if q == nil { return @@ -48,6 +79,7 @@ func compileQuickStartQueueSnippet(q *queue.Queue) { ) } +// compileQuickStartWorkflowSnippet pins the workflow quick start to the supported builder signatures. func compileQuickStartWorkflowSnippet(q *queue.Queue) { q, _ = queue.NewWorkerpool(queue.WithWorkers(2)) @@ -73,6 +105,7 @@ func compileQuickStartWorkflowSnippet(q *queue.Queue) { _ = chainID } +// compileRunAsWorkerServiceSnippet pins the worker-service example to the supported lifecycle signatures. func compileRunAsWorkerServiceSnippet(q *queue.Queue) { if q == nil { return @@ -88,6 +121,7 @@ func compileRunAsWorkerServiceSnippet(q *queue.Queue) { } } +// compileJobBuilderOptionsSnippet pins the job-options example to the supported fluent builder API. func compileJobBuilderOptionsSnippet(q *queue.Queue) { if q == nil { return @@ -115,6 +149,7 @@ func compileJobBuilderOptionsSnippet(q *queue.Queue) { }) } +// compileMiddlewareSnippet pins the middleware example to the supported middleware contracts. func compileMiddlewareSnippet() { var errValidation = errors.New("validation failed") maintenanceMode := false @@ -145,6 +180,46 @@ func compileMiddlewareSnippet() { _ = q } +// compileObservabilitySnippet mirrors the README's basic observer composition example. +func compileObservabilitySnippet() { + collector := queue.NewStatsCollector() + observer := queue.MultiObserver( + collector, + queue.ObserverFunc(func(_ context.Context, event queue.Event) { + _ = event.Kind + }), + ) + + q, _ := queue.New(queue.Config{ + Driver: queue.DriverWorkerpool, + Observer: observer, + }) + _ = q +} + +// compileComposeObserversSnippet mirrors the README's multi-observer example. +func compileComposeObserversSnippet() { + events := make(chan queue.Event, 100) + collector := queue.NewStatsCollector() + observer := queue.MultiObserver( + collector, + queue.ChannelObserver{ + Events: events, + DropIfFull: true, + }, + queue.ObserverFunc(func(_ context.Context, e queue.Event) { + _ = e + }), + ) + + q, _ := queue.New(queue.Config{ + Driver: queue.DriverWorkerpool, + Observer: observer, + }) + _ = q +} + +// compileFakeQueueSnippet pins the fake-queue example to the supported testing API. func compileFakeQueueSnippet() { fake := queue.NewFake() fake.Register("emails:send", func(context.Context, queue.Job) error { return nil }) diff --git a/internal/uniqueness/memory.go b/internal/uniqueness/memory.go new file mode 100644 index 0000000..d44f446 --- /dev/null +++ b/internal/uniqueness/memory.go @@ -0,0 +1,112 @@ +// Package uniqueness provides shared identity-claim primitives for queue drivers. +package uniqueness + +import ( + "container/heap" + "sync" + "time" +) + +// MemoryStore holds TTL claims within one queue backend instance. +// Its zero value is ready for use. +type MemoryStore struct { + mu sync.Mutex + next uint64 + entries map[string]memoryClaim + expires memoryExpiryQueue +} + +type memoryClaim struct { + token uint64 + expiresAt time.Time + expiry *memoryExpiry +} + +type memoryExpiry struct { + key string + token uint64 + expiresAt time.Time + index int +} + +type memoryExpiryQueue []*memoryExpiry + +// Len returns the number of expirations waiting for reclamation. +func (q memoryExpiryQueue) Len() int { return len(q) } + +// Less keeps the earliest claim at the head so reclamation cost follows expired entries rather than live cardinality. +func (q memoryExpiryQueue) Less(i, j int) bool { return q[i].expiresAt.Before(q[j].expiresAt) } + +// Swap preserves heap indexes used to remove compensated claims immediately. +func (q memoryExpiryQueue) Swap(i, j int) { + q[i], q[j] = q[j], q[i] + q[i].index = i + q[j].index = j +} + +// Push appends an expiration through container/heap. +func (q *memoryExpiryQueue) Push(value any) { + expiry := value.(*memoryExpiry) + expiry.index = len(*q) + *q = append(*q, expiry) +} + +// Pop removes the latest heap slot after container/heap moves the minimum there. +func (q *memoryExpiryQueue) Pop() any { + old := *q + last := len(old) - 1 + expiry := old[last] + old[last] = nil + expiry.index = -1 + *q = old[:last] + return expiry +} + +// Acquire claims key for ttl and returns an ownership token when no live claim exists. +func (s *MemoryStore) Acquire(key string, ttl time.Duration) (uint64, bool) { + if key == "" || ttl <= 0 { + return 0, false + } + now := time.Now() + + s.mu.Lock() + defer s.mu.Unlock() + s.pruneExpiredLocked(now) + if current, ok := s.entries[key]; ok && current.expiresAt.After(now) { + return 0, false + } + if s.entries == nil { + s.entries = make(map[string]memoryClaim) + } + s.next++ + if s.next == 0 { + s.next++ + } + expiry := &memoryExpiry{key: key, token: s.next, expiresAt: now.Add(ttl)} + heap.Push(&s.expires, expiry) + s.entries[key] = memoryClaim{token: s.next, expiresAt: expiry.expiresAt, expiry: expiry} + return s.next, true +} + +// Release removes key only when token still owns its current claim. +func (s *MemoryStore) Release(key string, token uint64) { + if key == "" || token == 0 { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if current, ok := s.entries[key]; ok && current.token == token { + heap.Remove(&s.expires, current.expiry.index) + delete(s.entries, key) + } +} + +// pruneExpiredLocked reclaims every elapsed claim before evaluating a new acquisition. +func (s *MemoryStore) pruneExpiredLocked(now time.Time) { + for len(s.expires) > 0 && !s.expires[0].expiresAt.After(now) { + expiry := heap.Pop(&s.expires).(*memoryExpiry) + if current, ok := s.entries[expiry.key]; ok && current.token == expiry.token { + delete(s.entries, expiry.key) + } + } +} diff --git a/internal/uniqueness/memory_test.go b/internal/uniqueness/memory_test.go new file mode 100644 index 0000000..304e0dc --- /dev/null +++ b/internal/uniqueness/memory_test.go @@ -0,0 +1,86 @@ +package uniqueness + +import ( + "sync" + "sync/atomic" + "testing" + "time" +) + +// TestMemoryStoreAcquireRelease verifies claims expire and compensation cannot delete a newer owner. +func TestMemoryStoreAcquireRelease(t *testing.T) { + var store MemoryStore + first, ok := store.Acquire("critical:job", 20*time.Millisecond) + if !ok || first == 0 { + t.Fatal("first claim was not acquired") + } + if _, duplicate := store.Acquire("critical:job", time.Second); duplicate { + t.Fatal("live claim admitted a duplicate") + } + + time.Sleep(25 * time.Millisecond) + second, ok := store.Acquire("critical:job", time.Second) + if !ok || second == first { + t.Fatal("expired claim was not replaced by a new owner") + } + store.Release("critical:job", first) + if _, duplicate := store.Acquire("critical:job", time.Second); duplicate { + t.Fatal("stale compensation removed the current owner") + } + store.Release("critical:job", second) + if _, ok := store.Acquire("critical:job", time.Second); !ok { + t.Fatal("current owner could not release its claim") + } +} + +// TestMemoryStoreConcurrentAcquireHasOneOwner verifies instance-scoped callers cannot both cross one claim boundary. +func TestMemoryStoreConcurrentAcquireHasOneOwner(t *testing.T) { + var store MemoryStore + start := make(chan struct{}) + var wait sync.WaitGroup + var winners atomic.Int32 + for range 64 { + wait.Add(1) + go func() { + defer wait.Done() + <-start + if _, ok := store.Acquire("shared", time.Minute); ok { + winners.Add(1) + } + }() + } + close(start) + wait.Wait() + if winners.Load() != 1 { + t.Fatalf("concurrent claim winners = %d, want 1", winners.Load()) + } +} + +// TestMemoryStoreRejectsInvalidClaims verifies callers cannot create unbounded or anonymous entries. +func TestMemoryStoreRejectsInvalidClaims(t *testing.T) { + var store MemoryStore + if token, ok := store.Acquire("", time.Second); ok || token != 0 { + t.Fatalf("empty key claim = (%d, %t), want rejected", token, ok) + } + if token, ok := store.Acquire("job", 0); ok || token != 0 { + t.Fatalf("zero TTL claim = (%d, %t), want rejected", token, ok) + } +} + +// TestMemoryStoreReclaimsUnrelatedExpiredClaims verifies high-cardinality identities do not remain resident forever. +func TestMemoryStoreReclaimsUnrelatedExpiredClaims(t *testing.T) { + var store MemoryStore + if _, ok := store.Acquire("expired-a", time.Millisecond); !ok { + t.Fatal("expired-a claim was not acquired") + } + if _, ok := store.Acquire("expired-b", time.Millisecond); !ok { + t.Fatal("expired-b claim was not acquired") + } + time.Sleep(2 * time.Millisecond) + if _, ok := store.Acquire("live", time.Minute); !ok { + t.Fatal("live claim was not acquired") + } + if len(store.entries) != 1 || len(store.expires) != 1 { + t.Fatalf("resident claims = entries:%d expirations:%d, want 1/1", len(store.entries), len(store.expires)) + } +} diff --git a/internal/workflow/batch.go b/internal/workflow/batch.go new file mode 100644 index 0000000..d782090 --- /dev/null +++ b/internal/workflow/batch.go @@ -0,0 +1,757 @@ +package workflow + +import ( + "context" + "errors" + "fmt" + + "github.com/goforj/queue/busruntime" +) + +// BatchBuilder configures and dispatches an aggregate workflow. +type BatchBuilder interface { + // Name sets a display name for the batch. + Name(name string) BatchBuilder + // OnQueue applies a default queue to batch jobs that do not set one. + OnQueue(queue string) BatchBuilder + // AllowFailures keeps the batch running when individual jobs fail. + AllowFailures() BatchBuilder + // Progress registers a callback invoked as jobs complete. + Progress(fn func(ctx context.Context, st BatchState) error) BatchBuilder + // Then registers a callback invoked once when batch succeeds. + Then(fn func(ctx context.Context, st BatchState) error) BatchBuilder + // Catch registers a callback invoked when batch encounters a failure. + Catch(fn func(ctx context.Context, st BatchState, err error) error) BatchBuilder + // Finally registers a callback invoked once when batch reaches terminal state. + Finally(fn func(ctx context.Context, st BatchState) error) BatchBuilder + // Dispatch creates and starts the batch workflow. + Dispatch(ctx context.Context) (string, error) +} + +type batchBuilder struct { + r *runtime + jobs []Job + name string + queue string + allowFailed bool + progress func(ctx context.Context, st BatchState) error + then func(ctx context.Context, st BatchState) error + catch func(ctx context.Context, st BatchState, err error) error + finally func(ctx context.Context, st BatchState) error +} + +// Name retains an application-facing label alongside persisted batch state. +func (b *batchBuilder) Name(name string) BatchBuilder { b.name = name; return b } + +// OnQueue supplies a target only for batch jobs that do not already select one. +func (b *batchBuilder) OnQueue(queue string) BatchBuilder { + b.queue = queue + return b +} + +// AllowFailures records that terminal member failures should not cancel remaining work. +func (b *batchBuilder) AllowFailures() BatchBuilder { + b.allowFailed = true + return b +} + +// Progress retains the explicitly ephemeral progress closure for this process lifetime. +func (b *batchBuilder) Progress(fn func(ctx context.Context, st BatchState) error) BatchBuilder { + b.progress = fn + return b +} + +// Then retains the explicitly ephemeral successful-terminal closure for this process lifetime. +func (b *batchBuilder) Then(fn func(ctx context.Context, st BatchState) error) BatchBuilder { + b.then = fn + return b +} + +// Catch retains the explicitly ephemeral failure closure for this process lifetime. +func (b *batchBuilder) Catch(fn func(ctx context.Context, st BatchState, err error) error) BatchBuilder { + b.catch = fn + return b +} + +// Finally retains the explicitly ephemeral terminal closure for this process lifetime. +func (b *batchBuilder) Finally(fn func(ctx context.Context, st BatchState) error) BatchBuilder { + b.finally = fn + return b +} + +// Dispatch persists the complete batch before enqueueing its canonical jobs. +func (b *batchBuilder) Dispatch(ctx context.Context) (string, error) { + if len(b.jobs) == 0 { + return "", errors.New("batch requires at least one job") + } + batchID := newID("bat") + dispatchID := newID("dsp") + jobs := make([]BatchJob, 0, len(b.jobs)) + for _, job := range b.jobs { + wj, err := toStoredJob(job) + if err != nil { + return "", err + } + if b.queue != "" && wj.Options.Queue == "" { + wj.Options.Queue = b.queue + } + jobs = append(jobs, BatchJob{ + JobID: newID("job"), + Job: wj, + }) + } + if err := b.r.store.CreateBatch(ctx, BatchRecord{ + BatchID: batchID, + DispatchID: dispatchID, + Name: b.name, + Queue: b.queue, + AllowFailed: b.allowFailed, + Jobs: jobs, + CreatedAt: b.r.now(), + }); err != nil { + return "", err + } + + if !b.r.ephemeralCallbacksDisabled && (b.progress != nil || b.then != nil || b.catch != nil || b.finally != nil) { + b.r.mu.Lock() + b.r.batchCallbacks[batchID] = batchCallbacks{ + progress: b.progress, + then: b.then, + catch: b.catch, + finally: b.finally, + } + b.r.mu.Unlock() + } + + first := jobs[0] + b.r.emit(ctx, Event{SchemaVersion: eventSchemaVersion, EventID: newID("evt"), Kind: EventBatchStarted, DispatchID: dispatchID, BatchID: batchID, JobType: first.Job.Type, JobKey: storedJobEventKey(first.Job), Queue: first.Job.Options.Queue, Time: b.r.now()}) + var synchronousErr error + for _, job := range jobs { + if err := b.r.dispatchEnvelope(ctx, internalJobBatchJob, envelope{ + SchemaVersion: schemaVersion, + DispatchID: dispatchID, + Kind: "batch_job", + BatchID: batchID, + JobID: job.JobID, + Job: job.Job, + }); err != nil { + if executionErr, ok := acceptedDispatchExecutionError(err); ok { + if synchronousErr == nil { + synchronousErr = executionErr + } + if b.allowFailed && !busruntime.IsUncommitted(executionErr) { + continue + } + return batchID, executionErr + } + if st, stErr := b.r.store.GetBatch(ctx, batchID); stErr == nil && (st.Completed || st.Processed > 0 || st.Failed > 0) { + return batchID, err + } + if cancelErr := b.r.store.CancelBatch(ctx, batchID); cancelErr != nil { + return batchID, uncommittedMutationError("cancel batch after initial dispatch rejection", errors.Join(err, cancelErr)) + } + base := envelope{DispatchID: dispatchID, BatchID: batchID, Job: job.Job} + b.r.emit(ctx, Event{SchemaVersion: eventSchemaVersion, EventID: newID("evt"), Kind: EventBatchFailed, DispatchID: dispatchID, BatchID: batchID, JobID: job.JobID, JobType: job.Job.Type, JobKey: storedJobEventKey(job.Job), Queue: job.Job.Options.Queue, Time: b.r.now(), Err: err}) + b.r.emit(ctx, Event{SchemaVersion: eventSchemaVersion, EventID: newID("evt"), Kind: EventBatchCancelled, DispatchID: dispatchID, BatchID: batchID, JobID: job.JobID, JobType: job.Job.Type, JobKey: storedJobEventKey(job.Job), Queue: job.Job.Options.Queue, Time: b.r.now()}) + st, stErr := b.r.store.GetBatch(ctx, batchID) + if stErr != nil { + return batchID, errors.Join(err, uncommittedMutationError("read batch after initial dispatch rejection", stErr)) + } + b.r.prepareBatchTerminalCallbacks(batchID, false, st.Failed > 0) + catchErr := b.r.invokeCallbackInline(ctx, base, "batch_catch", err) + finallyErr := b.r.invokeCallbackInline(ctx, base, "batch_finally", nil) + b.r.cleanupBatchCallbacks(batchID) + return batchID, errors.Join(err, catchErr, finallyErr) + } + } + return batchID, synchronousErr +} + +type batchCallbacks struct { + progress func(ctx context.Context, st BatchState) error + then func(ctx context.Context, st BatchState) error + catch func(ctx context.Context, st BatchState, err error) error + finally func(ctx context.Context, st BatchState) error +} + +// errCallbackAlreadyInvoked suppresses duplicate terminal facts when a broker redelivers an already-claimed ephemeral callback. +var errCallbackAlreadyInvoked = errors.New("workflow callback already invoked") + +// errCallbackUnavailable reports an ephemeral callback whose owning process state no longer exists. +var errCallbackUnavailable = errors.New("workflow callback is unavailable") + +// errCallbackNotReady rejects callback delivery before its workflow reaches the required state. +var errCallbackNotReady = errors.New("workflow callback state is not ready") + +// prepareBatchTerminalCallbacks discards closures that cannot run for the selected terminal outcome. +func (r *runtime) prepareBatchTerminalCallbacks(batchID string, succeeded, hasFailures bool) { + r.mu.Lock() + callbacks, ok := r.batchCallbacks[batchID] + if ok { + callbacks.progress = nil + if succeeded && !hasFailures { + callbacks.catch = nil + } + if !succeeded { + callbacks.then = nil + } + if callbacks.progress == nil && callbacks.then == nil && callbacks.catch == nil && callbacks.finally == nil { + delete(r.batchCallbacks, batchID) + } else { + r.batchCallbacks[batchID] = callbacks + } + } + r.mu.Unlock() +} + +// finishBatchCallback clears only the closure that ran so concurrently scheduled terminal callbacks remain available. +func (r *runtime) finishBatchCallback(batchID, kind string) { + r.mu.Lock() + callbacks, ok := r.batchCallbacks[batchID] + if ok { + switch kind { + case "then": + callbacks.then = nil + case "catch": + callbacks.catch = nil + case "finally": + callbacks.finally = nil + } + if callbacks.progress == nil && callbacks.then == nil && callbacks.catch == nil && callbacks.finally == nil { + delete(r.batchCallbacks, batchID) + } else { + r.batchCallbacks[batchID] = callbacks + } + } + r.mu.Unlock() +} + +// cleanupBatchCallbacks removes terminal workflow entries that have no remaining configured closure. +func (r *runtime) cleanupBatchCallbacks(batchID string) { + r.mu.Lock() + callbacks, ok := r.batchCallbacks[batchID] + if ok && callbacks.progress == nil && callbacks.then == nil && callbacks.catch == nil && callbacks.finally == nil { + delete(r.batchCallbacks, batchID) + } + r.mu.Unlock() +} + +// dispatchBatchTerminal publishes one aggregate terminal outcome regardless of which job finishes last. +func (r *runtime) dispatchBatchTerminal(ctx context.Context, env envelope, st BatchState) { + succeeded := st.Completed && !st.Cancelled + if succeeded { + r.emit(ctx, Event{SchemaVersion: eventSchemaVersion, EventID: batchFactID(EventBatchCompleted, env), Kind: EventBatchCompleted, DispatchID: env.DispatchID, BatchID: env.BatchID, JobID: env.JobID, JobType: env.Job.Type, JobKey: storedJobEventKey(env.Job), Queue: env.Job.Options.Queue, Time: r.now()}) + } + r.dispatchBatchTerminalCallbacks(ctx, env, st) +} + +// dispatchBatchTerminalCallbacks retries only idempotently claimed callbacks +// when aggregate state committed before their earlier enqueue completed. +func (r *runtime) dispatchBatchTerminalCallbacks(ctx context.Context, env envelope, st BatchState) { + succeeded := st.Completed && !st.Cancelled + r.prepareBatchTerminalCallbacks(env.BatchID, succeeded, st.Failed > 0) + if succeeded { + _ = r.dispatchCallback(ctx, env, "batch_then", nil) + } + _ = r.dispatchCallback(ctx, env, "batch_finally", nil) + r.cleanupBatchCallbacks(env.BatchID) +} + +// batchFactID includes the retained queue-row job identity because the +// established aggregate store persists member outcome but not member payload. +func batchFactID(kind EventKind, env envelope) string { + return stableWorkflowFactID(kind, env.DispatchID, env.BatchID, env.JobID, env.Job.Type, storedJobEventKey(env.Job), env.Job.Options.Queue) +} + +// settleBatchJob uses first-writer outcome ownership when the store supports +// it and preserves the established compatibility path for custom stores. +func (r *runtime) settleBatchJob(ctx context.Context, batchID, jobID string, outcome BatchJobOutcome, cause error, claim transitionClaim) (batchSettlementResult, error) { + if store, ok := r.store.(batchSettlementStore); ok { + return store.settleBatchOutcome(ctx, batchID, jobID, outcome, cause, claim) + } + if store, ok := r.store.(outcomeStore); ok { + state, owned, err := store.SettleBatchJob(ctx, batchID, jobID, outcome, cause) + return batchSettlementResult{state: state, owned: owned, claimedNow: true}, err + } + switch outcome { + case BatchJobSucceeded: + state, _, err := r.store.MarkBatchJobSucceeded(ctx, batchID, jobID) + return batchSettlementResult{state: state, owned: true, claimedNow: true}, err + case BatchJobFailed: + state, _, err := r.store.MarkBatchJobFailed(ctx, batchID, jobID, cause) + return batchSettlementResult{state: state, owned: true, claimedNow: true}, err + default: + return batchSettlementResult{}, errors.New("unsupported batch job outcome") + } +} + +// batchSettlementOwnsTerminal preserves compatibility for established stores +// while requiring built-in receipt-backed settlements to prove that this exact +// member crossed its parent into the terminal state it now reports. +func batchSettlementOwnsTerminal(settled batchSettlementResult, outcome BatchJobOutcome) bool { + if !settled.state.Completed { + return false + } + if !settled.receiptKnown { + return true + } + return settled.receipt.supported() && + settled.receipt.outcome == outcome && + settled.receipt.aggregateCompleted && + settled.receipt.aggregateCancelled == settled.state.Cancelled +} + +// emitCommittedBatchSuccessFacts publishes the success category already owned +// by one member without coupling fact recovery to application callbacks. +func (r *runtime) emitCommittedBatchSuccessFacts(ctx context.Context, env envelope, outcome storedJobOutcome) { + committedOutcome := outcome + committedOutcome.err = nil + r.emitStoredJobOutcome(ctx, committedOutcome) + r.emit(ctx, Event{SchemaVersion: eventSchemaVersion, EventID: batchFactID(EventBatchProgressed, env), Kind: EventBatchProgressed, DispatchID: env.DispatchID, BatchID: env.BatchID, JobID: env.JobID, JobType: env.Job.Type, JobKey: storedJobEventKey(env.Job), Queue: env.Job.Options.Queue, Time: r.now()}) +} + +// validateRecoveredBatchState rejects aggregate representations that cannot +// safely support reconstruction of a persisted member success. +func validateRecoveredBatchState(env envelope, state BatchState) error { + if state.BatchID != env.BatchID { + return fmt.Errorf("requested batch %q returned state for %q", env.BatchID, state.BatchID) + } + if state.DispatchID != "" && env.DispatchID != "" && state.DispatchID != env.DispatchID { + return fmt.Errorf("batch %q dispatch mismatch", env.BatchID) + } + if state.Total <= 0 || state.Pending < 0 || state.Processed < 0 || state.Failed < 0 || state.Pending+state.Processed != state.Total || state.Failed > state.Processed { + return fmt.Errorf("batch %q has inconsistent counters", env.BatchID) + } + if !state.Completed && state.Pending == 0 { + return fmt.Errorf("batch %q exhausted pending members without completing", env.BatchID) + } + if state.Completed && !state.Cancelled && state.Pending != 0 { + return fmt.Errorf("batch %q completed with pending members", env.BatchID) + } + return nil +} + +// validateRecoveredBatchReceiptShape rejects terminal ownership that cannot +// have been produced atomically with the aggregate state it now describes. +func validateRecoveredBatchReceiptShape(state BatchState, receipt transitionReceipt) error { + if receipt.aggregateCancelled && !receipt.aggregateCompleted { + return errors.New("batch transition receipt cancellation is not completed") + } + if receipt.aggregateCancelled && receipt.outcome != BatchJobFailed { + return errors.New("batch transition receipt cancellation does not own failure") + } + if !receipt.aggregateCompleted { + return nil + } + if !state.Completed { + return errors.New("batch transition receipt owns completion for nonterminal state") + } + if receipt.aggregateCancelled != state.Cancelled { + return errors.New("batch transition receipt cancellation does not match aggregate state") + } + return nil +} + +// recoverCommittedBatchTransition handles a receipt-backed settled member before +// application code runs. Aggregate completion is reconstructed only when a +// separate receipt identifies this member as the transaction that completed it. +func (r *runtime) recoverCommittedBatchTransition(ctx context.Context, env envelope) (bool, error) { + provenance, recovering := recoveredDeliveryProvenance(ctx) + if !recovering { + return false, nil + } + state, stateErr := r.store.GetBatch(ctx, env.BatchID) + if stateErr != nil { + return true, uncommittedMutationError("recover committed batch transition", stateErr) + } + if err := validateRecoveredBatchState(env, state); err != nil { + return true, uncommittedMutationError("recover committed batch transition", err) + } + receiptStore, capable := r.store.(transitionReceiptStore) + if !capable { + return false, nil + } + receipt, receiptKnown, receiptErr := receiptStore.batchTransitionReceipt(ctx, env.BatchID, env.JobID) + if receiptErr != nil { + return true, uncommittedMutationError("recover committed batch transition", receiptErr) + } + if !receiptKnown { + return false, nil + } + if receipt.workflowKind != batchTransitionKind || receipt.workflowID != env.BatchID || receipt.memberID != env.JobID || receipt.workflowDispatchID != state.DispatchID || !receipt.workflowCreatedAt.Equal(state.CreatedAt) { + return true, uncommittedMutationError("recover committed batch transition", errors.New("transition receipt does not match batch state")) + } + if err := validateRecoveredTransitionReceipt(env, receipt, true); err != nil { + return true, uncommittedMutationError("recover committed batch transition", err) + } + if err := validateRecoveredBatchReceiptShape(state, receipt); err != nil { + return true, uncommittedMutationError("recover committed batch transition", err) + } + exactFactOwner := transitionReceiptOwnsRecoveredFacts(env, receipt, provenance) + var settlementErr error + switch receipt.outcome { + case BatchJobSucceeded: + if !exactFactOwner { + return true, nil + } + case BatchJobFailed: + // The receipt intentionally omits application error details, but its + // terminal classification must survive every physical redelivery. + settlementErr = busruntime.Permanent(fmt.Errorf("batch %q member %q was already committed as failed; original cause was not persisted", env.BatchID, env.JobID)) + default: + return true, uncommittedMutationError("recover committed batch transition", fmt.Errorf("unsupported transition receipt outcome %q", receipt.outcome)) + } + if receipt.outcome == BatchJobSucceeded { + committedOutcome, recoveryErr := recoveredStoredJobSuccess(storedJobOutcome{env: env}, receipt, r.now()) + if recoveryErr != nil { + return true, uncommittedMutationError("recover committed batch transition", recoveryErr) + } + r.emitCommittedBatchSuccessFacts(ctx, env, committedOutcome) + } + if exactFactOwner && receipt.aggregateCompleted && !receipt.aggregateCancelled && state.Pending == 0 && state.Completed && !state.Cancelled { + r.emit(ctx, Event{SchemaVersion: eventSchemaVersion, EventID: batchFactID(EventBatchCompleted, env), Kind: EventBatchCompleted, DispatchID: env.DispatchID, BatchID: env.BatchID, JobID: env.JobID, JobType: env.Job.Type, JobKey: storedJobEventKey(env.Job), Queue: env.Job.Options.Queue, Time: r.now()}) + } + return true, settlementErr +} + +// handleInternalBatchJob records each batch mutation before publishing its corresponding workflow fact. +func (r *runtime) handleInternalBatchJob(ctx context.Context, job busruntime.InboundJob) error { + var env envelope + if err := job.Bind(&env); err != nil { + return err + } + if _, recovering := recoveredDeliveryProvenance(ctx); recovering { + applyDeliveryAttempt(ctx, &env) + handled, recoveryErr := r.recoverCommittedBatchTransition(ctx, env) + if recoveryErr != nil || handled { + return recoveryErr + } + } + progress := r.batchProgressCallback(env.BatchID) + if markErr := r.store.MarkBatchJobStarted(ctx, env.BatchID, env.JobID); markErr != nil { + return uncommittedMutationError("mark batch job started", markErr) + } + + outcome := r.executeStoredJobAttempt(ctx, env) + switch busruntime.ClassifyAttempt(outcome.attempt, outcome.err) { + case busruntime.AttemptRetry, busruntime.AttemptRedeliver: + return outcome.err + case busruntime.AttemptFailed: + settled, markErr := r.settleBatchJob(ctx, env.BatchID, env.JobID, BatchJobFailed, outcome.err, transitionClaimFromOutcome(ctx, outcome)) + if markErr != nil { + return uncommittedMutationError("mark batch job failed", markErr) + } + markDeliveryTransitionCommitted(ctx, settled.claimedNow, settled.receiptKnown) + if !settled.owned { + if _, recovered := recoveredDeliveryProvenance(ctx); !recovered { + return nil + } + _, recoveryErr := r.recoverCommittedBatchTransition(ctx, outcome.env) + return recoveryErr + } + ownsTerminal := batchSettlementOwnsTerminal(settled, BatchJobFailed) + if !settled.claimedNow { + if _, recovered := recoveredDeliveryProvenance(ctx); recovered { + return nil + } + if settled.state.Failed == 1 { + _ = r.dispatchCallback(ctx, env, "batch_catch", outcome.err) + } + if ownsTerminal { + r.dispatchBatchTerminalCallbacks(ctx, env, settled.state) + } + return nil + } + st := settled.state + r.emitStoredJobOutcome(ctx, outcome) + r.emit(ctx, Event{SchemaVersion: eventSchemaVersion, EventID: batchFactID(EventBatchProgressed, env), Kind: EventBatchProgressed, DispatchID: env.DispatchID, BatchID: env.BatchID, JobID: env.JobID, JobType: env.Job.Type, JobKey: storedJobEventKey(env.Job), Queue: env.Job.Options.Queue, Time: r.now(), Err: outcome.err}) + if ownsTerminal && st.Cancelled { + r.emit(ctx, Event{SchemaVersion: eventSchemaVersion, EventID: newID("evt"), Kind: EventBatchFailed, DispatchID: env.DispatchID, BatchID: env.BatchID, JobID: env.JobID, JobType: env.Job.Type, JobKey: storedJobEventKey(env.Job), Queue: env.Job.Options.Queue, Time: r.now(), Err: outcome.err}) + r.emit(ctx, Event{SchemaVersion: eventSchemaVersion, EventID: newID("evt"), Kind: EventBatchCancelled, DispatchID: env.DispatchID, BatchID: env.BatchID, JobID: env.JobID, JobType: env.Job.Type, JobKey: storedJobEventKey(env.Job), Queue: env.Job.Options.Queue, Time: r.now()}) + } + if st.Failed == 1 { + _ = r.dispatchCallback(ctx, env, "batch_catch", outcome.err) + } + r.invokeBatchProgress(ctx, st, progress) + if ownsTerminal { + r.dispatchBatchTerminal(ctx, env, st) + } + return outcome.err + } + settled, markErr := r.settleBatchJob(ctx, env.BatchID, env.JobID, BatchJobSucceeded, nil, transitionClaimFromOutcome(ctx, outcome)) + if markErr != nil { + return uncommittedMutationError("mark batch job succeeded", markErr) + } + markDeliveryTransitionCommitted(ctx, settled.claimedNow, settled.receiptKnown) + if !settled.owned { + return nil + } + ownsTerminal := batchSettlementOwnsTerminal(settled, BatchJobSucceeded) + if !settled.claimedNow { + if _, recovered := recoveredDeliveryProvenance(ctx); recovered { + _, recoveryErr := r.recoverCommittedBatchTransition(ctx, outcome.env) + return recoveryErr + } + if ownsTerminal { + r.dispatchBatchTerminalCallbacks(ctx, env, settled.state) + } + return nil + } + st := settled.state + r.emitCommittedBatchSuccessFacts(ctx, env, outcome) + r.invokeBatchProgress(ctx, st, progress) + if ownsTerminal { + r.dispatchBatchTerminal(ctx, env, st) + } + return nil +} + +// batchProgressCallback snapshots an in-flight job's hook before another completion can prepare terminal callbacks. +func (r *runtime) batchProgressCallback(batchID string) func(context.Context, BatchState) error { + r.mu.RLock() + progress := r.batchCallbacks[batchID].progress + r.mu.RUnlock() + return progress +} + +// invokeBatchProgress runs the snapshotted ephemeral progress hook without treating it as durable state. +func (r *runtime) invokeBatchProgress(ctx context.Context, st BatchState, progress func(context.Context, BatchState) error) { + if progress != nil { + _ = runEphemeralCallback(func() error { return progress(ctx, st) }) + } +} + +// invokeBatchThen claims the successful terminal callback before application code can run. +func (r *runtime) invokeBatchThen(ctx context.Context, st BatchState) error { + return r.invokeBatchThenObserved(ctx, st, nil) +} + +// invokeBatchThenObserved emits lifecycle start only after state validation and idempotency claim succeed. +func (r *runtime) invokeBatchThenObserved(ctx context.Context, st BatchState, onClaimed func()) error { + if !st.Completed || st.Cancelled { + return errCallbackNotReady + } + key := "batch_then:" + st.BatchID + ok, onceErr := r.callbackOnce(ctx, key) + if onceErr != nil { + return onceErr + } + if !ok { + return errCallbackAlreadyInvoked + } + if onClaimed != nil { + onClaimed() + } + r.mu.RLock() + cb := r.batchCallbacks[st.BatchID] + r.mu.RUnlock() + if cb.then != nil { + defer r.finishBatchCallback(st.BatchID, "then") + return runEphemeralCallback(func() error { return cb.then(ctx, st) }) + } + return errCallbackUnavailable +} + +// invokeBatchCatch claims the failure callback before application code can run. +func (r *runtime) invokeBatchCatch(ctx context.Context, st BatchState, err error) error { + return r.invokeBatchCatchObserved(ctx, st, err, nil) +} + +// invokeBatchCatchObserved emits lifecycle start only after state validation and idempotency claim succeed. +func (r *runtime) invokeBatchCatchObserved(ctx context.Context, st BatchState, err error, onClaimed func()) error { + if st.Failed <= 0 && !st.Cancelled { + return errCallbackNotReady + } + key := "batch_catch:" + st.BatchID + ok, onceErr := r.callbackOnce(ctx, key) + if onceErr != nil { + return onceErr + } + if !ok { + return errCallbackAlreadyInvoked + } + if onClaimed != nil { + onClaimed() + } + r.mu.RLock() + cb := r.batchCallbacks[st.BatchID] + r.mu.RUnlock() + if cb.catch != nil { + defer r.finishBatchCallback(st.BatchID, "catch") + return runEphemeralCallback(func() error { return cb.catch(ctx, st, err) }) + } + return errCallbackUnavailable +} + +// invokeBatchFinally claims the terminal closure before application code can run. +func (r *runtime) invokeBatchFinally(ctx context.Context, st BatchState) error { + return r.invokeBatchFinallyObserved(ctx, st, nil) +} + +// invokeBatchFinallyObserved emits lifecycle start only after state validation and idempotency claim succeed. +func (r *runtime) invokeBatchFinallyObserved(ctx context.Context, st BatchState, onClaimed func()) error { + if !st.Completed { + return errCallbackNotReady + } + key := "batch_finally:" + st.BatchID + ok, onceErr := r.callbackOnce(ctx, key) + if onceErr != nil { + return onceErr + } + if !ok { + return errCallbackAlreadyInvoked + } + if onClaimed != nil { + onClaimed() + } + r.mu.RLock() + cb := r.batchCallbacks[st.BatchID] + r.mu.RUnlock() + if cb.finally == nil { + return errCallbackUnavailable + } + defer r.finishBatchCallback(st.BatchID, "finally") + return runEphemeralCallback(func() error { return cb.finally(ctx, st) }) +} + +// callbackOnce persists callback idempotency before invoking application code. +func (r *runtime) callbackOnce(ctx context.Context, key string) (bool, error) { + marked, err := r.store.MarkCallbackInvoked(ctx, key) + if err != nil { + return false, uncommittedMutationError("mark callback invoked", err) + } + return marked, nil +} + +// handleInternalCallback separates application callback failures from uncommitted store access. +func (r *runtime) handleInternalCallback(ctx context.Context, job busruntime.InboundJob) error { + var env envelope + if err := job.Bind(&env); err != nil { + return err + } + return r.handleCallbackEnvelope(ctx, env) +} + +// handleCallbackEnvelope validates, claims, invokes, and observes one decoded callback delivery. +func (r *runtime) handleCallbackEnvelope(ctx context.Context, env envelope) error { + cbErr := error(nil) + if env.Error != "" { + cbErr = errors.New(env.Error) + } + start := r.now() + onClaimed := func() { + start = r.now() + r.emit(ctx, Event{ + SchemaVersion: eventSchemaVersion, + EventID: newID("evt"), + Kind: EventCallbackStarted, + DispatchID: env.DispatchID, + JobID: env.JobID, + ChainID: env.ChainID, + BatchID: env.BatchID, + JobType: env.Job.Type, + JobKey: storedJobEventKey(env.Job), + Queue: env.Job.Options.Queue, + Time: start, + }) + } + var err error + switch env.CallbackKind { + case "chain_catch": + if env.ChainID == "" { + err = errors.New("chain callback requires chain_id") + break + } + st, stErr := r.store.GetChain(ctx, env.ChainID) + if stErr != nil { + err = uncommittedMutationError("read chain callback state", stErr) + break + } + err = r.invokeChainCatchObserved(ctx, st, cbErr, onClaimed) + case "chain_finally": + if env.ChainID == "" { + err = errors.New("chain callback requires chain_id") + break + } + st, stErr := r.store.GetChain(ctx, env.ChainID) + if stErr != nil { + err = uncommittedMutationError("read chain callback state", stErr) + break + } + err = r.invokeChainFinallyObserved(ctx, st, onClaimed) + case "batch_catch": + if env.BatchID == "" { + err = errors.New("batch callback requires batch_id") + break + } + st, stErr := r.store.GetBatch(ctx, env.BatchID) + if stErr != nil { + err = uncommittedMutationError("read batch callback state", stErr) + break + } + err = r.invokeBatchCatchObserved(ctx, st, cbErr, onClaimed) + case "batch_then": + if env.BatchID == "" { + err = errors.New("batch callback requires batch_id") + break + } + st, stErr := r.store.GetBatch(ctx, env.BatchID) + if stErr != nil { + err = uncommittedMutationError("read batch callback state", stErr) + break + } + err = r.invokeBatchThenObserved(ctx, st, onClaimed) + case "batch_finally": + if env.BatchID == "" { + err = errors.New("batch callback requires batch_id") + break + } + st, stErr := r.store.GetBatch(ctx, env.BatchID) + if stErr != nil { + err = uncommittedMutationError("read batch callback state", stErr) + break + } + err = r.invokeBatchFinallyObserved(ctx, st, onClaimed) + default: + err = errors.New("unknown callback kind") + } + if err != nil { + if errors.Is(err, errCallbackAlreadyInvoked) { + return nil + } + if busruntime.IsUncommitted(err) { + return err + } + r.emit(ctx, Event{ + SchemaVersion: eventSchemaVersion, + EventID: newID("evt"), + Kind: EventCallbackFailed, + DispatchID: env.DispatchID, + JobID: env.JobID, + ChainID: env.ChainID, + BatchID: env.BatchID, + JobType: env.Job.Type, + JobKey: storedJobEventKey(env.Job), + Queue: env.Job.Options.Queue, + Duration: r.now().Sub(start), + Time: r.now(), + Err: err, + }) + return err + } + r.emit(ctx, Event{ + SchemaVersion: eventSchemaVersion, + EventID: newID("evt"), + Kind: EventCallbackSucceeded, + DispatchID: env.DispatchID, + JobID: env.JobID, + ChainID: env.ChainID, + BatchID: env.BatchID, + JobType: env.Job.Type, + JobKey: storedJobEventKey(env.Job), + Queue: env.Job.Options.Queue, + Duration: r.now().Sub(start), + Time: r.now(), + }) + return nil +} diff --git a/internal/workflow/chain.go b/internal/workflow/chain.go new file mode 100644 index 0000000..1603ae6 --- /dev/null +++ b/internal/workflow/chain.go @@ -0,0 +1,687 @@ +package workflow + +import ( + "bytes" + "context" + "errors" + "fmt" + "sync" + + "github.com/goforj/queue/busruntime" +) + +// ChainBuilder configures and dispatches a sequential workflow. +type ChainBuilder interface { + // OnQueue applies a default queue to chain jobs that do not set one. + OnQueue(queue string) ChainBuilder + // Catch registers a callback invoked when chain execution fails. + Catch(fn func(ctx context.Context, st ChainState, err error) error) ChainBuilder + // Finally registers a callback invoked once when chain execution finishes. + Finally(fn func(ctx context.Context, st ChainState) error) ChainBuilder + // Dispatch creates and starts the chain workflow. + Dispatch(ctx context.Context) (string, error) +} + +type chainBuilder struct { + r *runtime + jobs []Job + queue string + catch func(ctx context.Context, st ChainState, err error) error + done func(ctx context.Context, st ChainState) error +} + +type synchronousChainResultContextKey struct{} + +type synchronousChainResult struct { + mu sync.Mutex + err error +} + +// withSynchronousChainResult lets inline continuation deliveries report their +// execution error without turning it into the predecessor's delivery outcome. +func withSynchronousChainResult(ctx context.Context) (context.Context, *synchronousChainResult) { + if ctx == nil { + ctx = context.Background() + } + result := &synchronousChainResult{} + return context.WithValue(ctx, synchronousChainResultContextKey{}, result), result +} + +// record stores the first downstream error because it is the causal terminal +// outcome observed by the caller that started this inline chain execution. +func (r *synchronousChainResult) record(err error) { + if r == nil || err == nil { + return + } + r.mu.Lock() + defer r.mu.Unlock() + if r.err == nil { + r.err = err + } +} + +// executionError returns the exact downstream error so errors.Is and errors.As +// retain the application's original error chain. +func (r *synchronousChainResult) executionError() error { + if r == nil { + return nil + } + r.mu.Lock() + defer r.mu.Unlock() + return r.err +} + +// recordSynchronousChainError propagates an inline continuation failure to the +// chain dispatch boundary when the caller is still waiting for execution. +func recordSynchronousChainError(ctx context.Context, err error) { + if ctx == nil { + return + } + result, _ := ctx.Value(synchronousChainResultContextKey{}).(*synchronousChainResult) + result.record(err) +} + +// OnQueue supplies a target only for chain jobs that do not already select one. +func (b *chainBuilder) OnQueue(queue string) ChainBuilder { + b.queue = queue + return b +} + +// Catch retains the explicitly ephemeral failure closure for this process lifetime. +func (b *chainBuilder) Catch(fn func(ctx context.Context, st ChainState, err error) error) ChainBuilder { + b.catch = fn + return b +} + +// Finally retains the explicitly ephemeral terminal closure for this process lifetime. +func (b *chainBuilder) Finally(fn func(ctx context.Context, st ChainState) error) ChainBuilder { + b.done = fn + return b +} + +// Dispatch persists every node before enqueueing the first canonical delivery. +func (b *chainBuilder) Dispatch(ctx context.Context) (string, error) { + if len(b.jobs) == 0 { + return "", errors.New("chain requires at least one job") + } + ctx, synchronousResult := withSynchronousChainResult(ctx) + chainID := newID("chn") + dispatchID := newID("dsp") + nodes := make([]ChainNode, 0, len(b.jobs)) + for i, job := range b.jobs { + wj, err := toStoredJob(job) + if err != nil { + return "", err + } + if b.queue != "" && wj.Options.Queue == "" { + wj.Options.Queue = b.queue + } + nodes = append(nodes, ChainNode{ + NodeID: nodeID(chainID, i), + Job: wj, + }) + } + if err := b.r.store.CreateChain(ctx, ChainRecord{ + ChainID: chainID, + DispatchID: dispatchID, + Queue: b.queue, + Nodes: nodes, + CreatedAt: b.r.now(), + }); err != nil { + return "", err + } + if !b.r.ephemeralCallbacksDisabled && (b.catch != nil || b.done != nil) { + b.r.mu.Lock() + b.r.chainCallbacks[chainID] = chainCallbacks{ + catch: b.catch, + finally: b.done, + } + b.r.mu.Unlock() + } + + first := nodes[0] + b.r.emit(ctx, Event{SchemaVersion: eventSchemaVersion, EventID: newID("evt"), Kind: EventChainStarted, DispatchID: dispatchID, ChainID: chainID, JobType: first.Job.Type, JobKey: storedJobEventKey(first.Job), Queue: first.Job.Options.Queue, Time: b.r.now()}) + if err := b.r.dispatchEnvelope(ctx, internalJobChainNode, envelope{ + SchemaVersion: schemaVersion, + DispatchID: dispatchID, + Kind: "chain_node", + ChainID: chainID, + NodeID: first.NodeID, + JobID: newID("job"), + Job: first.Job, + }); err != nil { + if executionErr, ok := acceptedDispatchExecutionError(err); ok { + return chainID, executionErr + } + _, owned, failErr := b.r.failChainNode(ctx, chainID, first.NodeID, err) + if failErr != nil { + return chainID, uncommittedMutationError("fail chain after initial dispatch rejection", errors.Join(err, failErr)) + } + if !owned { + return chainID, err + } + base := envelope{DispatchID: dispatchID, ChainID: chainID, Job: first.Job} + b.r.emit(ctx, Event{SchemaVersion: eventSchemaVersion, EventID: newID("evt"), Kind: EventChainFailed, DispatchID: dispatchID, ChainID: chainID, JobType: first.Job.Type, JobKey: storedJobEventKey(first.Job), Queue: first.Job.Options.Queue, Time: b.r.now(), Err: err}) + _, stErr := b.r.store.GetChain(ctx, chainID) + if stErr != nil { + return chainID, errors.Join(err, uncommittedMutationError("read chain after initial dispatch rejection", stErr)) + } + catchErr := b.r.invokeCallbackInline(ctx, base, "chain_catch", err) + finallyErr := b.r.invokeCallbackInline(ctx, base, "chain_finally", nil) + b.r.cleanupChainCallbacks(chainID) + return chainID, errors.Join(err, catchErr, finallyErr) + } + if executionErr := synchronousResult.executionError(); executionErr != nil { + return chainID, executionErr + } + return chainID, nil +} + +type chainCallbacks struct { + catch func(ctx context.Context, st ChainState, err error) error + finally func(ctx context.Context, st ChainState) error +} + +// prepareChainSuccessCallbacks discards the failure-only closure before a successful terminal callback is scheduled. +func (r *runtime) prepareChainSuccessCallbacks(chainID string) { + r.mu.Lock() + callbacks, ok := r.chainCallbacks[chainID] + if ok { + callbacks.catch = nil + r.chainCallbacks[chainID] = callbacks + } + r.mu.Unlock() +} + +// finishChainCallback clears only the closure that ran so concurrently scheduled terminal callbacks remain available. +func (r *runtime) finishChainCallback(chainID, kind string) { + r.mu.Lock() + callbacks, ok := r.chainCallbacks[chainID] + if ok { + switch kind { + case "catch": + callbacks.catch = nil + case "finally": + callbacks.finally = nil + } + if callbacks.catch == nil && callbacks.finally == nil { + delete(r.chainCallbacks, chainID) + } else { + r.chainCallbacks[chainID] = callbacks + } + } + r.mu.Unlock() +} + +// cleanupChainCallbacks removes terminal workflow entries that have no remaining configured closure. +func (r *runtime) cleanupChainCallbacks(chainID string) { + r.mu.Lock() + callbacks, ok := r.chainCallbacks[chainID] + if ok && callbacks.catch == nil && callbacks.finally == nil { + delete(r.chainCallbacks, chainID) + } + r.mu.Unlock() +} + +// nodeID combines chain ownership with random entropy so persisted completion markers cannot collide across chains. +func nodeID(chainID string, idx int) string { + return chainID + "_" + newID("n") +} + +// failChainNode uses the additive atomic capability when available while +// retaining a state-confirmed fallback for established custom stores. +func (r *runtime) failChainNode(ctx context.Context, chainID, nodeID string, cause error) (ChainState, bool, error) { + if store, ok := r.store.(outcomeStore); ok { + return store.FailChainNode(ctx, chainID, nodeID, cause) + } + state, err := r.store.GetChain(ctx, chainID) + if err != nil { + return ChainState{}, false, err + } + owned, claimable, err := chainNodeFailureDisposition(state, nodeID) + if err != nil || !claimable { + return state, owned, err + } + if err := r.store.FailChain(ctx, chainID, cause); err != nil { + return ChainState{}, false, err + } + state, err = r.store.GetChain(ctx, chainID) + if err != nil { + return ChainState{}, false, err + } + owned, claimable, err = chainNodeFailureDisposition(state, nodeID) + if err != nil { + return ChainState{}, false, err + } + if claimable { + return ChainState{}, false, errors.New("chain store accepted failure without terminal state") + } + return state, owned, nil +} + +// failChainNodeOutcome uses built-in receipt fencing when available while +// preserving the established arbitration behavior of decorated custom stores. +func (r *runtime) failChainNodeOutcome(ctx context.Context, chainID, nodeID string, cause error, claim transitionClaim) (chainFailureResult, error) { + if store, ok := r.store.(chainFailureStore); ok { + return store.failChainOutcome(ctx, chainID, nodeID, cause, claim) + } + state, owned, err := r.failChainNode(ctx, chainID, nodeID, cause) + return chainFailureResult{state: state, owned: owned, claimedNow: owned}, err +} + +// observedChainFailure preserves the committed cause across redelivery while +// retaining permanent classification without exposing a replayed cause. +func observedChainFailure(state ChainState, current error) error { + if state.Failure == "" || (current != nil && current.Error() == state.Failure) { + return current + } + committed := errors.New(state.Failure) + if busruntime.IsPermanent(current) { + return busruntime.Permanent(committed) + } + return committed +} + +// advanceChainNode uses built-in atomic ownership when available while +// retaining the established Store projection for compatibility implementations. +func (r *runtime) advanceChainNode(ctx context.Context, chainID, nodeID string, claim transitionClaim) (chainAdvanceResult, error) { + if store, ok := r.store.(chainAdvanceStore); ok { + return store.advanceChainOutcome(ctx, chainID, nodeID, claim) + } + next, done, err := r.store.AdvanceChain(ctx, chainID, nodeID) + if err != nil { + return chainAdvanceResult{}, err + } + return chainAdvanceResult{next: next, done: done, successOwned: true, claimedNow: true}, nil +} + +// storedJobsEqual compares persisted protocol identity before recovery trusts +// an envelope to reconstruct externally visible job correlation. +func storedJobsEqual(left, right StoredJob) bool { + return left.Type == right.Type && bytes.Equal(left.Payload, right.Payload) && left.Options == right.Options +} + +// chainFactID includes retained-row correlation so one deterministic ID never +// labels different event payloads when duplicate physical envelopes disagree. +func chainFactID(kind EventKind, env envelope) string { + return stableWorkflowFactID(kind, env.DispatchID, env.ChainID, env.NodeID, env.JobID, env.Job.Type, storedJobEventKey(env.Job), env.Job.Options.Queue) +} + +// recoverCommittedChainSuccessor preserves a committed predecessor's live +// continuation without reconstructing facts that require exact receipt ownership. +func (r *runtime) recoverCommittedChainSuccessor(ctx context.Context, env envelope, state ChainState, index int) error { + if state.Completed || state.Failed || state.NextIndex != index+1 { + return nil + } + next := state.Nodes[state.NextIndex] + return r.dispatchChainSuccessor(ctx, env, &next) +} + +// recoverCommittedChainSuccess handles a reclaimed row before application code +// runs when durable state proves the node already succeeded. Facts require a +// receipt owned by the exact unsettled generation; application effects are not +// replayed, while an immediate still-pending continuation remains recoverable. +func (r *runtime) recoverCommittedChainSuccess(ctx context.Context, env envelope) (bool, error) { + provenance, recovering := recoveredDeliveryProvenance(ctx) + if !recovering { + return false, nil + } + state, err := r.store.GetChain(ctx, env.ChainID) + if err != nil { + return true, uncommittedMutationError("recover committed chain success", err) + } + if state.ChainID != env.ChainID { + return true, uncommittedMutationError("recover committed chain success", fmt.Errorf("requested chain %q returned state for %q", env.ChainID, state.ChainID)) + } + if state.DispatchID != "" && env.DispatchID != "" && state.DispatchID != env.DispatchID { + return true, uncommittedMutationError("recover committed chain success", fmt.Errorf("chain %q dispatch mismatch", env.ChainID)) + } + index, known := chainNodePosition(state.Nodes, env.NodeID) + if !known { + return true, uncommittedMutationError("recover committed chain success", fmt.Errorf("chain %q does not contain node %q", env.ChainID, env.NodeID)) + } + successOwned, err := chainNodeSuccessDisposition(state, env.NodeID) + if err != nil { + return true, uncommittedMutationError("recover committed chain success", err) + } + if !storedJobsEqual(state.Nodes[index].Job, env.Job) { + return true, uncommittedMutationError("recover committed chain success", fmt.Errorf("chain %q node %q job mismatch", env.ChainID, env.NodeID)) + } + if !successOwned { + return false, nil + } + receiptStore, capable := r.store.(transitionReceiptStore) + if !capable { + return true, r.recoverCommittedChainSuccessor(ctx, env, state, index) + } + receipt, receiptKnown, receiptErr := receiptStore.chainTransitionReceipt(ctx, env.ChainID, env.NodeID) + if receiptErr != nil { + return true, uncommittedMutationError("recover committed chain success", receiptErr) + } + if !receiptKnown { + return true, r.recoverCommittedChainSuccessor(ctx, env, state, index) + } + if receipt.workflowKind != chainTransitionKind || receipt.workflowID != env.ChainID || receipt.memberID != env.NodeID || receipt.workflowDispatchID != state.DispatchID || !receipt.workflowCreatedAt.Equal(state.CreatedAt) { + return true, uncommittedMutationError("recover committed chain success", errors.New("transition receipt does not match chain state")) + } + if err := validateRecoveredTransitionReceipt(env, receipt, false); err != nil { + return true, uncommittedMutationError("recover committed chain success", err) + } + if receipt.outcome != BatchJobSucceeded { + return true, uncommittedMutationError("recover committed chain success", errors.New("transition receipt does not own success")) + } + if receipt.aggregateCancelled { + return true, uncommittedMutationError("recover committed chain success", errors.New("successful chain receipt cannot own cancellation")) + } + finalNode := index == len(state.Nodes)-1 + if receipt.aggregateCompleted != finalNode { + return true, uncommittedMutationError("recover committed chain success", errors.New("transition receipt completion does not match chain node position")) + } + if !transitionReceiptOwnsRecoveredFacts(env, receipt, provenance) { + return true, r.recoverCommittedChainSuccessor(ctx, env, state, index) + } + committedOutcome, recoveryErr := recoveredStoredJobSuccess(storedJobOutcome{env: env}, receipt, r.now()) + if recoveryErr != nil { + return true, uncommittedMutationError("recover committed chain success", recoveryErr) + } + if finalNode { + r.emitStoredJobOutcome(ctx, committedOutcome) + r.emit(ctx, Event{SchemaVersion: eventSchemaVersion, EventID: chainFactID(EventChainCompleted, env), Kind: EventChainCompleted, DispatchID: env.DispatchID, ChainID: env.ChainID, JobID: env.JobID, JobType: env.Job.Type, JobKey: storedJobEventKey(env.Job), Queue: env.Job.Options.Queue, Time: r.now()}) + return true, nil + } + + r.emitStoredJobOutcome(ctx, committedOutcome) + r.emit(ctx, Event{SchemaVersion: eventSchemaVersion, EventID: chainFactID(EventChainAdvanced, env), Kind: EventChainAdvanced, DispatchID: env.DispatchID, ChainID: env.ChainID, JobID: env.JobID, JobType: env.Job.Type, JobKey: storedJobEventKey(env.Job), Queue: env.Job.Options.Queue, Time: r.now()}) + if !state.Completed && !state.Failed && state.NextIndex == index+1 { + next := state.Nodes[state.NextIndex] + return true, r.dispatchChainSuccessor(ctx, env, &next) + } + return true, nil +} + +// recoverCommittedChainFailure settles a reclaimed terminal failure from its +// persisted cause without fabricating another handler occurrence or callback. +func (r *runtime) recoverCommittedChainFailure(ctx context.Context, env envelope) (bool, error) { + _, recovering := recoveredDeliveryProvenance(ctx) + if !recovering { + return false, nil + } + state, err := r.store.GetChain(ctx, env.ChainID) + if err != nil { + return true, uncommittedMutationError("recover committed chain failure", err) + } + if state.ChainID != env.ChainID { + return true, uncommittedMutationError("recover committed chain failure", fmt.Errorf("requested chain %q returned state for %q", env.ChainID, state.ChainID)) + } + if state.DispatchID != "" && env.DispatchID != "" && state.DispatchID != env.DispatchID { + return true, uncommittedMutationError("recover committed chain failure", fmt.Errorf("chain %q dispatch mismatch", env.ChainID)) + } + if !state.Failed || state.Completed { + return false, nil + } + index, known := chainNodePosition(state.Nodes, env.NodeID) + if !known { + return true, uncommittedMutationError("recover committed chain failure", fmt.Errorf("chain %q does not contain node %q", env.ChainID, env.NodeID)) + } + owned, _, err := chainNodeFailureDisposition(state, env.NodeID) + if err != nil { + return true, uncommittedMutationError("recover committed chain failure", err) + } + if !owned { + return false, nil + } + if !storedJobsEqual(state.Nodes[index].Job, env.Job) { + return true, uncommittedMutationError("recover committed chain failure", fmt.Errorf("chain %q node %q job mismatch", env.ChainID, env.NodeID)) + } + receiptStore, capable := r.store.(transitionReceiptStore) + if !capable { + return false, nil + } + receipt, receiptKnown, receiptErr := receiptStore.chainTransitionReceipt(ctx, env.ChainID, env.NodeID) + if receiptErr != nil { + return true, uncommittedMutationError("recover committed chain failure", receiptErr) + } + if !receiptKnown { + return false, nil + } + if receipt.workflowKind != chainTransitionKind || receipt.workflowID != env.ChainID || receipt.memberID != env.NodeID || receipt.workflowDispatchID != state.DispatchID || !receipt.workflowCreatedAt.Equal(state.CreatedAt) { + return true, uncommittedMutationError("recover committed chain failure", errors.New("transition receipt does not match chain state")) + } + if receipt.outcome != BatchJobFailed { + return true, uncommittedMutationError("recover committed chain failure", errors.New("transition receipt does not own failure")) + } + if receipt.aggregateCompleted || receipt.aggregateCancelled { + return true, uncommittedMutationError("recover committed chain failure", errors.New("failed chain receipt cannot own completion")) + } + if err := validateRecoveredTransitionReceipt(env, receipt, false); err != nil { + return true, uncommittedMutationError("recover committed chain failure", err) + } + if state.Failure == "" { + return true, busruntime.Permanent(fmt.Errorf("chain %q node %q was already committed as failed; original cause was empty", env.ChainID, env.NodeID)) + } + return true, busruntime.Permanent(errors.New(state.Failure)) +} + +// dispatchChainSuccessor retains at-least-once continuation recovery after a +// predecessor transition committed but its first enqueue did not complete. +// A surviving predecessor cannot distinguish a missing successor from one +// already enqueued but not yet progressed, so recovery may enqueue a duplicate +// under the queue's existing at-least-once contract. +func (r *runtime) dispatchChainSuccessor(ctx context.Context, env envelope, next *ChainNode) error { + if next == nil { + return uncommittedMutationError("dispatch next chain node", errors.New("chain store omitted successor")) + } + dispatchErr := r.dispatchEnvelope(ctx, internalJobChainNode, envelope{ + SchemaVersion: schemaVersion, + DispatchID: env.DispatchID, + Kind: "chain_node", + ChainID: env.ChainID, + NodeID: next.NodeID, + JobID: newID("job"), + Job: next.Job, + }) + if executionErr, ok := acceptedDispatchExecutionError(dispatchErr); ok { + recordSynchronousChainError(ctx, executionErr) + return nil + } + if dispatchErr != nil { + return uncommittedMutationError("dispatch next chain node", dispatchErr) + } + return nil +} + +// handleInternalChainNode advances or fails a chain only after its application attempt reaches a committable outcome. +func (r *runtime) handleInternalChainNode(ctx context.Context, job busruntime.InboundJob) error { + var env envelope + if err := job.Bind(&env); err != nil { + return err + } + if _, recovering := recoveredDeliveryProvenance(ctx); recovering { + applyDeliveryAttempt(ctx, &env) + handled, recoveryErr := r.recoverCommittedChainFailure(ctx, env) + if recoveryErr != nil || handled { + return recoveryErr + } + handled, recoveryErr = r.recoverCommittedChainSuccess(ctx, env) + if recoveryErr != nil || handled { + return recoveryErr + } + } + outcome := r.executeStoredJobAttempt(ctx, env) + switch busruntime.ClassifyAttempt(outcome.attempt, outcome.err) { + case busruntime.AttemptRetry, busruntime.AttemptRedeliver: + return outcome.err + case busruntime.AttemptFailed: + failed, markErr := r.failChainNodeOutcome(ctx, env.ChainID, env.NodeID, outcome.err, transitionClaimFromOutcome(ctx, outcome)) + if markErr != nil { + return uncommittedMutationError("fail chain", markErr) + } + markDeliveryTransitionCommitted(ctx, failed.claimedNow, failed.receiptKnown) + if !failed.owned { + if _, recovered := recoveredDeliveryProvenance(ctx); !recovered { + return nil + } + recovered, recoveryErr := r.recoverCommittedChainFailure(ctx, outcome.env) + if recoveryErr != nil || recovered { + return recoveryErr + } + recovered, recoveryErr = r.recoverCommittedChainSuccess(ctx, outcome.env) + if recoveryErr != nil || recovered { + return recoveryErr + } + return nil + } + if !failed.claimedNow { + if _, recovered := recoveredDeliveryProvenance(ctx); recovered { + recovered, recoveryErr := r.recoverCommittedChainFailure(ctx, outcome.env) + if recoveryErr != nil || recovered { + return recoveryErr + } + return outcome.err + } + return nil + } + state := failed.state + if state.Completed { + return nil + } + if !state.Failed { + return uncommittedMutationError("confirm chain failure", errors.New("chain store accepted failure without terminal state")) + } + observedErr := observedChainFailure(state, outcome.err) + observedOutcome := outcome + observedOutcome.err = observedErr + r.emitStoredJobOutcome(ctx, observedOutcome) + r.emit(ctx, Event{SchemaVersion: eventSchemaVersion, EventID: newID("evt"), Kind: EventChainFailed, DispatchID: env.DispatchID, ChainID: env.ChainID, JobID: env.JobID, JobType: env.Job.Type, JobKey: storedJobEventKey(env.Job), Queue: env.Job.Options.Queue, Time: r.now(), Err: observedErr}) + _ = r.dispatchCallback(ctx, env, "chain_catch", observedErr) + _ = r.dispatchCallback(ctx, env, "chain_finally", nil) + r.cleanupChainCallbacks(env.ChainID) + return outcome.err + } + advance, advErr := r.advanceChainNode(ctx, env.ChainID, env.NodeID, transitionClaimFromOutcome(ctx, outcome)) + if advErr != nil { + return uncommittedMutationError("advance chain", advErr) + } + markDeliveryTransitionCommitted(ctx, advance.claimedNow, advance.receiptKnown) + _, recovering := recoveredDeliveryProvenance(ctx) + if !advance.claimedNow { + if !advance.successOwned { + return nil + } + if recovering { + recovered, recoveryErr := r.recoverCommittedChainSuccess(ctx, outcome.env) + if recoveryErr != nil || recovered { + return recoveryErr + } + return uncommittedMutationError("recover committed chain success", errors.New("store reported success ownership without advanced state")) + } + if advance.done { + state := advance.state + index, known := chainNodePosition(state.Nodes, env.NodeID) + if known && state.Completed && index == len(state.Nodes)-1 { + r.prepareChainSuccessCallbacks(env.ChainID) + _ = r.dispatchCallback(ctx, env, "chain_finally", nil) + r.cleanupChainCallbacks(env.ChainID) + } + return nil + } + return r.dispatchChainSuccessor(ctx, env, advance.next) + } + next, done := advance.next, advance.done + if done { + state := advance.state + if state.ChainID == "" { + var stateErr error + state, stateErr = r.store.GetChain(ctx, env.ChainID) + if stateErr != nil { + return uncommittedMutationError("confirm chain completion", stateErr) + } + } + // Old SQL stores could record failure after completion, so completion + // retains precedence for those otherwise-unreachable dual-terminal rows. + if !state.Completed && state.Failed { + return nil + } + if !state.Completed { + return uncommittedMutationError("confirm chain completion", errors.New("chain store returned done without terminal state")) + } + index, known := chainNodePosition(state.Nodes, env.NodeID) + if !known { + return uncommittedMutationError("confirm chain completion", errors.New("chain store returned done for an unknown node")) + } + if index != len(state.Nodes)-1 { + return nil + } + r.emitStoredJobOutcome(ctx, outcome) + r.prepareChainSuccessCallbacks(env.ChainID) + r.emit(ctx, Event{SchemaVersion: eventSchemaVersion, EventID: chainFactID(EventChainCompleted, env), Kind: EventChainCompleted, DispatchID: env.DispatchID, ChainID: env.ChainID, JobID: env.JobID, JobType: env.Job.Type, JobKey: storedJobEventKey(env.Job), Queue: env.Job.Options.Queue, Time: r.now()}) + _ = r.dispatchCallback(ctx, env, "chain_finally", nil) + r.cleanupChainCallbacks(env.ChainID) + return nil + } + r.emitStoredJobOutcome(ctx, outcome) + r.emit(ctx, Event{SchemaVersion: eventSchemaVersion, EventID: chainFactID(EventChainAdvanced, env), Kind: EventChainAdvanced, DispatchID: env.DispatchID, ChainID: env.ChainID, JobID: env.JobID, JobType: env.Job.Type, JobKey: storedJobEventKey(env.Job), Queue: env.Job.Options.Queue, Time: r.now()}) + return r.dispatchChainSuccessor(ctx, env, next) +} + +// invokeChainCatch claims the ephemeral catch callback before application code can run. +func (r *runtime) invokeChainCatch(ctx context.Context, st ChainState, err error) error { + return r.invokeChainCatchObserved(ctx, st, err, nil) +} + +// invokeChainCatchObserved emits lifecycle start only after state validation and idempotency claim succeed. +func (r *runtime) invokeChainCatchObserved(ctx context.Context, st ChainState, err error, onClaimed func()) error { + if !st.Failed || st.Completed { + return errCallbackNotReady + } + key := "chain_catch:" + st.ChainID + ok, onceErr := r.callbackOnce(ctx, key) + if onceErr != nil { + return onceErr + } + if !ok { + return errCallbackAlreadyInvoked + } + if onClaimed != nil { + onClaimed() + } + r.mu.RLock() + cb := r.chainCallbacks[st.ChainID] + r.mu.RUnlock() + if cb.catch != nil { + defer r.finishChainCallback(st.ChainID, "catch") + return runEphemeralCallback(func() error { return cb.catch(ctx, st, err) }) + } + return errCallbackUnavailable +} + +// invokeChainFinally claims the terminal closure before application code can run. +func (r *runtime) invokeChainFinally(ctx context.Context, st ChainState) error { + return r.invokeChainFinallyObserved(ctx, st, nil) +} + +// invokeChainFinallyObserved emits lifecycle start only after state validation and idempotency claim succeed. +func (r *runtime) invokeChainFinallyObserved(ctx context.Context, st ChainState, onClaimed func()) error { + if !st.Failed && !st.Completed { + return errCallbackNotReady + } + key := "chain_finally:" + st.ChainID + ok, onceErr := r.callbackOnce(ctx, key) + if onceErr != nil { + return onceErr + } + if !ok { + return errCallbackAlreadyInvoked + } + if onClaimed != nil { + onClaimed() + } + r.mu.RLock() + cb := r.chainCallbacks[st.ChainID] + r.mu.RUnlock() + if cb.finally == nil { + return errCallbackUnavailable + } + defer r.finishChainCallback(st.ChainID, "finally") + return runEphemeralCallback(func() error { return cb.finally(ctx, st) }) +} diff --git a/internal/workflow/chain_failure_receipt_test.go b/internal/workflow/chain_failure_receipt_test.go new file mode 100644 index 0000000..d726500 --- /dev/null +++ b/internal/workflow/chain_failure_receipt_test.go @@ -0,0 +1,298 @@ +package workflow + +import ( + "context" + "errors" + "strings" + "testing" + "time" + + "github.com/goforj/queue/busruntime" +) + +// TestChainCommittedFailureRecoveryPreservesOneApplicationOccurrence proves a +// receipt recovery archives the durable cause without replaying work or facts. +func TestChainCommittedFailureRecoveryPreservesOneApplicationOccurrence(t *testing.T) { + const ( + chainID = "chain-committed-failure-recovery" + nodeID = "node-committed-failure-recovery" + dispatchID = "dispatch-committed-failure-recovery" + jobID = "job-committed-failure-recovery" + jobType = "workflow:chain:committed-failure-recovery" + owner = "generation-chain-committed-failure" + ) + store := NewMemoryStore() + env := envelope{SchemaVersion: schemaVersion, DispatchID: dispatchID, Kind: "chain_node", ChainID: chainID, NodeID: nodeID, JobID: jobID, Job: StoredJob{Type: jobType, Payload: []byte(`{"id":1}`)}} + if err := store.CreateChain(context.Background(), ChainRecord{ChainID: chainID, DispatchID: dispatchID, Nodes: []ChainNode{{NodeID: nodeID, Job: env.Job}}}); err != nil { + t.Fatalf("create chain: %v", err) + } + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, store) + committedCause := errors.New("persisted terminal chain cause") + var handlerCalls, catchCalls, finallyCalls int + runtime.Register(jobType, func(context.Context, Context) error { + handlerCalls++ + return busruntime.Permanent(committedCause) + }) + runtime.chainCallbacks[chainID] = chainCallbacks{ + catch: func(context.Context, ChainState, error) error { + catchCalls++ + return nil + }, + finally: func(context.Context, ChainState) error { + finallyCalls++ + return nil + }, + } + + firstContext, firstSettlement := busruntime.WithDeliverySettlement(exhaustedWorkflowContext()) + firstContext = workflowGenerationContext(firstContext, owner) + if err := queueRuntime.DispatchJSON(firstContext, internalJobChainNode, env); !errors.Is(err, committedCause) || !busruntime.IsPermanent(err) { + t.Fatalf("initial failure = %v, want permanent committed cause", err) + } + if !firstSettlement.ApplicationStateCommitted() { + t.Fatal("initial failure did not signal committed application state") + } + receipt, known, err := requireTransitionReceiptStore(t, store).chainTransitionReceipt(context.Background(), chainID, nodeID) + if err != nil || !known || receipt.owner != workflowTransitionClaim(env, 2, owner) || receipt.outcome != BatchJobFailed || receipt.aggregateCompleted || receipt.aggregateCancelled { + t.Fatalf("failed chain receipt = known:%t receipt:%+v err:%v", known, receipt, err) + } + if handlerCalls != 1 || catchCalls != 1 || finallyCalls != 1 { + t.Fatalf("initial handler/catch/finally calls = %d/%d/%d, want 1/1/1", handlerCalls, catchCalls, finallyCalls) + } + initialEvents := len(recorder.events) + if countWorkflowEvents(recorder.events, EventJobFailed) != 1 || countWorkflowEvents(recorder.events, EventChainFailed) != 1 { + t.Fatalf("initial job/chain failure facts = %d/%d, want 1/1", countWorkflowEvents(recorder.events, EventJobFailed), countWorkflowEvents(recorder.events, EventChainFailed)) + } + + provenance := []struct { + name string + recoveredGeneration string + }{ + {name: "exact owner", recoveredGeneration: owner}, + {name: "different owner", recoveredGeneration: "generation-chain-different-owner"}, + {name: "legacy recovery"}, + } + for index, test := range provenance { + t.Run(test.name, func(t *testing.T) { + recoveryContext, recoverySettlement := busruntime.WithDeliverySettlement(exhaustedWorkflowContext()) + recoveryContext = workflowRecoveryContext(recoveryContext, "generation-chain-recovery-"+test.name, test.recoveredGeneration) + recoveryErr := queueRuntime.DispatchJSON(recoveryContext, internalJobChainNode, env) + if recoveryErr == nil || recoveryErr.Error() != committedCause.Error() || !busruntime.IsPermanent(recoveryErr) { + t.Fatalf("recovery %d error = %v, want permanent persisted cause", index, recoveryErr) + } + if recoverySettlement.ApplicationStateCommitted() { + t.Fatal("read-only failure recovery signaled a new application mutation") + } + if handlerCalls != 1 || catchCalls != 1 || finallyCalls != 1 || len(recorder.events) != initialEvents { + t.Fatalf("recovery occurrence changed handler/catch/finally/events = %d/%d/%d/%d, want 1/1/1/%d", handlerCalls, catchCalls, finallyCalls, len(recorder.events), initialEvents) + } + }) + } +} + +// TestChainFailureRecoveryRejectsInvalidReceiptIdentity proves every durable +// identity and terminal-shape mismatch fails closed before application code. +func TestChainFailureRecoveryRejectsInvalidReceiptIdentity(t *testing.T) { + mutations := []struct { + name string + mutate func(*transitionReceipt) + }{ + {name: "receipt version", mutate: func(receipt *transitionReceipt) { receipt.version++ }}, + {name: "event schema", mutate: func(receipt *transitionReceipt) { receipt.eventSchemaVersion++ }}, + {name: "success outcome", mutate: func(receipt *transitionReceipt) { receipt.outcome = BatchJobSucceeded }}, + {name: "completion owner", mutate: func(receipt *transitionReceipt) { receipt.aggregateCompleted = true }}, + {name: "cancellation owner", mutate: func(receipt *transitionReceipt) { receipt.aggregateCancelled = true }}, + {name: "empty delivery owner", mutate: func(receipt *transitionReceipt) { receipt.owner.deliveryID = "" }}, + {name: "negative owner attempt", mutate: func(receipt *transitionReceipt) { receipt.owner.attempt = -1 }}, + {name: "job dispatch", mutate: func(receipt *transitionReceipt) { receipt.owner.dispatchID = "dispatch-other" }}, + {name: "empty owner job id", mutate: func(receipt *transitionReceipt) { receipt.owner.jobID = "" }}, + {name: "job fingerprint", mutate: func(receipt *transitionReceipt) { receipt.owner.jobFingerprint = "fingerprint-other" }}, + {name: "workflow dispatch", mutate: func(receipt *transitionReceipt) { receipt.workflowDispatchID = "workflow-dispatch-other" }}, + {name: "workflow creation", mutate: func(receipt *transitionReceipt) { + receipt.workflowCreatedAt = receipt.workflowCreatedAt.Add(time.Second) + }}, + {name: "workflow kind", mutate: func(receipt *transitionReceipt) { receipt.workflowKind = batchTransitionKind }}, + {name: "workflow id", mutate: func(receipt *transitionReceipt) { receipt.workflowID = "chain-other" }}, + {name: "member id", mutate: func(receipt *transitionReceipt) { receipt.memberID = "node-other" }}, + } + for _, test := range mutations { + t.Run(test.name, func(t *testing.T) { + const ( + chainID = "chain-invalid-failure-receipt" + nodeID = "node-invalid-failure-receipt" + dispatchID = "dispatch-invalid-failure-receipt" + jobID = "job-invalid-failure-receipt" + jobType = "workflow:chain:invalid-failure-receipt" + owner = "generation-invalid-failure-receipt" + ) + store := NewMemoryStore().(*memoryStore) + env := envelope{SchemaVersion: schemaVersion, DispatchID: dispatchID, Kind: "chain_node", ChainID: chainID, NodeID: nodeID, JobID: jobID, Job: StoredJob{Type: jobType, Payload: []byte(`{"id":2}`)}} + if err := store.CreateChain(context.Background(), ChainRecord{ChainID: chainID, DispatchID: dispatchID, Nodes: []ChainNode{{NodeID: nodeID, Job: env.Job}}}); err != nil { + t.Fatalf("create chain: %v", err) + } + if result, err := store.failChainOutcome(context.Background(), chainID, nodeID, errors.New("persisted failure"), workflowTransitionClaim(env, 2, owner)); err != nil || !result.receiptKnown { + t.Fatalf("commit failed chain = %+v err:%v", result, err) + } + key := transitionReceiptKey{workflowKind: chainTransitionKind, workflowID: chainID, memberID: nodeID} + store.mu.Lock() + receipt := store.transitionReceipts[key] + test.mutate(&receipt) + store.transitionReceipts[key] = receipt + store.mu.Unlock() + + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, store) + var handlerCalls, callbackCalls int + runtime.Register(jobType, func(context.Context, Context) error { + handlerCalls++ + return busruntime.Permanent(errors.New("unexpected replay")) + }) + runtime.chainCallbacks[chainID] = chainCallbacks{finally: func(context.Context, ChainState) error { + callbackCalls++ + return nil + }} + recoveryContext, settlement := busruntime.WithDeliverySettlement(exhaustedWorkflowContext()) + recoveryContext = workflowRecoveryContext(recoveryContext, "generation-invalid-recovery", owner) + recoveryErr := queueRuntime.DispatchJSON(recoveryContext, internalJobChainNode, env) + if !busruntime.IsUncommitted(recoveryErr) { + t.Fatalf("invalid receipt recovery error = %v, want uncommitted", recoveryErr) + } + if handlerCalls != 0 || callbackCalls != 0 || settlement.ApplicationStateCommitted() || len(recorder.events) != 0 { + t.Fatalf("invalid receipt handler/callback/committed/events = %d/%d/%t/%d, want 0/0/false/0", handlerCalls, callbackCalls, settlement.ApplicationStateCommitted(), len(recorder.events)) + } + }) + } +} + +// TestChainFailureRecoveryAllowsDifferentPhysicalDeliveryIdentity proves a +// logical failure receipt archives duplicate jobs and attempts without replay. +func TestChainFailureRecoveryAllowsDifferentPhysicalDeliveryIdentity(t *testing.T) { + for _, test := range []struct { + name string + currentAttempt int + currentJobID string + }{ + {name: "different physical job", currentAttempt: 2, currentJobID: "job-chain-failure-duplicate"}, + {name: "different physical attempt", currentAttempt: 3, currentJobID: "job-chain-failure-owner"}, + {name: "negative current attempt", currentAttempt: -1, currentJobID: "job-chain-failure-owner"}, + } { + t.Run(test.name, func(t *testing.T) { + const ( + chainID = "chain-failure-physical-nonowner" + nodeID = "node-chain-failure-physical-nonowner" + dispatchID = "dispatch-chain-failure-physical-nonowner" + jobType = "workflow:chain:failure-physical-nonowner" + owner = "generation-chain-failure-physical-owner" + ) + store := NewMemoryStore() + ownerEnv := envelope{SchemaVersion: schemaVersion, DispatchID: dispatchID, Kind: "chain_node", ChainID: chainID, NodeID: nodeID, JobID: "job-chain-failure-owner", Job: StoredJob{Type: jobType, Payload: []byte(`{"id":3}`)}} + if err := store.CreateChain(context.Background(), ChainRecord{ChainID: chainID, DispatchID: dispatchID, Nodes: []ChainNode{{NodeID: nodeID, Job: ownerEnv.Job}}}); err != nil { + t.Fatalf("create chain: %v", err) + } + committedCause := errors.New("persisted duplicate-delivery chain failure") + if result, err := requireChainFailureStore(t, store).failChainOutcome(context.Background(), chainID, nodeID, committedCause, workflowTransitionClaim(ownerEnv, 2, owner)); err != nil || !result.receiptKnown { + t.Fatalf("commit failed chain = %+v err:%v", result, err) + } + + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, store) + var handlerCalls, callbackCalls int + runtime.Register(jobType, func(context.Context, Context) error { + handlerCalls++ + return busruntime.Permanent(errors.New("duplicate application execution")) + }) + runtime.chainCallbacks[chainID] = chainCallbacks{finally: func(context.Context, ChainState) error { + callbackCalls++ + return nil + }} + currentEnv := ownerEnv + currentEnv.JobID = test.currentJobID + attemptContext := busruntime.WithDeliveryAttempt(context.Background(), busruntime.DeliveryAttempt{Number: test.currentAttempt, MaxRetry: 3}) + recoveryContext, settlement := busruntime.WithDeliverySettlement(attemptContext) + recoveryContext = workflowRecoveryContext(recoveryContext, "generation-chain-failure-current", owner) + recoveryErr := queueRuntime.DispatchJSON(recoveryContext, internalJobChainNode, currentEnv) + if recoveryErr == nil || recoveryErr.Error() != committedCause.Error() || !busruntime.IsPermanent(recoveryErr) || busruntime.IsUncommitted(recoveryErr) { + t.Fatalf("physical nonowner recovery error = %v, want persisted permanent cause", recoveryErr) + } + settlement.Commit() + if handlerCalls != 0 || callbackCalls != 0 || settlement.ApplicationStateCommitted() || len(recorder.events) != 0 { + t.Fatalf("handler/callback/committed/events = %d/%d/%t/%d, want 0/0/false/0", handlerCalls, callbackCalls, settlement.ApplicationStateCommitted(), len(recorder.events)) + } + }) + } +} + +// TestChainLegacyFailureRecoveryRetainsCurrentFailureClassification proves a +// receipt-absent built-in row cannot turn a terminal replay into success. +func TestChainLegacyFailureRecoveryRetainsCurrentFailureClassification(t *testing.T) { + const ( + chainID = "chain-legacy-failure-classification" + nodeID = "node-legacy-failure-classification" + jobType = "workflow:chain:legacy-failure-classification" + ) + store := NewMemoryStore() + env := envelope{SchemaVersion: schemaVersion, DispatchID: "dispatch-legacy-failure-classification", Kind: "chain_node", ChainID: chainID, NodeID: nodeID, JobID: "job-legacy-failure-classification", Job: StoredJob{Type: jobType}} + if err := store.CreateChain(context.Background(), ChainRecord{ChainID: chainID, DispatchID: env.DispatchID, Nodes: []ChainNode{{NodeID: nodeID, Job: env.Job}}}); err != nil { + t.Fatalf("create chain: %v", err) + } + if _, owned, err := requireOutcomeStore(t, store).FailChainNode(context.Background(), chainID, nodeID, errors.New("legacy committed cause")); err != nil || !owned { + t.Fatalf("fail legacy chain = owned:%t err:%v", owned, err) + } + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, store) + currentCause := errors.New("current terminal replay") + var handlerCalls int + runtime.Register(jobType, func(context.Context, Context) error { + handlerCalls++ + return busruntime.Permanent(currentCause) + }) + recoveryContext := workflowRecoveryContext(exhaustedWorkflowContext(), "generation-legacy-current", "generation-legacy-absent") + recoveryErr := queueRuntime.DispatchJSON(recoveryContext, internalJobChainNode, env) + if !errors.Is(recoveryErr, currentCause) || !busruntime.IsPermanent(recoveryErr) { + t.Fatalf("legacy recovery error = %v, want current permanent failure", recoveryErr) + } + if handlerCalls != 1 || countWorkflowEvents(recorder.events, EventJobFailed) != 0 || countWorkflowEvents(recorder.events, EventChainFailed) != 0 { + t.Fatalf("legacy recovery handler/job/chain failure facts = %d/%d/%d, want 1/0/0", handlerCalls, countWorkflowEvents(recorder.events, EventJobFailed), countWorkflowEvents(recorder.events, EventChainFailed)) + } +} + +// TestChainFailureRecoveryWithoutPersistedCauseUsesTerminalDiagnostic proves +// even an empty legacy cause remains a failed physical settlement. +func TestChainFailureRecoveryWithoutPersistedCauseUsesTerminalDiagnostic(t *testing.T) { + const ( + chainID = "chain-empty-persisted-failure" + nodeID = "node-empty-persisted-failure" + jobType = "workflow:chain:empty-persisted-failure" + owner = "generation-empty-persisted-failure" + ) + store := NewMemoryStore().(*memoryStore) + env := envelope{SchemaVersion: schemaVersion, DispatchID: "dispatch-empty-persisted-failure", Kind: "chain_node", ChainID: chainID, NodeID: nodeID, JobID: "job-empty-persisted-failure", Job: StoredJob{Type: jobType}} + if err := store.CreateChain(context.Background(), ChainRecord{ChainID: chainID, DispatchID: env.DispatchID, Nodes: []ChainNode{{NodeID: nodeID, Job: env.Job}}}); err != nil { + t.Fatalf("create chain: %v", err) + } + if result, err := store.failChainOutcome(context.Background(), chainID, nodeID, nil, workflowTransitionClaim(env, 2, owner)); err != nil || !result.receiptKnown || result.state.Failure != "" { + t.Fatalf("commit empty-cause failure = %+v err:%v", result, err) + } + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, store) + var handlerCalls int + runtime.Register(jobType, func(context.Context, Context) error { + handlerCalls++ + return nil + }) + recoveryContext := workflowRecoveryContext(exhaustedWorkflowContext(), "generation-empty-cause-recovery", owner) + recoveryErr := queueRuntime.DispatchJSON(recoveryContext, internalJobChainNode, env) + if recoveryErr == nil || !busruntime.IsPermanent(recoveryErr) || !strings.Contains(recoveryErr.Error(), "original cause was empty") { + t.Fatalf("empty-cause recovery error = %v, want permanent diagnostic", recoveryErr) + } + if handlerCalls != 0 || len(recorder.events) != 0 { + t.Fatalf("empty-cause recovery handler/events = %d/%d, want 0/0", handlerCalls, len(recorder.events)) + } +} + +// countWorkflowEvents counts one event kind in a focused synchronous fixture. +func countWorkflowEvents(events []Event, kind EventKind) int { + count := 0 + for _, event := range events { + if event.Kind == kind { + count++ + } + } + return count +} diff --git a/internal/workflow/direct_delivery_test.go b/internal/workflow/direct_delivery_test.go new file mode 100644 index 0000000..cb005ae --- /dev/null +++ b/internal/workflow/direct_delivery_test.go @@ -0,0 +1,119 @@ +package workflow + +import ( + "bytes" + "context" + "testing" + + "github.com/goforj/queue/busruntime" +) + +// TestDirectDeliveryMetadataTrustBoundary verifies the engine trusts only the +// one metadata version it owns while direct application bytes remain usable. +func TestDirectDeliveryMetadataTrustBoundary(t *testing.T) { + t.Run("canonical dispatch", func(t *testing.T) { + runtime := newDirectTestRuntime() + engine, err := New(runtime) + if err != nil { + t.Fatalf("new engine: %v", err) + } + var received Context + engine.Register("reports:build", func(_ context.Context, message Context) error { + received = message + return nil + }) + + result, err := engine.DispatchDirect(context.Background(), StoredJob{ + Type: "reports:build", + Payload: []byte{0, 1, 0xff}, + Options: JobOptions{Queue: "critical"}, + }) + if err != nil { + t.Fatalf("dispatch direct job: %v", err) + } + if received.SchemaVersion != schemaVersion || received.DispatchID != result.DispatchID || received.JobID == "" { + t.Fatalf("canonical correlation = %+v, receipt = %+v", received, result) + } + if received.JobType != "reports:build" || !bytes.Equal(received.PayloadBytes(), []byte{0, 1, 0xff}) { + t.Fatalf("canonical application message = %+v payload=%v", received, received.PayloadBytes()) + } + }) + + tests := []struct { + name string + ctx context.Context + }{ + {name: "missing", ctx: context.Background()}, + { + name: "future version", + ctx: busruntime.WithDeliveryMetadata(context.Background(), busruntime.DeliveryMetadata{ + SchemaVersion: busruntime.DeliveryMetadataVersion + 1, + DispatchID: "spoofed", + JobID: "spoofed", + }), + }, + { + name: "unversioned fields", + ctx: busruntime.WithDeliveryMetadata(context.Background(), busruntime.DeliveryMetadata{ + DispatchID: "spoofed", + JobID: "spoofed", + }), + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + runtime := newDirectTestRuntime() + engine, err := New(runtime) + if err != nil { + t.Fatalf("new engine: %v", err) + } + var received Context + engine.Register("reports:build", func(_ context.Context, message Context) error { + received = message + return nil + }) + handler := runtime.handlers["reports:build"] + if handler == nil { + t.Fatal("direct application handler was not registered") + } + if err := handler(test.ctx, testInboundJob{payload: []byte{4, 5, 6}}); err != nil { + t.Fatalf("handle untrusted delivery: %v", err) + } + if received.DispatchID != "" || received.JobID != "" || received.ChainID != "" || received.BatchID != "" { + t.Fatalf("untrusted delivery supplied correlation: %+v", received) + } + if received.SchemaVersion != schemaVersion || received.JobType != "reports:build" || !bytes.Equal(received.PayloadBytes(), []byte{4, 5, 6}) { + t.Fatalf("untrusted delivery lost application identity: %+v payload=%v", received, received.PayloadBytes()) + } + }) + } +} + +// TestDirectDeliveryFallsBackToLegacyRuntime proves custom runtimes can adopt +// the new engine without implementing direct dispatch in the same release. +func TestDirectDeliveryFallsBackToLegacyRuntime(t *testing.T) { + runtime := newSyncTestRuntime() + engine, err := New(runtime) + if err != nil { + t.Fatalf("new legacy runtime engine: %v", err) + } + var received Context + engine.Register("reports:legacy-runtime", func(_ context.Context, message Context) error { + received = message + return nil + }) + + result, err := engine.DispatchDirect(context.Background(), StoredJob{ + Type: "reports:legacy-runtime", + Payload: []byte{9, 8, 7}, + }) + if err != nil { + t.Fatalf("dispatch through legacy runtime: %v", err) + } + if received.DispatchID != result.DispatchID || received.JobID == "" || received.JobType != "reports:legacy-runtime" { + t.Fatalf("legacy-runtime correlation = %+v, receipt = %+v", received, result) + } + if !bytes.Equal(received.PayloadBytes(), []byte{9, 8, 7}) { + t.Fatalf("legacy-runtime payload = %v", received.PayloadBytes()) + } +} diff --git a/internal/workflow/doc.go b/internal/workflow/doc.go new file mode 100644 index 0000000..b111434 --- /dev/null +++ b/internal/workflow/doc.go @@ -0,0 +1,2 @@ +// Package workflow owns the private workflow protocol and orchestration boundary. +package workflow diff --git a/internal/workflow/events.go b/internal/workflow/events.go new file mode 100644 index 0000000..402e7cb --- /dev/null +++ b/internal/workflow/events.go @@ -0,0 +1,114 @@ +package workflow + +import ( + "context" + "time" +) + +// EventKind identifies one workflow lifecycle fact. +type EventKind string + +const ( + // EventDispatchStarted marks the beginning of logical dispatch submission. + EventDispatchStarted EventKind = "dispatch_started" + // EventDispatchSucceeded records that a logical dispatch was accepted. + EventDispatchSucceeded EventKind = "dispatch_succeeded" + // EventDispatchFailed records that a logical dispatch was rejected. + EventDispatchFailed EventKind = "dispatch_failed" + // EventJobStarted records the beginning of a logical handler attempt. + EventJobStarted EventKind = "job_started" + // EventJobSucceeded records a committed logical job success. + EventJobSucceeded EventKind = "job_succeeded" + // EventJobFailed records a permanent or exhausted logical job failure. + EventJobFailed EventKind = "job_failed" + // EventChainStarted records creation and initial scheduling of a chain. + EventChainStarted EventKind = "chain_started" + // EventChainAdvanced records a committed transition to the next chain node. + EventChainAdvanced EventKind = "chain_advanced" + // EventChainCompleted records terminal chain success. + EventChainCompleted EventKind = "chain_completed" + // EventChainFailed records terminal chain failure. + EventChainFailed EventKind = "chain_failed" + // EventBatchStarted records creation and initial scheduling of a batch. + EventBatchStarted EventKind = "batch_started" + // EventBatchProgressed records a committed change to aggregate batch state. + EventBatchProgressed EventKind = "batch_progressed" + // EventBatchCompleted records terminal batch completion. + EventBatchCompleted EventKind = "batch_completed" + // EventBatchFailed records a logical batch failure. + EventBatchFailed EventKind = "batch_failed" + // EventBatchCancelled records cancellation after a batch can no longer continue. + EventBatchCancelled EventKind = "batch_cancelled" + // EventCallbackStarted records the beginning of an ephemeral callback attempt. + EventCallbackStarted EventKind = "callback_started" + // EventCallbackSucceeded records successful callback completion. + EventCallbackSucceeded EventKind = "callback_succeeded" + // EventCallbackFailed records callback failure. + EventCallbackFailed EventKind = "callback_failed" +) + +// Event carries internal workflow facts and correlation into public observer adapters. +type Event struct { + SchemaVersion int + EventID string + Kind EventKind + DispatchID string + JobID string + ChainID string + BatchID string + Attempt int + JobType string + JobKey string + Queue string + Duration time.Duration + Time time.Time + Err error +} + +// Observer receives internal workflow events. +type Observer interface { + // Observe consumes one best-effort workflow fact. + Observe(ctx context.Context, event Event) +} + +// ObserverFunc adapts a function to Observer. +type ObserverFunc func(ctx context.Context, event Event) + +// Observe calls the wrapped observer function. +func (f ObserverFunc) Observe(ctx context.Context, event Event) { + f(ctx, event) +} + +// MultiObserver fans out one event to multiple observers. +func MultiObserver(observers ...Observer) Observer { + filtered := make([]Observer, 0, len(observers)) + for _, observer := range observers { + if observer != nil { + filtered = append(filtered, observer) + } + } + return multiObserver(filtered) +} + +type multiObserver []Observer + +// Observe forwards one event to every configured observer while preserving panic isolation. +func (m multiObserver) Observe(ctx context.Context, event Event) { + for _, observer := range m { + safeObserve(ctx, observer, event) + } +} + +// safeObserve prevents observer panics or nil contexts from changing workflow execution. +func safeObserve(ctx context.Context, observer Observer, event Event) { + if observer == nil { + return + } + if ctx == nil { + ctx = context.Background() + } + defer func() { + _ = recover() + }() + observer.Observe(ctx, event) +} diff --git a/internal/workflow/events_test.go b/internal/workflow/events_test.go new file mode 100644 index 0000000..2a0241e --- /dev/null +++ b/internal/workflow/events_test.go @@ -0,0 +1,745 @@ +package workflow + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/goforj/queue/busruntime" +) + +type failingDispatchQueue struct { + err error + handlers map[string]busruntime.Handler + workerCnt int +} + +func (q *failingDispatchQueue) StartWorkers(context.Context) error { return nil } +func (q *failingDispatchQueue) Shutdown(context.Context) error { return nil } + +func (q *failingDispatchQueue) BusRegister(jobType string, handler busruntime.Handler) { + if q.handlers == nil { + q.handlers = make(map[string]busruntime.Handler) + } + q.handlers[jobType] = handler +} + +func (q *failingDispatchQueue) BusDispatch(context.Context, string, []byte, busruntime.JobOptions) error { + return q.err +} + +func TestDispatchEnqueueFailureEmitsStartedThenFailed(t *testing.T) { + q := &failingDispatchQueue{err: errors.New("enqueue failed")} + var kinds []EventKind + b, err := NewWithStore(q, NewMemoryStore(), WithObserver(ObserverFunc(func(_ context.Context, e Event) { + kinds = append(kinds, e.Kind) + }))) + if err != nil { + t.Fatalf("new bus: %v", err) + } + + res, err := b.Dispatch(context.Background(), NewJob("monitor:poll", nil)) + if err == nil { + t.Fatal("expected dispatch enqueue failure") + } + if res.DispatchID == "" { + t.Fatal("expected non-empty dispatch id on enqueue failure") + } + if len(kinds) != 2 { + t.Fatalf("expected 2 events, got %d (%v)", len(kinds), kinds) + } + if kinds[0] != EventDispatchStarted || kinds[1] != EventDispatchFailed { + t.Fatalf("expected started then failed, got %v", kinds) + } +} + +func TestUnknownCallbackKindEmitsCallbackFailed(t *testing.T) { + q := newSyncTestRuntime() + var started int + var failed int + b, err := New(q, WithObserver(ObserverFunc(func(_ context.Context, e Event) { + if e.Kind == EventCallbackStarted { + started++ + } + if e.Kind == EventCallbackFailed { + failed++ + } + }))) + if err != nil { + t.Fatalf("new bus: %v", err) + } + if err := b.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + + payload := map[string]any{ + "schema_version": 1, + "dispatch_id": "d1", + "kind": "callback", + "job_id": "j1", + "callback_kind": "unknown_kind", + } + if err := q.DispatchJSON(context.Background(), internalJobCallback, payload); err == nil { + t.Fatal("expected unknown callback kind error") + } + if started != 0 { + t.Fatalf("invalid callback emitted %d started events, want 0", started) + } + if failed != 1 { + t.Fatalf("expected callback failed once, got %d", failed) + } +} + +func TestCallbackMissingRequiredIDsEmitsCallbackFailed(t *testing.T) { + q := newSyncTestRuntime() + var failed int + b, err := New(q, WithObserver(ObserverFunc(func(_ context.Context, e Event) { + if e.Kind == EventCallbackFailed { + failed++ + } + }))) + if err != nil { + t.Fatalf("new bus: %v", err) + } + if err := b.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + + tests := []map[string]any{ + { + "schema_version": 1, + "dispatch_id": "d1", + "kind": "callback", + "job_id": "j1", + "callback_kind": "chain_catch", + // missing chain_id + }, + { + "schema_version": 1, + "dispatch_id": "d2", + "kind": "callback", + "job_id": "j2", + "callback_kind": "batch_then", + // missing batch_id + }, + } + + for i, payloadMap := range tests { + if err := q.DispatchJSON(context.Background(), internalJobCallback, payloadMap); err == nil { + t.Fatalf("expected callback validation error for case %d", i) + } + } + + if failed != len(tests) { + t.Fatalf("expected %d callback failed events, got %d", len(tests), failed) + } +} + +// TestCallbackFunctionErrorEmitsFailed verifies an invoked ephemeral callback cannot be reported as successful. +func TestCallbackFunctionErrorEmitsFailed(t *testing.T) { + tests := []struct { + name string + handlerErr error + dispatch func(Engine, error) + }{ + { + name: "chain catch", + handlerErr: errors.New("handler failed"), + dispatch: func(b Engine, callbackErr error) { + _, _ = b.Chain(NewJob("job:callback-error", nil)). + Catch(func(context.Context, ChainState, error) error { return callbackErr }). + Dispatch(context.Background()) + }, + }, + { + name: "chain finally", + dispatch: func(b Engine, callbackErr error) { + _, _ = b.Chain(NewJob("job:callback-error", nil)). + Finally(func(context.Context, ChainState) error { return callbackErr }). + Dispatch(context.Background()) + }, + }, + { + name: "batch catch", + handlerErr: errors.New("handler failed"), + dispatch: func(b Engine, callbackErr error) { + _, _ = b.Batch(NewJob("job:callback-error", nil)). + Catch(func(context.Context, BatchState, error) error { return callbackErr }). + Dispatch(context.Background()) + }, + }, + { + name: "batch then", + dispatch: func(b Engine, callbackErr error) { + _, _ = b.Batch(NewJob("job:callback-error", nil)). + Then(func(context.Context, BatchState) error { return callbackErr }). + Dispatch(context.Background()) + }, + }, + { + name: "batch finally", + dispatch: func(b Engine, callbackErr error) { + _, _ = b.Batch(NewJob("job:callback-error", nil)). + Finally(func(context.Context, BatchState) error { return callbackErr }). + Dispatch(context.Background()) + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + q := newSyncTestRuntime() + callbackErr := errors.New("callback failed") + var ( + failed []Event + succeeded []Event + ) + b, err := New(q, WithObserver(ObserverFunc(func(_ context.Context, event Event) { + switch event.Kind { + case EventCallbackFailed: + failed = append(failed, event) + case EventCallbackSucceeded: + succeeded = append(succeeded, event) + } + }))) + if err != nil { + t.Fatalf("new bus: %v", err) + } + b.Register("job:callback-error", func(context.Context, Context) error { return test.handlerErr }) + if err := b.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + test.dispatch(b, callbackErr) + if len(failed) != 1 || !errors.Is(failed[0].Err, callbackErr) { + t.Fatalf("callback failed events = %#v, want callback cause", failed) + } + for _, event := range succeeded { + if event.JobID == failed[0].JobID { + t.Fatalf("failed callback job %q later emitted success", event.JobID) + } + } + }) + } +} + +// TestCallbackPanicEmitsFailed verifies callback recovery preserves a terminal lifecycle fact and the panic cause. +func TestCallbackPanicEmitsFailed(t *testing.T) { + queueRuntime := newSyncTestRuntime() + panicErr := errors.New("callback panic") + var started int + var failed []Event + var succeeded int + busRuntime, err := New(queueRuntime, WithObserver(ObserverFunc(func(_ context.Context, event Event) { + switch event.Kind { + case EventCallbackStarted: + started++ + case EventCallbackFailed: + failed = append(failed, event) + case EventCallbackSucceeded: + succeeded++ + } + }))) + if err != nil { + t.Fatalf("new bus: %v", err) + } + busRuntime.Register("job:callback-panic", func(context.Context, Context) error { return nil }) + if err := busRuntime.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + if _, err := busRuntime.Batch(NewJob("job:callback-panic", nil)). + Then(func(context.Context, BatchState) error { panic(panicErr) }). + Dispatch(context.Background()); err != nil { + t.Fatalf("dispatch batch: %v", err) + } + if started != 1 || len(failed) != 1 || succeeded != 0 { + t.Fatalf("callback started/failed/succeeded = %d/%d/%d, want 1/1/0", started, len(failed), succeeded) + } + if !errors.Is(failed[0].Err, panicErr) { + t.Fatalf("callback panic event error = %v, want cause %v", failed[0].Err, panicErr) + } +} + +// TestPositiveWorkflowEventsWaitForDeliverySettlement verifies broker-backed success facts remain pending until acknowledgement. +func TestPositiveWorkflowEventsWaitForDeliverySettlement(t *testing.T) { + positive := []EventKind{ + EventJobSucceeded, + EventChainAdvanced, + EventChainCompleted, + EventBatchProgressed, + EventBatchCompleted, + EventCallbackSucceeded, + } + for _, kind := range positive { + t.Run(string(kind), func(t *testing.T) { + var events []Event + runtime := &runtime{observer: ObserverFunc(func(_ context.Context, event Event) { + events = append(events, event) + })} + ctx, settlement := busruntime.WithDeliverySettlement(context.Background()) + runtime.emit(ctx, Event{Kind: kind}) + if len(events) != 0 { + t.Fatalf("event %q emitted before settlement: %+v", kind, events) + } + settlement.Commit() + if len(events) != 1 || events[0].Kind != kind { + t.Fatalf("events after settlement = %+v, want %q", events, kind) + } + }) + } +} + +// TestStableWorkflowFactIDPreservesLogicalIdentity verifies replay identifiers +// are deterministic, kind-specific, and immune to ambiguous field partitioning. +func TestStableWorkflowFactIDPreservesLogicalIdentity(t *testing.T) { + want := stableWorkflowFactID(EventChainCompleted, "chain", "node") + if len(want) != len("evt_")+32 { + t.Fatalf("stable fact id length = %d, want %d", len(want), len("evt_")+32) + } + if got := stableWorkflowFactID(EventChainCompleted, "chain", "node"); got != want { + t.Fatalf("replayed fact id = %q, want %q", got, want) + } + if got := stableWorkflowFactID(EventChainAdvanced, "chain", "node"); got == want { + t.Fatalf("different event kinds share fact id %q", got) + } + if left, right := stableWorkflowFactID(EventJobSucceeded, "ab", "c"), stableWorkflowFactID(EventJobSucceeded, "a", "bc"); left == right { + t.Fatalf("differently framed identities share fact id %q", left) + } +} + +// TestStoredJobOutcomeFactIDRequiresCorrelation keeps unrelated legacy +// deliveries from sharing a deterministic identifier merely by type/attempt. +func TestStoredJobOutcomeFactIDRequiresCorrelation(t *testing.T) { + correlated := storedJobOutcome{env: envelope{ + DispatchID: "dispatch-fact-id", + JobID: "job-fact-id", + Job: StoredJob{ + Type: "workflow:fact-id", + Payload: []byte(`{"version":1}`), + Options: JobOptions{Queue: "critical"}, + }, + Attempt: 2, + }} + first := storedJobOutcomeFactID(EventJobSucceeded, correlated) + if second := storedJobOutcomeFactID(EventJobSucceeded, correlated); second != first { + t.Fatalf("correlated fact ids = %q/%q, want stable", first, second) + } + for _, test := range []struct { + name string + mutate func(*storedJobOutcome) + }{ + {name: "payload", mutate: func(outcome *storedJobOutcome) { outcome.env.Job.Payload = []byte(`{"version":2}`) }}, + {name: "queue", mutate: func(outcome *storedJobOutcome) { outcome.env.Job.Options.Queue = "bulk" }}, + {name: "job type", mutate: func(outcome *storedJobOutcome) { outcome.env.Job.Type = "workflow:other-fact" }}, + {name: "attempt", mutate: func(outcome *storedJobOutcome) { outcome.env.Attempt++ }}, + } { + t.Run(test.name, func(t *testing.T) { + changed := correlated + test.mutate(&changed) + if got := storedJobOutcomeFactID(EventJobSucceeded, changed); got == first { + t.Fatalf("changed %s reused fact id %q", test.name, got) + } + }) + } + + uncorrelated := correlated + uncorrelated.env.DispatchID = "" + if left, right := storedJobOutcomeFactID(EventJobSucceeded, uncorrelated), storedJobOutcomeFactID(EventJobSucceeded, uncorrelated); left == right { + t.Fatalf("uncorrelated delivery reused fact id %q", left) + } + uncorrelated = correlated + uncorrelated.env.JobID = "" + if left, right := storedJobOutcomeFactID(EventJobSucceeded, uncorrelated), storedJobOutcomeFactID(EventJobSucceeded, uncorrelated); left == right { + t.Fatalf("partially correlated delivery reused fact id %q", left) + } + if left, right := storedJobOutcomeFactID(EventJobFailed, correlated), storedJobOutcomeFactID(EventJobFailed, correlated); left == right { + t.Fatalf("non-recoverable failures reused fact id %q", left) + } +} + +// TestAggregateFactIDsIncludeObservableCorrelation proves deterministic chain +// and batch identifiers cannot label events whose visible job fields disagree. +func TestAggregateFactIDsIncludeObservableCorrelation(t *testing.T) { + base := envelope{ + DispatchID: "dispatch-aggregate-fact-id", + JobID: "job-aggregate-fact-id", + ChainID: "chain-aggregate-fact-id", + BatchID: "batch-aggregate-fact-id", + NodeID: "node-aggregate-fact-id", + Job: StoredJob{ + Type: "workflow:aggregate-fact-id", + Payload: []byte(`{"version":1}`), + Options: JobOptions{Queue: "critical"}, + }, + } + chainID := chainFactID(EventChainAdvanced, base) + batchID := batchFactID(EventBatchProgressed, base) + if got := chainFactID(EventChainAdvanced, base); got != chainID { + t.Fatalf("replayed chain fact ids = %q/%q, want stable", chainID, got) + } + if got := batchFactID(EventBatchProgressed, base); got != batchID { + t.Fatalf("replayed batch fact ids = %q/%q, want stable", batchID, got) + } + for _, test := range []struct { + name string + mutate func(*envelope) + }{ + {name: "dispatch", mutate: func(env *envelope) { env.DispatchID = "dispatch-other" }}, + {name: "job", mutate: func(env *envelope) { env.JobID = "job-other" }}, + {name: "job type", mutate: func(env *envelope) { env.Job.Type = "workflow:other-fact" }}, + {name: "payload", mutate: func(env *envelope) { env.Job.Payload = []byte(`{"version":2}`) }}, + {name: "queue", mutate: func(env *envelope) { env.Job.Options.Queue = "bulk" }}, + } { + t.Run(test.name, func(t *testing.T) { + changed := base + test.mutate(&changed) + if got := chainFactID(EventChainAdvanced, changed); got == chainID { + t.Fatalf("changed %s reused chain fact id %q", test.name, got) + } + if got := batchFactID(EventBatchProgressed, changed); got == batchID { + t.Fatalf("changed %s reused batch fact id %q", test.name, got) + } + }) + } +} + +// TestRecoveredSuccessRetainsLogicalIdentityWithoutReplayTiming proves recovery +// deduplicates the same application attempt without borrowing failed replay telemetry. +func TestRecoveredSuccessRetainsLogicalIdentityWithoutReplayTiming(t *testing.T) { + started := time.Unix(10, 0) + finished := time.Unix(12, 0) + normal := storedJobOutcome{ + env: envelope{ + DispatchID: "dispatch-recovered-success-id", + JobID: "job-recovered-success-id", + ChainID: "chain-recovered-success-id", + Attempt: 0, + Job: StoredJob{Type: "workflow:recovered-success-id"}, + }, + started: started, + finished: finished, + } + replayed := normal + replayed.attempt = busruntime.DeliveryAttempt{Number: 0, MaxRetry: 1} + replayed.started = time.Unix(20, 0) + replayed.finished = time.Unix(25, 0) + replayed.err = errors.New("contradictory replay failure") + observed := time.Unix(30, 0) + receipt := transitionReceipt{ + version: transitionReceiptVersion, + eventSchemaVersion: eventSchemaVersion, + outcome: BatchJobSucceeded, + owner: transitionClaim{ + deliveryID: "generation-recovered-success-id", + attempt: 0, + dispatchID: normal.env.DispatchID, + jobID: normal.env.JobID, + jobFingerprint: storedJobReceiptFingerprint(normal.env.Job), + }, + } + recovered, err := recoveredStoredJobSuccess(replayed, receipt, observed) + if err != nil { + t.Fatalf("recover stored job success: %v", err) + } + if got, want := storedJobOutcomeFactID(EventJobSucceeded, recovered), storedJobOutcomeFactID(EventJobSucceeded, normal); got != want { + t.Fatalf("recovered/normal fact ids = %q/%q", got, want) + } + if recovered.err != nil || recovered.env.Attempt != 0 || recovered.attempt.Number != 0 || !recovered.started.Equal(observed) || !recovered.finished.Equal(observed) || recovered.finished.Sub(recovered.started) != 0 { + t.Fatalf("recovered outcome retained replay telemetry: %+v", recovered) + } + nextAttempt := normal + nextAttempt.env.Attempt++ + if got, previous := storedJobOutcomeFactID(EventJobSucceeded, nextAttempt), storedJobOutcomeFactID(EventJobSucceeded, normal); got == previous { + t.Fatalf("different application attempts share fact id %q", got) + } +} + +// TestRecoveredSuccessRequiresValidTransitionReceipt rejects incomplete or +// mismatched durable identity before a reconstructed fact can be emitted. +func TestRecoveredSuccessRequiresValidTransitionReceipt(t *testing.T) { + job := StoredJob{Type: "workflow:receipt-validation", Payload: []byte(`{"id":1}`)} + outcome := storedJobOutcome{env: envelope{ + DispatchID: "dispatch-receipt-validation", + JobID: "job-receipt-validation", + Attempt: 1, + Job: job, + }} + valid := transitionReceipt{ + version: transitionReceiptVersion, + eventSchemaVersion: eventSchemaVersion, + outcome: BatchJobSucceeded, + owner: transitionClaim{ + deliveryID: "generation-receipt-validation", + attempt: 1, + dispatchID: outcome.env.DispatchID, + jobID: outcome.env.JobID, + jobFingerprint: storedJobReceiptFingerprint(job), + }, + } + for _, test := range []struct { + name string + mutate func(*transitionReceipt) + }{ + {name: "failed outcome", mutate: func(receipt *transitionReceipt) { receipt.outcome = BatchJobFailed }}, + {name: "unknown receipt version", mutate: func(receipt *transitionReceipt) { receipt.version++ }}, + {name: "unknown event schema", mutate: func(receipt *transitionReceipt) { receipt.eventSchemaVersion++ }}, + {name: "empty delivery owner", mutate: func(receipt *transitionReceipt) { receipt.owner.deliveryID = "" }}, + {name: "negative attempt", mutate: func(receipt *transitionReceipt) { receipt.owner.attempt = -1 }}, + {name: "different attempt", mutate: func(receipt *transitionReceipt) { receipt.owner.attempt = 2 }}, + {name: "dispatch mismatch", mutate: func(receipt *transitionReceipt) { receipt.owner.dispatchID = "different" }}, + {name: "job mismatch", mutate: func(receipt *transitionReceipt) { receipt.owner.jobID = "different" }}, + {name: "fingerprint mismatch", mutate: func(receipt *transitionReceipt) { receipt.owner.jobFingerprint = "different" }}, + } { + t.Run(test.name, func(t *testing.T) { + receipt := valid + test.mutate(&receipt) + if _, err := recoveredStoredJobSuccess(outcome, receipt, time.Now()); err == nil { + t.Fatal("invalid transition receipt was accepted") + } + }) + } +} + +// TestRecoveredTransitionReceiptLogicalValidationSeparatesPhysicalOwnership +// keeps malformed identity fail-closed while permitting legitimate nonowners. +func TestRecoveredTransitionReceiptLogicalValidationSeparatesPhysicalOwnership(t *testing.T) { + job := StoredJob{Type: "workflow:logical-receipt-validation", Payload: []byte(`{"id":2}`)} + env := envelope{ + DispatchID: "dispatch-logical-receipt-validation", + JobID: "job-logical-receipt-validation", + Attempt: 2, + Job: job, + } + valid := transitionReceipt{ + version: transitionReceiptVersion, + eventSchemaVersion: eventSchemaVersion, + owner: transitionClaim{ + deliveryID: "generation-logical-receipt-validation", + attempt: env.Attempt, + dispatchID: env.DispatchID, + jobID: env.JobID, + jobFingerprint: storedJobReceiptFingerprint(job), + }, + } + for _, test := range []struct { + name string + requireOwnerJobID bool + mutateEnv func(*envelope) + mutateReceipt func(*transitionReceipt) + wantErr bool + }{ + {name: "chain different owner attempt", mutateReceipt: func(receipt *transitionReceipt) { receipt.owner.attempt++ }}, + {name: "chain different owner job", mutateReceipt: func(receipt *transitionReceipt) { receipt.owner.jobID = "job-other-physical-delivery" }}, + {name: "batch different owner attempt", requireOwnerJobID: true, mutateReceipt: func(receipt *transitionReceipt) { receipt.owner.attempt++ }}, + {name: "batch different logical member", requireOwnerJobID: true, mutateReceipt: func(receipt *transitionReceipt) { receipt.owner.jobID = "job-other-member" }, wantErr: true}, + {name: "chain negative current attempt", mutateEnv: func(env *envelope) { env.Attempt = -1 }}, + {name: "empty delivery dispatch", mutateEnv: func(env *envelope) { env.DispatchID = "" }, wantErr: true}, + {name: "empty delivery job", mutateEnv: func(env *envelope) { env.JobID = "" }, wantErr: true}, + {name: "empty owner generation", mutateReceipt: func(receipt *transitionReceipt) { receipt.owner.deliveryID = "" }, wantErr: true}, + {name: "negative owner attempt", mutateReceipt: func(receipt *transitionReceipt) { receipt.owner.attempt = -1 }, wantErr: true}, + {name: "empty owner job", mutateReceipt: func(receipt *transitionReceipt) { receipt.owner.jobID = "" }, wantErr: true}, + {name: "owner dispatch mismatch", mutateReceipt: func(receipt *transitionReceipt) { receipt.owner.dispatchID = "dispatch-other" }, wantErr: true}, + {name: "owner fingerprint mismatch", mutateReceipt: func(receipt *transitionReceipt) { receipt.owner.jobFingerprint = "fingerprint-other" }, wantErr: true}, + } { + t.Run(test.name, func(t *testing.T) { + currentEnv := env + currentReceipt := valid + if test.mutateEnv != nil { + test.mutateEnv(¤tEnv) + } + if test.mutateReceipt != nil { + test.mutateReceipt(¤tReceipt) + } + err := validateRecoveredTransitionReceipt(currentEnv, currentReceipt, test.requireOwnerJobID) + if (err != nil) != test.wantErr { + t.Fatalf("logical receipt validation error = %v, want error %t", err, test.wantErr) + } + }) + } +} + +// TestDuplicateFailedCallbackDoesNotBecomeSuccessful verifies an at-most-once callback marker cannot turn redelivery into a false success. +func TestDuplicateFailedCallbackDoesNotBecomeSuccessful(t *testing.T) { + const batchID = "batch_callback_failed_duplicate" + store := NewMemoryStore() + if err := store.CreateBatch(context.Background(), BatchRecord{ + BatchID: batchID, + Jobs: []BatchJob{{JobID: "batch_callback_job", Job: StoredJob{Type: "callback:source"}}}, + }); err != nil { + t.Fatalf("create batch: %v", err) + } + if _, _, err := store.MarkBatchJobSucceeded(context.Background(), batchID, "batch_callback_job"); err != nil { + t.Fatalf("complete batch: %v", err) + } + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, store) + callbackErr := errors.New("callback failed") + runtime.batchCallbacks[batchID] = batchCallbacks{ + then: func(context.Context, BatchState) error { return callbackErr }, + } + env := envelope{ + SchemaVersion: schemaVersion, + DispatchID: "dispatch_callback_failed_duplicate", + JobID: "job_callback_failed_duplicate", + BatchID: batchID, + CallbackKind: "batch_then", + } + if err := queueRuntime.DispatchJSON(exhaustedWorkflowContext(), internalJobCallback, env); !errors.Is(err, callbackErr) { + t.Fatalf("first callback error = %v, want %v", err, callbackErr) + } + if err := queueRuntime.DispatchJSON(exhaustedWorkflowContext(), internalJobCallback, env); err != nil { + t.Fatalf("duplicate callback delivery: %v", err) + } + failed := 0 + succeeded := 0 + for _, event := range recorder.events { + switch event.Kind { + case EventCallbackFailed: + failed++ + case EventCallbackSucceeded: + succeeded++ + } + } + if failed != 1 || succeeded != 0 { + t.Fatalf("callback failed/succeeded events = %d/%d, want 1/0", failed, succeeded) + } +} + +func TestMultiObserverPanicsAreIsolated(t *testing.T) { + var called int + observer := MultiObserver( + ObserverFunc(func(context.Context, Event) { panic("boom") }), + ObserverFunc(func(context.Context, Event) { called++ }), + ) + observer.Observe(context.Background(), Event{Kind: EventDispatchStarted}) + if called != 1 { + t.Fatalf("expected second observer called once despite panic, got %d", called) + } +} + +func TestChainEnqueueFailureInvokesCatchAndFinally(t *testing.T) { + q := &failingDispatchQueue{err: errors.New("enqueue failed")} + bi, err := NewWithStore(q, NewMemoryStore()) + if err != nil { + t.Fatalf("new bus: %v", err) + } + b := bi.(*runtime) + + var catchCount int + var finallyCount int + chainID, err := b.Chain(NewJob("monitor:poll", nil)). + Catch(func(context.Context, ChainState, error) error { + catchCount++ + return nil + }). + Finally(func(context.Context, ChainState) error { + finallyCount++ + return nil + }). + Dispatch(context.Background()) + if err == nil { + t.Fatal("expected chain enqueue error") + } + if catchCount != 1 { + t.Fatalf("expected catch once, got %d", catchCount) + } + if finallyCount != 1 { + t.Fatalf("expected finally once, got %d", finallyCount) + } + st, err := b.FindChain(context.Background(), chainID) + if err != nil { + t.Fatalf("find failed chain: %v", err) + } + if !st.Failed { + t.Fatalf("expected chain marked failed, got %+v", st) + } + b.mu.RLock() + cbCount := len(b.chainCallbacks) + b.mu.RUnlock() + if cbCount != 0 { + t.Fatalf("expected chain callbacks cleaned, got %d", cbCount) + } +} + +func TestBatchEnqueueFailureInvokesCatchAndFinally(t *testing.T) { + q := &failingDispatchQueue{err: errors.New("enqueue failed")} + bi, err := NewWithStore(q, NewMemoryStore()) + if err != nil { + t.Fatalf("new bus: %v", err) + } + b := bi.(*runtime) + + var catchCount int + var finallyCount int + batchID, err := b.Batch(NewJob("monitor:poll", nil)). + Catch(func(context.Context, BatchState, error) error { + catchCount++ + return nil + }). + Finally(func(context.Context, BatchState) error { + finallyCount++ + return nil + }). + Dispatch(context.Background()) + if err == nil { + t.Fatal("expected batch enqueue error") + } + if catchCount != 1 { + t.Fatalf("expected catch once, got %d", catchCount) + } + if finallyCount != 1 { + t.Fatalf("expected finally once, got %d", finallyCount) + } + st, err := b.FindBatch(context.Background(), batchID) + if err != nil { + t.Fatalf("find failed batch: %v", err) + } + if !st.Completed || !st.Cancelled { + t.Fatalf("expected batch cancelled+completed, got %+v", st) + } + b.mu.RLock() + cbCount := len(b.batchCallbacks) + b.mu.RUnlock() + if cbCount != 0 { + t.Fatalf("expected batch callbacks cleaned, got %d", cbCount) + } +} + +func TestChainDispatchFailureStillReturnsChainID(t *testing.T) { + q := newSyncTestRuntime() + b, err := New(q) + if err != nil { + t.Fatalf("new bus: %v", err) + } + if err := b.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + b.Register("monitor:downsample", func(context.Context, Context) error { return errors.New("boom") }) + + chainID, err := b.Chain(NewJob("monitor:downsample", nil)).Dispatch(context.Background()) + if err == nil { + t.Fatal("expected chain dispatch error") + } + if chainID == "" { + t.Fatal("expected non-empty chain id on dispatch error") + } +} + +func TestBatchDispatchFailureStillReturnsBatchID(t *testing.T) { + q := newSyncTestRuntime() + b, err := New(q) + if err != nil { + t.Fatalf("new bus: %v", err) + } + if err := b.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + b.Register("monitor:downsample", func(context.Context, Context) error { return errors.New("boom") }) + + batchID, err := b.Batch(NewJob("monitor:downsample", nil)).Dispatch(context.Background()) + if err == nil { + t.Fatal("expected batch dispatch error") + } + if batchID == "" { + t.Fatal("expected non-empty batch id on dispatch error") + } +} diff --git a/internal/workflow/middleware.go b/internal/workflow/middleware.go new file mode 100644 index 0000000..086457e --- /dev/null +++ b/internal/workflow/middleware.go @@ -0,0 +1,33 @@ +package workflow + +import ( + "context" +) + +// Next represents the remaining middleware and handler execution chain. +type Next func(ctx context.Context, jc Context) error + +// Middleware can intercept logical workflow job execution. +type Middleware interface { + // Handle wraps the remaining middleware and handler chain. + Handle(ctx context.Context, jc Context, next Next) error +} + +// chainMiddleware composes middleware in declaration order around the final handler. +func chainMiddleware(middlewares []Middleware, final Next) Next { + if len(middlewares) == 0 { + return final + } + next := final + for i := len(middlewares) - 1; i >= 0; i-- { + m := middlewares[i] + if m == nil { + continue + } + currentNext := next + next = func(ctx context.Context, jc Context) error { + return m.Handle(ctx, jc, currentNext) + } + } + return next +} diff --git a/internal/workflow/protocol.go b/internal/workflow/protocol.go new file mode 100644 index 0000000..19bac08 --- /dev/null +++ b/internal/workflow/protocol.go @@ -0,0 +1,78 @@ +package workflow + +import "encoding/json" + +const ( + // ProtocolSchemaVersion identifies the workflow delivery envelope understood by this version of the library. + ProtocolSchemaVersion = 1 + // DirectDeliveryType identifies the retained version-one envelope for legacy + // direct jobs and application types that collide with reserved protocol names. + DirectDeliveryType = "bus:job" + // ChainNodeDeliveryType identifies one sequential workflow node delivery. + ChainNodeDeliveryType = "bus:chain:node" + // BatchJobDeliveryType identifies one aggregate workflow member delivery. + BatchJobDeliveryType = "bus:batch:job" + // CallbackDeliveryType identifies one ephemeral workflow callback delivery. + CallbackDeliveryType = "bus:callback" +) + +// ResolvedDelivery contains the logical application identity and workflow +// correlation recovered from a physical workflow delivery. +type ResolvedDelivery struct { + // JobType is the logical application type when a supported envelope provides one, otherwise the physical type. + JobType string + // Payload is the logical application payload when a supported envelope provides one, otherwise the physical payload. + Payload []byte + // DispatchID correlates deliveries created by the same application dispatch. + DispatchID string + // JobID identifies the logical workflow job represented by this delivery. + JobID string + // ChainID identifies the owning chain when the delivery belongs to one. + ChainID string + // BatchID identifies the owning batch when the delivery belongs to one. + BatchID string +} + +// ResolveDelivery decodes the owned protocol while preserving physical +// identity for unsupported or malformed input. +func ResolveDelivery(deliveryType string, payload []byte) ResolvedDelivery { + metadata := ResolvedDelivery{JobType: deliveryType, Payload: payload} + if deliveryType == "" || len(payload) == 0 || !IsDeliveryType(deliveryType) { + return metadata + } + + var envelope struct { + SchemaVersion int `json:"schema_version"` + DispatchID string `json:"dispatch_id"` + JobID string `json:"job_id"` + ChainID string `json:"chain_id"` + BatchID string `json:"batch_id"` + Job struct { + Type string `json:"type"` + Payload []byte `json:"payload"` + } `json:"job"` + } + if err := json.Unmarshal(payload, &envelope); err != nil || envelope.SchemaVersion != ProtocolSchemaVersion { + return metadata + } + + metadata.DispatchID = envelope.DispatchID + metadata.JobID = envelope.JobID + metadata.ChainID = envelope.ChainID + metadata.BatchID = envelope.BatchID + if envelope.Job.Type != "" { + metadata.JobType = envelope.Job.Type + metadata.Payload = envelope.Job.Payload + } + return metadata +} + +// IsDeliveryType reports whether a physical job type belongs to the workflow protocol. +func IsDeliveryType(deliveryType string) bool { + switch deliveryType { + case DirectDeliveryType, ChainNodeDeliveryType, BatchJobDeliveryType, CallbackDeliveryType: + return true + default: + return false + } +} diff --git a/internal/workflow/protocol_test.go b/internal/workflow/protocol_test.go new file mode 100644 index 0000000..a851ef4 --- /dev/null +++ b/internal/workflow/protocol_test.go @@ -0,0 +1,158 @@ +package workflow + +import ( + "reflect" + "testing" +) + +// TestProtocolConstants pins the persisted schema and physical delivery namespace. +func TestProtocolConstants(t *testing.T) { + if ProtocolSchemaVersion != 1 { + t.Fatalf("protocol schema version = %d, want 1", ProtocolSchemaVersion) + } + types := []struct { + name string + got string + want string + }{ + {name: "direct", got: DirectDeliveryType, want: "bus:job"}, + {name: "chain node", got: ChainNodeDeliveryType, want: "bus:chain:node"}, + {name: "batch job", got: BatchJobDeliveryType, want: "bus:batch:job"}, + {name: "callback", got: CallbackDeliveryType, want: "bus:callback"}, + } + for _, test := range types { + t.Run(test.name, func(t *testing.T) { + if test.got != test.want { + t.Fatalf("delivery type = %q, want %q", test.got, test.want) + } + if !IsDeliveryType(test.got) { + t.Fatalf("IsDeliveryType(%q) = false, want true", test.got) + } + }) + } + if IsDeliveryType("bus:tenant") { + t.Fatal("IsDeliveryType accepted an unowned bus namespace") + } +} + +// TestResolveDelivery decodes every version-one delivery shape without constructing production envelopes. +func TestResolveDelivery(t *testing.T) { + tests := []struct { + name string + deliveryType string + payload []byte + want ResolvedDelivery + }{ + { + name: "direct base64 payload", + deliveryType: DirectDeliveryType, + payload: []byte(`{"schema_version":1,"dispatch_id":"dsp_direct","job_id":"job_direct","job":{"type":"reports:build","payload":"eyJpZCI6MX0="}}`), + want: ResolvedDelivery{ + JobType: "reports:build", + Payload: []byte(`{"id":1}`), + DispatchID: "dsp_direct", + JobID: "job_direct", + }, + }, + { + name: "chain null payload", + deliveryType: ChainNodeDeliveryType, + payload: []byte(`{"schema_version":1,"dispatch_id":"dsp_chain","job_id":"job_chain","chain_id":"chn_1","job":{"type":"reports:chain","payload":null}}`), + want: ResolvedDelivery{ + JobType: "reports:chain", + Payload: nil, + DispatchID: "dsp_chain", + JobID: "job_chain", + ChainID: "chn_1", + }, + }, + { + name: "batch empty payload", + deliveryType: BatchJobDeliveryType, + payload: []byte(`{"schema_version":1,"dispatch_id":"dsp_batch","job_id":"job_batch","batch_id":"bat_1","job":{"type":"reports:batch","payload":""}}`), + want: ResolvedDelivery{ + JobType: "reports:batch", + Payload: []byte{}, + DispatchID: "dsp_batch", + JobID: "job_batch", + BatchID: "bat_1", + }, + }, + { + name: "callback omitted payload", + deliveryType: CallbackDeliveryType, + payload: []byte(`{"schema_version":1,"dispatch_id":"dsp_callback","job_id":"job_callback","chain_id":"chn_2","job":{"type":"reports:callback"}}`), + want: ResolvedDelivery{ + JobType: "reports:callback", + Payload: nil, + DispatchID: "dsp_callback", + JobID: "job_callback", + ChainID: "chn_2", + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := ResolveDelivery(test.deliveryType, test.payload); !reflect.DeepEqual(got, test.want) { + t.Fatalf("ResolveDelivery() = %#v, want %#v", got, test.want) + } + }) + } +} + +// TestResolveDeliveryFallbacks pins physical identity whenever the protocol cannot safely unwrap a delivery. +func TestResolveDeliveryFallbacks(t *testing.T) { + unknownSchema := []byte(`{"schema_version":2,"dispatch_id":"dsp_unknown","job":{"type":"reports:unknown","payload":"e30="}}`) + malformed := []byte(`{"schema_version":1`) + nonWorkflow := []byte(`{"schema_version":1,"dispatch_id":"dsp_spoofed","job":{"type":"reports:spoofed","payload":"e30="}}`) + emptyJob := []byte(`{"schema_version":1,"dispatch_id":"dsp_callback","job_id":"job_callback","batch_id":"bat_2","job":{}}`) + tests := []struct { + name string + deliveryType string + payload []byte + want ResolvedDelivery + }{ + { + name: "nil payload", + deliveryType: DirectDeliveryType, + want: ResolvedDelivery{JobType: DirectDeliveryType}, + }, + { + name: "malformed json", + deliveryType: DirectDeliveryType, + payload: malformed, + want: ResolvedDelivery{JobType: DirectDeliveryType, Payload: malformed}, + }, + { + name: "unknown schema", + deliveryType: DirectDeliveryType, + payload: unknownSchema, + want: ResolvedDelivery{JobType: DirectDeliveryType, Payload: unknownSchema}, + }, + { + name: "non-workflow type", + deliveryType: "application:job", + payload: nonWorkflow, + want: ResolvedDelivery{JobType: "application:job", Payload: nonWorkflow}, + }, + { + name: "valid callback without logical type", + deliveryType: CallbackDeliveryType, + payload: emptyJob, + want: ResolvedDelivery{ + JobType: CallbackDeliveryType, + Payload: emptyJob, + DispatchID: "dsp_callback", + JobID: "job_callback", + BatchID: "bat_2", + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := ResolveDelivery(test.deliveryType, test.payload); !reflect.DeepEqual(got, test.want) { + t.Fatalf("ResolveDelivery() = %#v, want %#v", got, test.want) + } + }) + } +} diff --git a/internal/workflow/runtime.go b/internal/workflow/runtime.go new file mode 100644 index 0000000..84064cd --- /dev/null +++ b/internal/workflow/runtime.go @@ -0,0 +1,780 @@ +package workflow + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "sync" + "time" + + "github.com/goforj/queue/busruntime" + "github.com/goforj/queue/internal/jobidentity" + "github.com/goforj/queue/internal/observation" +) + +const ( + // schemaVersion keeps the existing private name while the protocol becomes engine-owned. + schemaVersion = ProtocolSchemaVersion + // eventSchemaVersion follows the shared observer envelope independently of + // the workflow delivery protocol used to decode queued messages. + eventSchemaVersion = observation.EventSchemaVersion + // internalJob keeps the existing direct-delivery name stable on the wire. + internalJob = DirectDeliveryType + // internalJobChainNode keeps the existing chain-delivery name stable on the wire. + internalJobChainNode = ChainNodeDeliveryType + // internalJobBatchJob keeps the existing batch-delivery name stable on the wire. + internalJobBatchJob = BatchJobDeliveryType + // internalJobCallback keeps the existing callback-delivery name stable on the wire. + internalJobCallback = CallbackDeliveryType +) + +// Engine defines the orchestration surface shared by the public facade and compatibility adapters. +type Engine interface { + // Register binds a logical job type to its handler. + Register(jobType string, handler Handler) + + // Dispatch submits one logical job through the underlying delivery runtime. + Dispatch(ctx context.Context, job Job) (DispatchResult, error) + // DispatchDirect submits one frozen ordinary job without the legacy workflow envelope. + DispatchDirect(ctx context.Context, job StoredJob) (DispatchResult, error) + // Chain creates a sequential workflow builder. + Chain(jobs ...Job) ChainBuilder + // Batch creates an aggregate workflow builder. + Batch(jobs ...Job) BatchBuilder + + // StartWorkers starts the underlying delivery runtime. + StartWorkers(ctx context.Context) error + // Shutdown stops the underlying delivery runtime. + Shutdown(ctx context.Context) error + + // FindBatch returns persisted aggregate workflow state. + FindBatch(ctx context.Context, batchID string) (BatchState, error) + // FindChain returns persisted sequential workflow state. + FindChain(ctx context.Context, chainID string) (ChainState, error) + // Prune removes terminal workflow state older than before. + Prune(ctx context.Context, before time.Time) error +} + +// Option configures one workflow runtime before its internal handlers are registered. +type Option func(*runtime) + +// WithoutEphemeralCallbacks disables process-local callback retention for +// recording runtimes that never execute workflow deliveries. +func WithoutEphemeralCallbacks() Option { + return func(r *runtime) { + r.ephemeralCallbacksDisabled = true + } +} + +// WithObserver installs an event observer for workflow lifecycle facts. +func WithObserver(observer Observer) Option { + return func(r *runtime) { + r.observer = observer + } +} + +// WithStore overrides the orchestration store used for chain/batch/callback state. +func WithStore(store Store) Option { + return func(r *runtime) { + if store != nil { + r.store = store + } + } +} + +// WithClock overrides the runtime clock used for event/state timestamps. +func WithClock(clock func() time.Time) Option { + return func(r *runtime) { + if clock != nil { + r.now = clock + } + } +} + +// WithMiddleware appends middleware to the runtime execution chain. +func WithMiddleware(middlewares ...Middleware) Option { + return func(r *runtime) { + for _, m := range middlewares { + if m != nil { + r.middlewares = append(r.middlewares, m) + } + } + } +} + +// New creates a workflow engine using an in-memory orchestration store. +func New(q any, opts ...Option) (Engine, error) { + return NewWithStore(q, NewMemoryStore(), opts...) +} + +// NewWithStore creates a workflow engine with a custom orchestration store. +func NewWithStore(q any, store Store, opts ...Option) (Engine, error) { + if q == nil { + return nil, errors.New("queue is required") + } + qr, err := asRuntime(q) + if err != nil { + return nil, err + } + if store == nil { + store = NewMemoryStore() + } + r := &runtime{ + q: qr, + store: store, + now: time.Now, + handlers: make(map[string]Handler), + chainCallbacks: make(map[string]chainCallbacks), + batchCallbacks: make(map[string]batchCallbacks), + } + for _, opt := range opts { + if opt != nil { + opt(r) + } + } + + qr.BusRegister(internalJob, r.handleInternalJob) + qr.BusRegister(internalJobChainNode, r.handleInternalChainNode) + qr.BusRegister(internalJobBatchJob, r.handleInternalBatchJob) + qr.BusRegister(internalJobCallback, r.handleInternalCallback) + return r, nil +} + +// asRuntime narrows compatibility inputs to the transport contract required by workflow orchestration. +func asRuntime(v any) (busruntime.Runtime, error) { + if v == nil { + return nil, errors.New("queue is required") + } + if q, ok := v.(busruntime.Runtime); ok { + return q, nil + } + return nil, fmt.Errorf("queue does not support bus runtime adapter") +} + +type runtime struct { + q busruntime.Runtime + store Store + now func() time.Time + + observer Observer + + mu sync.RWMutex + handlers map[string]Handler + chainCallbacks map[string]chainCallbacks + batchCallbacks map[string]batchCallbacks + middlewares []Middleware + ephemeralCallbacksDisabled bool +} + +var _ Engine = (*runtime)(nil) + +// Register binds a job type to a handler. +func (r *runtime) Register(jobType string, handler Handler) { + if handler == nil { + return + } + r.mu.Lock() + r.handlers[jobType] = handler + r.mu.Unlock() + + _, ok := r.q.(busruntime.DirectRuntime) + if !ok || IsDeliveryType(jobType) { + return + } + r.q.BusRegister(jobType, func(ctx context.Context, job busruntime.InboundJob) error { + return r.handleDirectJob(ctx, jobType, job) + }) +} + +// Dispatch enqueues one job for execution. +func (r *runtime) Dispatch(ctx context.Context, job Job) (DispatchResult, error) { + stored, err := toStoredJob(job) + if err != nil { + return DispatchResult{}, err + } + return r.dispatch(ctx, stored, false) +} + +// DispatchDirect enqueues one ordinary job using its application type and payload. +func (r *runtime) DispatchDirect(ctx context.Context, job StoredJob) (DispatchResult, error) { + if job.Type == "" { + return DispatchResult{}, errors.New("bus job type is required") + } + job.Payload = append([]byte(nil), job.Payload...) + return r.dispatch(ctx, job, true) +} + +// dispatch applies one receipt and event contract to both canonical direct +// delivery and the retained legacy envelope route. +func (r *runtime) dispatch(ctx context.Context, job StoredJob, direct bool) (DispatchResult, error) { + dispatchID := newID("dsp") + env := envelope{ + SchemaVersion: schemaVersion, + DispatchID: dispatchID, + Kind: "job", + JobID: newID("job"), + Job: job, + } + r.emit(ctx, Event{SchemaVersion: eventSchemaVersion, EventID: newID("evt"), Kind: EventDispatchStarted, DispatchID: dispatchID, JobID: env.JobID, JobType: job.Type, JobKey: storedJobEventKey(job), Queue: job.Options.Queue, Time: r.now()}) + dispatch := func() error { return r.dispatchEnvelope(ctx, internalJob, env) } + if direct { + dispatch = func() error { return r.dispatchDirectEnvelope(ctx, env) } + } + if err := dispatch(); err != nil { + if executionErr, ok := acceptedDispatchExecutionError(err); ok { + r.emit(ctx, Event{SchemaVersion: eventSchemaVersion, EventID: newID("evt"), Kind: EventDispatchSucceeded, DispatchID: dispatchID, JobID: env.JobID, JobType: job.Type, JobKey: storedJobEventKey(job), Queue: job.Options.Queue, Time: r.now()}) + return DispatchResult{DispatchID: dispatchID}, executionErr + } + r.emit(ctx, Event{SchemaVersion: eventSchemaVersion, EventID: newID("evt"), Kind: EventDispatchFailed, DispatchID: dispatchID, JobID: env.JobID, JobType: job.Type, JobKey: storedJobEventKey(job), Queue: job.Options.Queue, Time: r.now(), Err: err}) + return DispatchResult{DispatchID: dispatchID}, err + } + r.emit(ctx, Event{SchemaVersion: eventSchemaVersion, EventID: newID("evt"), Kind: EventDispatchSucceeded, DispatchID: dispatchID, JobID: env.JobID, JobType: job.Type, JobKey: storedJobEventKey(job), Queue: job.Options.Queue, Time: r.now()}) + return DispatchResult{DispatchID: dispatchID}, nil +} + +type acceptedDispatchError interface { + error + DispatchAccepted() bool + Unwrap() error +} + +// acceptedDispatchExecutionError separates synchronous execution failure from enqueue rejection without coupling workflow to root types. +func acceptedDispatchExecutionError(err error) (error, bool) { + var accepted acceptedDispatchError + if !errors.As(err, &accepted) || !accepted.DispatchAccepted() { + return nil, false + } + return accepted.Unwrap(), true +} + +// Chain creates a sequential workflow where each job runs only after the prior job succeeds. +func (r *runtime) Chain(jobs ...Job) ChainBuilder { + return &chainBuilder{r: r, jobs: append([]Job(nil), jobs...)} +} + +// Batch creates a parallel workflow and tracks aggregate completion state. +func (r *runtime) Batch(jobs ...Job) BatchBuilder { + return &batchBuilder{r: r, jobs: append([]Job(nil), jobs...)} +} + +// StartWorkers starts the underlying queue worker runtime. +func (r *runtime) StartWorkers(ctx context.Context) error { return r.q.StartWorkers(ctx) } + +// Shutdown stops the underlying queue worker runtime. +func (r *runtime) Shutdown(ctx context.Context) error { return r.q.Shutdown(ctx) } + +// FindBatch returns persisted batch state by id. +func (r *runtime) FindBatch(ctx context.Context, batchID string) (BatchState, error) { + return r.store.GetBatch(ctx, batchID) +} + +// FindChain returns persisted chain state by id. +func (r *runtime) FindChain(ctx context.Context, chainID string) (ChainState, error) { + return r.store.GetChain(ctx, chainID) +} + +// Prune removes terminal orchestration records older than before. +func (r *runtime) Prune(ctx context.Context, before time.Time) error { + return r.store.Prune(ctx, before) +} + +// dispatchEnvelope serializes one workflow delivery onto the underlying queue runtime. +func (r *runtime) dispatchEnvelope(ctx context.Context, jobType string, env envelope) error { + payload, err := json.Marshal(env) + if err != nil { + return err + } + return r.q.BusDispatch(ctx, jobType, payload, busruntime.JobOptions{ + Queue: env.Job.Options.Queue, + Delay: env.Job.Options.Delay, + Timeout: env.Job.Options.Timeout, + Retry: env.Job.Options.Retry, + Backoff: env.Job.Options.Backoff, + UniqueFor: env.Job.Options.UniqueFor, + }) +} + +// dispatchDirectEnvelope carries correlation through the driver metadata plane. +// Runtimes without that capability and reserved legacy names keep the frozen +// version-one envelope route. +func (r *runtime) dispatchDirectEnvelope(ctx context.Context, env envelope) error { + direct, ok := r.q.(busruntime.DirectRuntime) + if !ok || IsDeliveryType(env.Job.Type) { + return r.dispatchEnvelope(ctx, internalJob, env) + } + return direct.BusDispatchDirect(ctx, env.Job.Type, env.Job.Payload, busruntime.DeliveryMetadata{ + SchemaVersion: busruntime.DeliveryMetadataVersion, + DispatchID: env.DispatchID, + JobID: env.JobID, + ChainID: env.ChainID, + BatchID: env.BatchID, + Queue: env.Job.Options.Queue, + }, busruntime.JobOptions{ + Queue: env.Job.Options.Queue, + Delay: env.Job.Options.Delay, + Timeout: env.Job.Options.Timeout, + Retry: env.Job.Options.Retry, + Backoff: env.Job.Options.Backoff, + UniqueFor: env.Job.Options.UniqueFor, + }) +} + +// dispatchCallback schedules only configured ephemeral closures through the same queue delivery path. +func (r *runtime) dispatchCallback(ctx context.Context, base envelope, kind string, err error) error { + callback, ok := r.callbackEnvelope(base, kind, err) + if !ok { + return nil + } + return r.dispatchEnvelope(ctx, internalJobCallback, callback) +} + +// callbackEnvelope constructs one configured ephemeral callback delivery without coupling invocation to its transport. +func (r *runtime) callbackEnvelope(base envelope, kind string, err error) (envelope, bool) { + if !r.callbackConfigured(base, kind) { + return envelope{}, false + } + callback := envelope{ + SchemaVersion: schemaVersion, + DispatchID: base.DispatchID, + Kind: "callback", + JobID: newID("job"), + ChainID: base.ChainID, + BatchID: base.BatchID, + CallbackKind: kind, + Job: StoredJob{ + Type: base.Job.Type, + Payload: append([]byte(nil), base.Job.Payload...), + Options: JobOptions{ + Queue: base.Job.Options.Queue, + }, + }, + } + if err != nil { + callback.Error = err.Error() + } + return callback, true +} + +// invokeCallbackInline preserves callback lifecycle semantics when initial workflow enqueue never reaches a worker. +func (r *runtime) invokeCallbackInline(ctx context.Context, base envelope, kind string, err error) error { + callback, ok := r.callbackEnvelope(base, kind, err) + if !ok { + return nil + } + return r.handleCallbackEnvelope(ctx, callback) +} + +// callbackConfigured keeps absent optional closures from becoming artificial callback deliveries and success facts. +func (r *runtime) callbackConfigured(base envelope, kind string) bool { + r.mu.RLock() + defer r.mu.RUnlock() + switch kind { + case "chain_catch": + return r.chainCallbacks[base.ChainID].catch != nil + case "chain_finally": + return r.chainCallbacks[base.ChainID].finally != nil + case "batch_catch": + return r.batchCallbacks[base.BatchID].catch != nil + case "batch_then": + return r.batchCallbacks[base.BatchID].then != nil + case "batch_finally": + return r.batchCallbacks[base.BatchID].finally != nil + default: + return false + } +} + +// handleInternalJob decodes and executes one direct workflow delivery. +func (r *runtime) handleInternalJob(ctx context.Context, job busruntime.InboundJob) error { + var env envelope + if err := job.Bind(&env); err != nil { + return err + } + return r.executeStoredJob(ctx, env) +} + +// handleDirectJob reconstructs engine context from driver metadata while +// leaving the application's type and payload untouched. +func (r *runtime) handleDirectJob(ctx context.Context, jobType string, job busruntime.InboundJob) error { + metadata, _ := busruntime.DeliveryMetadataFromContext(ctx) + attempt, _ := busruntime.DeliveryAttemptFromContext(ctx) + return r.executeStoredJob(ctx, envelope{ + SchemaVersion: schemaVersion, + DispatchID: metadata.DispatchID, + Kind: "job", + JobID: metadata.JobID, + ChainID: metadata.ChainID, + BatchID: metadata.BatchID, + Attempt: attempt.Number, + Job: StoredJob{ + Type: jobType, + Payload: job.PayloadBytes(), + Options: JobOptions{ + Queue: metadata.Queue, + Retry: attempt.MaxRetry, + }, + }, + }) +} + +// storedJobOutcome carries an attempt result until its owning workflow mutation commits. +type storedJobOutcome struct { + env envelope + attempt busruntime.DeliveryAttempt + started time.Time + finished time.Time + err error +} + +// executeStoredJob preserves direct-job behavior while allowing workflows to defer terminal facts until their state commits. +func (r *runtime) executeStoredJob(ctx context.Context, env envelope) error { + outcome := r.executeStoredJobAttempt(ctx, env) + r.emitStoredJobOutcome(ctx, outcome) + return outcome.err +} + +// executeStoredJobAttempt runs one logical handler attempt without claiming its terminal workflow state committed. +func (r *runtime) executeStoredJobAttempt(ctx context.Context, env envelope) storedJobOutcome { + attempt := applyDeliveryAttempt(ctx, &env) + started := r.now() + r.emit(ctx, Event{ + SchemaVersion: eventSchemaVersion, + EventID: newID("evt"), + Kind: EventJobStarted, + DispatchID: env.DispatchID, + JobID: env.JobID, + ChainID: env.ChainID, + BatchID: env.BatchID, + Attempt: env.Attempt, + JobType: env.Job.Type, + JobKey: storedJobEventKey(env.Job), + Queue: env.Job.Options.Queue, + Time: started, + }) + handler, ok := r.lookupHandler(env.Job.Type) + if !ok { + err := fmt.Errorf("bus handler not registered for %q", env.Job.Type) + return storedJobOutcome{env: env, attempt: attempt, started: started, finished: r.now(), err: err} + } + jc := NewContext( + schemaVersion, + env.DispatchID, + env.JobID, + env.ChainID, + env.BatchID, + env.Attempt, + env.Job.Type, + env.Job.Payload, + ) + err := chainMiddleware(r.middlewareSnapshot(), func(ctx context.Context, c Context) error { + return handler(ctx, c) + })(ctx, jc) + return storedJobOutcome{env: env, attempt: attempt, started: started, finished: r.now(), err: err} +} + +// emitStoredJobOutcome publishes only terminal logical facts selected by the shared attempt classifier. +func (r *runtime) emitStoredJobOutcome(ctx context.Context, outcome storedJobOutcome) { + kind := EventJobSucceeded + if outcome.err != nil { + if busruntime.ClassifyAttempt(outcome.attempt, outcome.err) != busruntime.AttemptFailed { + return + } + kind = EventJobFailed + } + r.emit(ctx, Event{ + SchemaVersion: eventSchemaVersion, + EventID: storedJobOutcomeFactID(kind, outcome), + Kind: kind, + DispatchID: outcome.env.DispatchID, + JobID: outcome.env.JobID, + ChainID: outcome.env.ChainID, + BatchID: outcome.env.BatchID, + Attempt: outcome.env.Attempt, + JobType: outcome.env.Job.Type, + JobKey: storedJobEventKey(outcome.env.Job), + Queue: outcome.env.Job.Options.Queue, + Duration: outcome.finished.Sub(outcome.started), + Time: outcome.finished, + Err: outcome.err, + }) +} + +// recoveredStoredJobSuccess reconstructs only the success identity persisted +// by an immutable transition receipt, never from queue recovery evidence alone. +func recoveredStoredJobSuccess(outcome storedJobOutcome, receipt transitionReceipt, observed time.Time) (storedJobOutcome, error) { + if receipt.outcome != BatchJobSucceeded { + return storedJobOutcome{}, errors.New("transition receipt does not own success") + } + if err := validateRecoveredTransitionFactIdentity(outcome.env, receipt); err != nil { + return storedJobOutcome{}, err + } + outcome.attempt.Number = receipt.owner.attempt + outcome.started = observed + outcome.finished = observed + outcome.err = nil + return outcome, nil +} + +// validateRecoveredTransitionReceipt proves durable logical identity without +// requiring this physical delivery to own reconstructed observer facts. +func validateRecoveredTransitionReceipt(env envelope, receipt transitionReceipt, requireOwnerJobID bool) error { + if err := validateTransitionReceiptSupport(receipt); err != nil { + return err + } + if !receipt.owner.valid() { + return errors.New("transition receipt has incomplete owner identity") + } + if env.DispatchID == "" { + return errors.New("delivery dispatch id is required") + } + if env.JobID == "" { + return errors.New("delivery job id is required") + } + if receipt.owner.dispatchID != env.DispatchID { + return errors.New("transition receipt dispatch does not match delivery") + } + if receipt.owner.jobFingerprint != storedJobReceiptFingerprint(env.Job) { + return errors.New("transition receipt job fingerprint does not match delivery") + } + if requireOwnerJobID && receipt.owner.jobID != env.JobID { + return errors.New("transition receipt member job id does not match delivery") + } + return nil +} + +// validateRecoveredTransitionFactIdentity requires the exact attempt and +// physical job that originally committed a receipt-backed observer fact. +func validateRecoveredTransitionFactIdentity(env envelope, receipt transitionReceipt) error { + if err := validateRecoveredTransitionReceipt(env, receipt, true); err != nil { + return err + } + if receipt.owner.attempt != env.Attempt { + return fmt.Errorf("transition receipt attempt %d does not match delivery attempt %d", receipt.owner.attempt, env.Attempt) + } + return nil +} + +// transitionReceiptOwnsRecoveredFacts reports whether the recovered physical +// generation, attempt, and job all match the receipt's immutable fact owner. +func transitionReceiptOwnsRecoveredFacts(env envelope, receipt transitionReceipt, provenance busruntime.DeliveryProvenance) bool { + return provenance.RecoveredGenerationID != "" && + receipt.owner.deliveryID == provenance.RecoveredGenerationID && + receipt.owner.attempt == env.Attempt && + receipt.owner.jobID == env.JobID +} + +// transitionClaimFromOutcome binds workflow mutation provenance to the exact +// physical settlement generation currently executing the logical attempt. +func transitionClaimFromOutcome(ctx context.Context, outcome storedJobOutcome) transitionClaim { + provenance, _ := busruntime.DeliveryProvenanceFromContext(ctx) + return transitionClaim{ + deliveryID: provenance.GenerationID, + attempt: outcome.env.Attempt, + dispatchID: outcome.env.DispatchID, + jobID: outcome.env.JobID, + jobFingerprint: storedJobReceiptFingerprint(outcome.env.Job), + } +} + +// recoveredDeliveryProvenance reports stale-generation evidence without +// confusing it with proof that the earlier generation changed workflow state. +func recoveredDeliveryProvenance(ctx context.Context) (busruntime.DeliveryProvenance, bool) { + provenance, ok := busruntime.DeliveryProvenanceFromContext(ctx) + return provenance, ok && provenance.Recovered +} + +// markDeliveryTransitionCommitted lets a retaining transport preserve the +// current receipt owner if later workflow infrastructure still needs same-attempt redelivery. +func markDeliveryTransitionCommitted(ctx context.Context, claimedNow, receiptKnown bool) { + if claimedNow && receiptKnown { + busruntime.MarkDeliveryApplicationStateCommitted(ctx) + } +} + +// storedJobReceiptFingerprint hashes every immutable job field needed to +// reject a retained row whose observable payload or delivery policy differs. +func storedJobReceiptFingerprint(job StoredJob) string { + hash := sha256.New() + values := []string{ + job.Type, + string(job.Payload), + job.Options.Queue, + fmt.Sprintf("%d", job.Options.Delay), + fmt.Sprintf("%d", job.Options.Timeout), + fmt.Sprintf("%d", job.Options.Retry), + fmt.Sprintf("%d", job.Options.Backoff), + fmt.Sprintf("%d", job.Options.UniqueFor), + } + var size [8]byte + for _, value := range values { + binary.BigEndian.PutUint64(size[:], uint64(len(value))) + _, _ = hash.Write(size[:]) + _, _ = hash.Write([]byte(value)) + } + return hex.EncodeToString(hash.Sum(nil)) +} + +// storedJobOutcomeFactID uses deterministic identity only when canonical +// correlation distinguishes this logical delivery from unrelated legacy or +// malformed inputs that carry the same type and attempt number. +func storedJobOutcomeFactID(kind EventKind, outcome storedJobOutcome) string { + if kind != EventJobSucceeded || outcome.env.DispatchID == "" || outcome.env.JobID == "" { + return newID("evt") + } + return stableWorkflowFactID( + kind, + outcome.env.DispatchID, + outcome.env.JobID, + outcome.env.ChainID, + outcome.env.BatchID, + outcome.env.Job.Type, + storedJobEventKey(outcome.env.Job), + outcome.env.Job.Options.Queue, + fmt.Sprintf("%d", outcome.env.Attempt), + ) +} + +// stableWorkflowFactID length-frames logical identity before hashing so a +// redelivery can republish the same truthful fact without inventing an +// unrelated identifier or conflating differently partitioned values. +func stableWorkflowFactID(kind EventKind, identity ...string) string { + hash := sha256.New() + values := make([]string, 0, len(identity)+2) + values = append(values, fmt.Sprintf("%d", eventSchemaVersion), string(kind)) + values = append(values, identity...) + var size [8]byte + for _, value := range values { + binary.BigEndian.PutUint64(size[:], uint64(len(value))) + _, _ = hash.Write(size[:]) + _, _ = hash.Write([]byte(value)) + } + sum := hash.Sum(nil) + return "evt_" + hex.EncodeToString(sum[:16]) +} + +// storedJobEventKey keeps workflow facts on the same logical type-and-payload correlation as queue and worker facts. +func storedJobEventKey(job StoredJob) string { + return jobidentity.ObservedKey(job.Type, job.Payload) +} + +// uncommittedMutationError marks state persistence failures for same-attempt infrastructure redelivery. +func uncommittedMutationError(operation string, err error) error { + return busruntime.Uncommitted(fmt.Errorf("%s: %w", operation, err)) +} + +// applyDeliveryAttempt replaces the stale envelope attempt with metadata supplied by the physical worker. +func applyDeliveryAttempt(ctx context.Context, env *envelope) busruntime.DeliveryAttempt { + if attempt, ok := busruntime.DeliveryAttemptFromContext(ctx); ok { + env.Attempt = attempt.Number + return attempt + } + return busruntime.DeliveryAttempt{ + Number: env.Attempt, + MaxRetry: env.Job.Options.Retry, + } +} + +// middlewareSnapshot isolates an execution attempt from concurrent middleware registration. +func (r *runtime) middlewareSnapshot() []Middleware { + r.mu.RLock() + defer r.mu.RUnlock() + out := make([]Middleware, len(r.middlewares)) + copy(out, r.middlewares) + return out +} + +// lookupHandler resolves a handler without retaining the registration lock during application execution. +func (r *runtime) lookupHandler(jobType string) (Handler, bool) { + r.mu.RLock() + defer r.mu.RUnlock() + handler, ok := r.handlers[jobType] + return handler, ok +} + +// emit delays positive workflow facts when the physical driver owns a later settlement boundary. +func (r *runtime) emit(ctx context.Context, event Event) { + if eventWaitsForDeliverySettlement(event.Kind) && busruntime.DeferUntilDeliveryCommitted(ctx, func() { + safeObserve(ctx, r.observer, event) + }) { + return + } + safeObserve(ctx, r.observer, event) +} + +// eventWaitsForDeliverySettlement identifies positive workflow facts that would be false if broker acknowledgement remains unresolved. +func eventWaitsForDeliverySettlement(kind EventKind) bool { + switch kind { + case EventJobSucceeded, + EventChainAdvanced, + EventChainCompleted, + EventBatchProgressed, + EventBatchCompleted, + EventCallbackSucceeded: + return true + default: + return false + } +} + +// runEphemeralCallback converts application panics into callback failures without unwinding committed workflow state. +func runEphemeralCallback(callback func() error) (err error) { + defer func() { + if recovered := recover(); recovered != nil { + if recoveredErr, ok := recovered.(error); ok { + err = fmt.Errorf("workflow callback panicked: %w", recoveredErr) + return + } + err = fmt.Errorf("workflow callback panicked: %v", recovered) + } + }() + return callback() +} + +// StoredJob is the version-one logical job representation embedded in workflow state and delivery envelopes. +type StoredJob struct { + Type string `json:"type"` + Payload []byte `json:"payload"` + Options JobOptions `json:"options"` +} + +// toStoredJob validates and serializes an application payload into the stable workflow representation. +func toStoredJob(job Job) (StoredJob, error) { + if job.Type == "" { + return StoredJob{}, errors.New("bus job type is required") + } + payload, err := json.Marshal(job.Payload) + if err != nil { + return StoredJob{}, err + } + return StoredJob{ + Type: job.Type, + Payload: payload, + Options: job.Options, + }, nil +} + +type envelope struct { + SchemaVersion int `json:"schema_version"` + DispatchID string `json:"dispatch_id"` + Kind string `json:"kind"` + JobID string `json:"job_id"` + ChainID string `json:"chain_id,omitempty"` + BatchID string `json:"batch_id,omitempty"` + NodeID string `json:"node_id,omitempty"` + Attempt int `json:"attempt"` + Job StoredJob `json:"job"` + CallbackKind string `json:"callback_kind,omitempty"` + Error string `json:"error,omitempty"` +} + +// newID creates correlation identifiers with the legacy prefix and random hexadecimal shape. +func newID(prefix string) string { + var b [8]byte + _, _ = rand.Read(b[:]) + return prefix + "_" + hex.EncodeToString(b[:]) +} diff --git a/internal/workflow/runtime_cleanup_test.go b/internal/workflow/runtime_cleanup_test.go new file mode 100644 index 0000000..90fb75c --- /dev/null +++ b/internal/workflow/runtime_cleanup_test.go @@ -0,0 +1,610 @@ +package workflow + +import ( + "context" + "encoding/json" + "errors" + "sync" + "sync/atomic" + "testing" + "time" +) + +type blockingBatchCompletionStore struct { + Store + blockedJob string + committed chan struct{} + release chan struct{} + once sync.Once +} + +// MarkBatchJobSucceeded pauses one already-committed outcome so another job can prepare terminal callbacks first. +func (s *blockingBatchCompletionStore) MarkBatchJobSucceeded(ctx context.Context, batchID, jobID string) (BatchState, bool, error) { + state, done, err := s.Store.MarkBatchJobSucceeded(ctx, batchID, jobID) + if err == nil && jobID == s.blockedJob { + s.once.Do(func() { close(s.committed) }) + <-s.release + } + return state, done, err +} + +func TestRuntimeCleansChainCallbacksAfterFinally(t *testing.T) { + q := newSyncTestRuntime() + bi, err := New(q) + if err != nil { + t.Fatalf("new bus: %v", err) + } + r := bi.(*runtime) + if err := r.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + + r.Register("monitor:poll", func(context.Context, Context) error { return nil }) + if _, err := r.Chain(NewJob("monitor:poll", nil)). + Finally(func(context.Context, ChainState) error { return nil }). + Dispatch(context.Background()); err != nil { + t.Fatalf("dispatch chain: %v", err) + } + + r.mu.RLock() + n := len(r.chainCallbacks) + r.mu.RUnlock() + if n != 0 { + t.Fatalf("expected chain callbacks map cleaned, got len=%d", n) + } +} + +func TestRuntimeCleansBatchCallbacksAfterFinally(t *testing.T) { + q := newSyncTestRuntime() + bi, err := New(q) + if err != nil { + t.Fatalf("new bus: %v", err) + } + r := bi.(*runtime) + if err := r.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + + r.Register("monitor:poll", func(context.Context, Context) error { return nil }) + if _, err := r.Batch(NewJob("monitor:poll", nil)). + Finally(func(context.Context, BatchState) error { return nil }). + Dispatch(context.Background()); err != nil { + t.Fatalf("dispatch batch: %v", err) + } + + r.mu.RLock() + n := len(r.batchCallbacks) + r.mu.RUnlock() + if n != 0 { + t.Fatalf("expected batch callbacks map cleaned, got len=%d", n) + } +} + +// TestRuntimeWithoutEphemeralCallbacksDoesNotRetainClosures verifies recording +// runtimes can accept fluent callback APIs without owning process closures forever. +func TestRuntimeWithoutEphemeralCallbacksDoesNotRetainClosures(t *testing.T) { + engine, err := New(&failingDispatchQueue{}, WithoutEphemeralCallbacks()) + if err != nil { + t.Fatalf("new recording runtime: %v", err) + } + runtime := engine.(*runtime) + if _, err := runtime.Chain(NewJob("chain:record", nil)). + Catch(func(context.Context, ChainState, error) error { return nil }). + Finally(func(context.Context, ChainState) error { return nil }). + Dispatch(context.Background()); err != nil { + t.Fatalf("dispatch recording chain: %v", err) + } + if _, err := runtime.Batch(NewJob("batch:record", nil)). + Progress(func(context.Context, BatchState) error { return nil }). + Then(func(context.Context, BatchState) error { return nil }). + Catch(func(context.Context, BatchState, error) error { return nil }). + Finally(func(context.Context, BatchState) error { return nil }). + Dispatch(context.Background()); err != nil { + t.Fatalf("dispatch recording batch: %v", err) + } + runtime.mu.RLock() + chainCallbacks := len(runtime.chainCallbacks) + batchCallbacks := len(runtime.batchCallbacks) + runtime.mu.RUnlock() + if chainCallbacks != 0 || batchCallbacks != 0 { + t.Fatalf("recording runtime retained callbacks: chains=%d batches=%d", chainCallbacks, batchCallbacks) + } +} + +// TestChainFinallyPreservesPendingCatch verifies independently delivered terminal callbacks do not delete one another. +func TestChainFinallyPreservesPendingCatch(t *testing.T) { + const chainID = "chain_out_of_order_callbacks" + var catchCount int + var finallyCount int + runtime := &runtime{ + store: NewMemoryStore(), + chainCallbacks: map[string]chainCallbacks{ + chainID: { + catch: func(context.Context, ChainState, error) error { + catchCount++ + return nil + }, + finally: func(context.Context, ChainState) error { + finallyCount++ + return nil + }, + }, + }, + } + state := ChainState{ChainID: chainID, Failed: true} + + if err := runtime.invokeChainFinally(context.Background(), state); err != nil { + t.Fatalf("invoke finally: %v", err) + } + if err := runtime.invokeChainCatch(context.Background(), state, context.Canceled); err != nil { + t.Fatalf("invoke catch: %v", err) + } + if catchCount != 1 || finallyCount != 1 { + t.Fatalf("catch/finally count = %d/%d, want 1/1", catchCount, finallyCount) + } + if len(runtime.chainCallbacks) != 0 { + t.Fatalf("expected chain callbacks cleaned, got len=%d", len(runtime.chainCallbacks)) + } +} + +// TestBatchFinallyPreservesPendingTerminalCallbacks verifies terminal cleanup is independent of delivery order. +func TestBatchFinallyPreservesPendingTerminalCallbacks(t *testing.T) { + const batchID = "batch_out_of_order_callbacks" + var catchCount int + var thenCount int + var finallyCount int + runtime := &runtime{ + store: NewMemoryStore(), + batchCallbacks: map[string]batchCallbacks{ + batchID: { + progress: func(context.Context, BatchState) error { + t.Fatal("terminal preparation retained progress callback") + return nil + }, + then: func(context.Context, BatchState) error { + thenCount++ + return nil + }, + catch: func(context.Context, BatchState, error) error { + catchCount++ + return nil + }, + finally: func(context.Context, BatchState) error { + finallyCount++ + return nil + }, + }, + }, + } + state := BatchState{BatchID: batchID, Failed: 1, AllowFailed: true, Completed: true} + runtime.prepareBatchTerminalCallbacks(batchID, true, true) + + if err := runtime.invokeBatchFinally(context.Background(), state); err != nil { + t.Fatalf("invoke finally: %v", err) + } + if err := runtime.invokeBatchThen(context.Background(), state); err != nil { + t.Fatalf("invoke then: %v", err) + } + if err := runtime.invokeBatchCatch(context.Background(), state, context.Canceled); err != nil { + t.Fatalf("invoke catch: %v", err) + } + if catchCount != 1 || thenCount != 1 || finallyCount != 1 { + t.Fatalf("catch/then/finally count = %d/%d/%d, want 1/1/1", catchCount, thenCount, finallyCount) + } + if len(runtime.batchCallbacks) != 0 { + t.Fatalf("expected batch callbacks cleaned, got len=%d", len(runtime.batchCallbacks)) + } +} + +// TestBatchCallbackEnvelopesCompleteInReverseOrder verifies serialized deliveries preserve sibling callbacks, lifecycle facts, and idempotency. +func TestBatchCallbackEnvelopesCompleteInReverseOrder(t *testing.T) { + const batchID = "batch_reverse_callback_envelopes" + store := NewMemoryStore() + if err := store.CreateBatch(context.Background(), BatchRecord{ + BatchID: batchID, + DispatchID: "dispatch_reverse_callback_envelopes", + AllowFailed: true, + Jobs: []BatchJob{ + {JobID: "job_failed", Job: StoredJob{Type: "batch:failure"}}, + {JobID: "job_succeeded", Job: StoredJob{Type: "batch:success"}}, + }, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create batch: %v", err) + } + if _, _, err := store.MarkBatchJobFailed(context.Background(), batchID, "job_failed", errors.New("allowed failure")); err != nil { + t.Fatalf("mark failed job: %v", err) + } + state, done, err := store.MarkBatchJobSucceeded(context.Background(), batchID, "job_succeeded") + if err != nil { + t.Fatalf("mark successful job: %v", err) + } + if !done || !state.Completed || state.Failed != 1 || state.Cancelled { + t.Fatalf("terminal batch state = %+v, done=%t", state, done) + } + + var callbackCalls []string + var events []Event + runtime := &runtime{ + store: store, + now: time.Now, + observer: ObserverFunc(func(_ context.Context, event Event) { + events = append(events, event) + }), + chainCallbacks: make(map[string]chainCallbacks), + batchCallbacks: map[string]batchCallbacks{ + batchID: { + then: func(context.Context, BatchState) error { + callbackCalls = append(callbackCalls, "then") + return nil + }, + catch: func(context.Context, BatchState, error) error { + callbackCalls = append(callbackCalls, "catch") + return nil + }, + finally: func(context.Context, BatchState) error { + callbackCalls = append(callbackCalls, "finally") + return nil + }, + }, + }, + } + runtime.prepareBatchTerminalCallbacks(batchID, true, true) + + deliver := func(kind string) { + env := envelope{ + SchemaVersion: schemaVersion, + DispatchID: state.DispatchID, + JobID: "callback_" + kind, + BatchID: batchID, + Job: StoredJob{Options: JobOptions{Queue: "bulk"}}, + CallbackKind: kind, + Error: "allowed failure", + } + payload, marshalErr := json.Marshal(env) + if marshalErr != nil { + t.Fatalf("marshal %s callback: %v", kind, marshalErr) + } + if callbackErr := runtime.handleInternalCallback(context.Background(), testInboundJob{payload: payload}); callbackErr != nil { + t.Fatalf("deliver %s callback: %v", kind, callbackErr) + } + } + + deliver("batch_finally") + deliver("batch_then") + deliver("batch_catch") + if got := callbackCalls; len(got) != 3 || got[0] != "finally" || got[1] != "then" || got[2] != "catch" { + t.Fatalf("callback order = %v, want [finally then catch]", got) + } + if len(runtime.batchCallbacks) != 0 { + t.Fatalf("callback state retained after all siblings completed: %+v", runtime.batchCallbacks) + } + + var started, succeeded int + for _, event := range events { + switch event.Kind { + case EventCallbackStarted: + started++ + case EventCallbackSucceeded: + succeeded++ + } + } + if started != 3 || succeeded != 3 { + t.Fatalf("callback lifecycle facts = started:%d succeeded:%d, want 3/3", started, succeeded) + } + + deliver("batch_finally") + if len(events) != 6 || len(callbackCalls) != 3 { + t.Fatalf("duplicate callback emitted facts or ran application code: events=%d calls=%v", len(events), callbackCalls) + } +} + +// TestConcurrentBatchProgressUsesPerDeliverySnapshot verifies terminal cleanup cannot erase an earlier committed job's Progress hook. +func TestConcurrentBatchProgressUsesPerDeliverySnapshot(t *testing.T) { + const batchID = "batch_concurrent_progress" + baseStore := NewMemoryStore() + if err := baseStore.CreateBatch(context.Background(), BatchRecord{ + BatchID: batchID, + Jobs: []BatchJob{ + {JobID: "job_paused", Job: StoredJob{Type: "batch:item"}}, + {JobID: "job_final", Job: StoredJob{Type: "batch:item"}}, + }, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create batch: %v", err) + } + store := &blockingBatchCompletionStore{ + Store: baseStore, + blockedJob: "job_paused", + committed: make(chan struct{}), + release: make(chan struct{}), + } + var progressCalls atomic.Int32 + runtime := &runtime{ + store: store, + now: time.Now, + handlers: map[string]Handler{ + "batch:item": func(context.Context, Context) error { return nil }, + }, + chainCallbacks: make(map[string]chainCallbacks), + batchCallbacks: map[string]batchCallbacks{ + batchID: { + progress: func(context.Context, BatchState) error { + progressCalls.Add(1) + return nil + }, + }, + }, + } + + delivery := func(jobID string) testInboundJob { + payload, err := json.Marshal(envelope{ + SchemaVersion: schemaVersion, + DispatchID: "dispatch_concurrent_progress", + Kind: "batch_job", + BatchID: batchID, + JobID: jobID, + Job: StoredJob{Type: "batch:item"}, + }) + if err != nil { + t.Fatalf("marshal %s delivery: %v", jobID, err) + } + return testInboundJob{payload: payload} + } + pausedDelivery := delivery("job_paused") + finalDelivery := delivery("job_final") + + pausedResult := make(chan error, 1) + go func() { + pausedResult <- runtime.handleInternalBatchJob(context.Background(), pausedDelivery) + }() + <-store.committed + if err := runtime.handleInternalBatchJob(context.Background(), finalDelivery); err != nil { + t.Fatalf("final job: %v", err) + } + close(store.release) + if err := <-pausedResult; err != nil { + t.Fatalf("paused job: %v", err) + } + if got := progressCalls.Load(); got != 2 { + t.Fatalf("progress calls = %d, want one for each processed job", got) + } +} + +// TestAllowFailuresBatchCompletesIndependentOfFailureOrder verifies aggregate outcome does not depend on the final physical job. +func TestAllowFailuresBatchCompletesIndependentOfFailureOrder(t *testing.T) { + for _, failureFirst := range []bool{true, false} { + name := "failure_last" + if failureFirst { + name = "failure_first" + } + t.Run(name, func(t *testing.T) { + queueRuntime := newSyncTestRuntime() + var events []Event + busRuntime, err := New(queueRuntime, WithObserver(ObserverFunc(func(_ context.Context, event Event) { + events = append(events, event) + }))) + if err != nil { + t.Fatalf("new bus: %v", err) + } + runtime := busRuntime.(*runtime) + if err := runtime.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + t.Cleanup(func() { _ = runtime.Shutdown(context.Background()) }) + + failureErr := errors.New("allowed job failure") + var handled int + runtime.Register("batch:success", func(context.Context, Context) error { + handled++ + return nil + }) + runtime.Register("batch:failure", func(context.Context, Context) error { + handled++ + return failureErr + }) + jobs := []Job{NewJob("batch:success", nil), NewJob("batch:failure", nil)} + if failureFirst { + jobs[0], jobs[1] = jobs[1], jobs[0] + } + + var catchCount int + var thenCount int + var finallyCount int + batchID, dispatchErr := runtime.Batch(jobs...). + AllowFailures(). + Catch(func(_ context.Context, _ BatchState, callbackErr error) error { + if callbackErr == nil || callbackErr.Error() != failureErr.Error() { + t.Fatalf("catch error = %v, want %v", callbackErr, failureErr) + } + catchCount++ + return nil + }). + Then(func(context.Context, BatchState) error { + thenCount++ + return nil + }). + Finally(func(context.Context, BatchState) error { + finallyCount++ + return nil + }). + Dispatch(context.Background()) + if !errors.Is(dispatchErr, failureErr) { + t.Fatalf("dispatch error = %v, want %v", dispatchErr, failureErr) + } + state, err := runtime.FindBatch(context.Background(), batchID) + if err != nil { + t.Fatalf("find batch: %v", err) + } + if handled != 2 || state.Processed != 2 || state.Pending != 0 || state.Failed != 1 || !state.Completed || state.Cancelled { + t.Fatalf("handled/state = %d/%+v, want two processed and completed with one allowed failure", handled, state) + } + if catchCount != 1 || thenCount != 1 || finallyCount != 1 { + t.Fatalf("catch/then/finally count = %d/%d/%d, want 1/1/1", catchCount, thenCount, finallyCount) + } + var progressed, completed, failed int + for _, event := range events { + switch event.Kind { + case EventBatchProgressed: + progressed++ + case EventBatchCompleted: + completed++ + case EventBatchFailed: + failed++ + } + } + if progressed != 2 || completed != 1 || failed != 0 { + t.Fatalf("batch progressed/completed/failed events = %d/%d/%d, want 2/1/0", progressed, completed, failed) + } + runtime.mu.RLock() + callbackCount := len(runtime.batchCallbacks) + runtime.mu.RUnlock() + if callbackCount != 0 { + t.Fatalf("expected batch callbacks cleaned, got len=%d", callbackCount) + } + }) + } +} + +// TestCallbackStateValidationPreservesLegitimateInvocation verifies premature jobs cannot consume callback markers. +func TestCallbackStateValidationPreservesLegitimateInvocation(t *testing.T) { + tests := []struct { + name string + configure func(*runtime, *int) + invalid func(*runtime) error + valid func(*runtime) error + }{ + { + name: "chain catch", + configure: func(runtime *runtime, calls *int) { + runtime.chainCallbacks["chain_state"] = chainCallbacks{catch: func(context.Context, ChainState, error) error { *calls++; return nil }} + }, + invalid: func(runtime *runtime) error { + return runtime.invokeChainCatch(context.Background(), ChainState{ChainID: "chain_state"}, context.Canceled) + }, + valid: func(runtime *runtime) error { + return runtime.invokeChainCatch(context.Background(), ChainState{ChainID: "chain_state", Failed: true}, context.Canceled) + }, + }, + { + name: "chain catch completed precedence", + configure: func(runtime *runtime, calls *int) { + runtime.chainCallbacks["chain_state"] = chainCallbacks{catch: func(context.Context, ChainState, error) error { *calls++; return nil }} + }, + invalid: func(runtime *runtime) error { + return runtime.invokeChainCatch(context.Background(), ChainState{ChainID: "chain_state", Completed: true, Failed: true}, context.Canceled) + }, + valid: func(runtime *runtime) error { + return runtime.invokeChainCatch(context.Background(), ChainState{ChainID: "chain_state", Failed: true}, context.Canceled) + }, + }, + { + name: "chain finally", + configure: func(runtime *runtime, calls *int) { + runtime.chainCallbacks["chain_state"] = chainCallbacks{finally: func(context.Context, ChainState) error { *calls++; return nil }} + }, + invalid: func(runtime *runtime) error { + return runtime.invokeChainFinally(context.Background(), ChainState{ChainID: "chain_state"}) + }, + valid: func(runtime *runtime) error { + return runtime.invokeChainFinally(context.Background(), ChainState{ChainID: "chain_state", Completed: true}) + }, + }, + { + name: "batch catch", + configure: func(runtime *runtime, calls *int) { + runtime.batchCallbacks["batch_state"] = batchCallbacks{catch: func(context.Context, BatchState, error) error { *calls++; return nil }} + }, + invalid: func(runtime *runtime) error { + return runtime.invokeBatchCatch(context.Background(), BatchState{BatchID: "batch_state"}, context.Canceled) + }, + valid: func(runtime *runtime) error { + return runtime.invokeBatchCatch(context.Background(), BatchState{BatchID: "batch_state", Failed: 1}, context.Canceled) + }, + }, + { + name: "batch then incomplete", + configure: func(runtime *runtime, calls *int) { + runtime.batchCallbacks["batch_state"] = batchCallbacks{then: func(context.Context, BatchState) error { *calls++; return nil }} + }, + invalid: func(runtime *runtime) error { + return runtime.invokeBatchThen(context.Background(), BatchState{BatchID: "batch_state"}) + }, + valid: func(runtime *runtime) error { + return runtime.invokeBatchThen(context.Background(), BatchState{BatchID: "batch_state", Completed: true}) + }, + }, + { + name: "batch then cancelled", + configure: func(runtime *runtime, calls *int) { + runtime.batchCallbacks["batch_state"] = batchCallbacks{then: func(context.Context, BatchState) error { *calls++; return nil }} + }, + invalid: func(runtime *runtime) error { + return runtime.invokeBatchThen(context.Background(), BatchState{BatchID: "batch_state", Completed: true, Cancelled: true}) + }, + valid: func(runtime *runtime) error { + return runtime.invokeBatchThen(context.Background(), BatchState{BatchID: "batch_state", Completed: true}) + }, + }, + { + name: "batch finally", + configure: func(runtime *runtime, calls *int) { + runtime.batchCallbacks["batch_state"] = batchCallbacks{finally: func(context.Context, BatchState) error { *calls++; return nil }} + }, + invalid: func(runtime *runtime) error { + return runtime.invokeBatchFinally(context.Background(), BatchState{BatchID: "batch_state"}) + }, + valid: func(runtime *runtime) error { + return runtime.invokeBatchFinally(context.Background(), BatchState{BatchID: "batch_state", Completed: true}) + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + runtime := &runtime{ + store: NewMemoryStore(), + chainCallbacks: make(map[string]chainCallbacks), + batchCallbacks: make(map[string]batchCallbacks), + } + var calls int + test.configure(runtime, &calls) + if err := test.invalid(runtime); !errors.Is(err, errCallbackNotReady) { + t.Fatalf("invalid state error = %v, want errCallbackNotReady", err) + } + if calls != 0 { + t.Fatalf("premature callback calls = %d, want 0", calls) + } + if err := test.valid(runtime); err != nil { + t.Fatalf("valid callback: %v", err) + } + if calls != 1 { + t.Fatalf("legitimate callback calls = %d, want 1", calls) + } + }) + } +} + +// TestBatchProgressPanicDoesNotBlockTerminalCallbacks verifies ephemeral progress cannot unwind committed batch completion. +func TestBatchProgressPanicDoesNotBlockTerminalCallbacks(t *testing.T) { + queueRuntime := newSyncTestRuntime() + busRuntime, err := New(queueRuntime) + if err != nil { + t.Fatalf("new bus: %v", err) + } + runtime := busRuntime.(*runtime) + runtime.Register("batch:progress-panic", func(context.Context, Context) error { return nil }) + var thenCalls int + var finallyCalls int + _, err = runtime.Batch(NewJob("batch:progress-panic", nil)). + Progress(func(context.Context, BatchState) error { panic("progress panic") }). + Then(func(context.Context, BatchState) error { thenCalls++; return nil }). + Finally(func(context.Context, BatchState) error { finallyCalls++; return nil }). + Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch batch: %v", err) + } + if thenCalls != 1 || finallyCalls != 1 { + t.Fatalf("then/finally calls = %d/%d, want 1/1", thenCalls, finallyCalls) + } +} diff --git a/internal/workflow/store.go b/internal/workflow/store.go new file mode 100644 index 0000000..d7066c4 --- /dev/null +++ b/internal/workflow/store.go @@ -0,0 +1,410 @@ +package workflow + +import ( + "context" + "errors" + "fmt" + "time" +) + +// ErrNotFound reports that workflow state is absent from a store. +var ErrNotFound = errors.New("bus record not found") + +// errUnsupportedTransitionReceipt keeps mixed-version workers from treating +// unreadable provenance as either a missing receipt or permission to replay. +var errUnsupportedTransitionReceipt = errors.New("unsupported workflow transition receipt") + +// ChainNode binds a stable node identifier to its serialized job. +type ChainNode struct { + NodeID string + Job StoredJob +} + +// ChainRecord contains the immutable data required to create a chain. +type ChainRecord struct { + ChainID string + DispatchID string + Queue string + Nodes []ChainNode + CreatedAt time.Time +} + +// ChainState is the persisted execution view of a chain. +type ChainState struct { + ChainID string + DispatchID string + Queue string + Nodes []ChainNode + NextIndex int + Completed bool + Failed bool + Failure string + CreatedAt time.Time + UpdatedAt time.Time +} + +// BatchRecord contains the immutable data required to create a batch. +type BatchRecord struct { + BatchID string + DispatchID string + Name string + Queue string + AllowFailed bool + Jobs []BatchJob + CreatedAt time.Time +} + +// BatchJob binds a stable member identifier to its serialized job. +type BatchJob struct { + JobID string + Job StoredJob +} + +// BatchJobOutcome identifies the durable result that first settled one member. +type BatchJobOutcome string + +const ( + // BatchJobSucceeded records successful member settlement. + BatchJobSucceeded BatchJobOutcome = "succeeded" + // BatchJobFailed records failed member settlement. + BatchJobFailed BatchJobOutcome = "failed" +) + +// BatchState is the persisted aggregate execution view of a batch. +type BatchState struct { + BatchID string + DispatchID string + Name string + Queue string + AllowFailed bool + Total int + Pending int + Processed int + Failed int + Cancelled bool + Completed bool + CreatedAt time.Time + UpdatedAt time.Time +} + +// Store defines the compatibility state transitions required by chain, batch, +// and callback orchestration; built-ins also implement outcomeStore. +type Store interface { + // CreateChain persists a newly accepted chain. + CreateChain(ctx context.Context, rec ChainRecord) error + // AdvanceChain atomically claims completedNode and returns the current successor. + // Repeating the same (chainID, completedNode) claim must not advance again. + // When done is true, GetChain must immediately expose Completed or Failed state. + AdvanceChain(ctx context.Context, chainID string, completedNode string) (next *ChainNode, done bool, err error) + // FailChain commits terminal failure without replacing completed state. + FailChain(ctx context.Context, chainID string, cause error) error + // GetChain returns current chain state. + GetChain(ctx context.Context, chainID string) (ChainState, error) + + // CreateBatch persists a newly accepted batch. + CreateBatch(ctx context.Context, rec BatchRecord) error + // MarkBatchJobStarted records that one batch member began execution. + MarkBatchJobStarted(ctx context.Context, batchID, jobID string) error + // MarkBatchJobSucceeded commits the first outcome for (batchID, jobID). + // Duplicate outcomes must return current state without changing counters. + MarkBatchJobSucceeded(ctx context.Context, batchID, jobID string) (BatchState, bool, error) + // MarkBatchJobFailed commits the first outcome for (batchID, jobID). + // Duplicate outcomes must return current state without changing counters. + MarkBatchJobFailed(ctx context.Context, batchID, jobID string, cause error) (BatchState, bool, error) + // CancelBatch commits aggregate batch cancellation. + CancelBatch(ctx context.Context, batchID string) error + // GetBatch returns current batch state. + GetBatch(ctx context.Context, batchID string) (BatchState, error) + + // MarkCallbackInvoked atomically claims one callback idempotency key. + MarkCallbackInvoked(ctx context.Context, key string) (bool, error) + // Prune removes terminal workflow state older than before. + Prune(ctx context.Context, before time.Time) error +} + +// outcomeStore exposes first-writer outcome arbitration without expanding the +// compatibility-critical Store interface implemented by existing consumers. +type outcomeStore interface { + FailChainNode(ctx context.Context, chainID, nodeID string, cause error) (ChainState, bool, error) + // SettleBatchJob arbitrates the durable category while the established batch + // model keeps failure detail local to the physical delivery that reports it. + SettleBatchJob(ctx context.Context, batchID, jobID string, outcome BatchJobOutcome, cause error) (BatchState, bool, error) +} + +type transitionClaim struct { + deliveryID string + attempt int + dispatchID string + jobID string + jobFingerprint string +} + +// valid reports whether a settlement generation supplied every identity field +// required for durable transition provenance. +func (c transitionClaim) valid() bool { + return c.deliveryID != "" && c.attempt >= 0 && c.dispatchID != "" && c.jobID != "" && c.jobFingerprint != "" +} + +type transitionReceipt struct { + version int + eventSchemaVersion int + workflowKind string + workflowID string + workflowDispatchID string + workflowCreatedAt time.Time + memberID string + outcome BatchJobOutcome + owner transitionClaim + aggregateCompleted bool + aggregateCancelled bool + createdAt time.Time +} + +type transitionReceiptKey struct { + workflowKind string + workflowID string + memberID string +} + +const ( + transitionReceiptVersion = 1 + chainTransitionKind = "chain" + batchTransitionKind = "batch" +) + +// supported reports whether this runtime can interpret both durable identity +// and the event contract reconstructed from it. +func (r transitionReceipt) supported() bool { + return r.version == transitionReceiptVersion && r.eventSchemaVersion == eventSchemaVersion +} + +// validateTransitionReceiptSupport fails closed when a worker cannot interpret +// either the durable receipt identity or the observer facts reconstructed from it. +func validateTransitionReceiptSupport(receipt transitionReceipt) error { + if receipt.supported() { + return nil + } + return fmt.Errorf("%w: receipt version %d, event schema %d", errUnsupportedTransitionReceipt, receipt.version, receipt.eventSchemaVersion) +} + +// chainAdvanceResult distinguishes the logical success owner from the physical +// delivery that claimed it so recovery never repeats continuation effects. +type chainAdvanceResult struct { + state ChainState + next *ChainNode + done bool + successOwned bool + claimedNow bool + receipt transitionReceipt + receiptKnown bool +} + +// chainAdvanceStore exposes built-in atomic transition ownership without +// expanding the compatibility-critical Store interface. +type chainAdvanceStore interface { + advanceChainOutcome(ctx context.Context, chainID, nodeID string, claim transitionClaim) (chainAdvanceResult, error) +} + +// chainFailureResult distinguishes durable failure ownership from the physical +// delivery that atomically persisted its recovery receipt. +type chainFailureResult struct { + state ChainState + owned bool + claimedNow bool + receipt transitionReceipt + receiptKnown bool +} + +// chainFailureStore exposes built-in atomic failure provenance without +// expanding either Store or the established outcomeStore capability. +type chainFailureStore interface { + failChainOutcome(ctx context.Context, chainID, nodeID string, cause error, claim transitionClaim) (chainFailureResult, error) +} + +// batchSettlementResult separates first-writer category ownership from the +// delivery that changed aggregate counters in this transaction. +type batchSettlementResult struct { + state BatchState + owned bool + claimedNow bool + receipt transitionReceipt + receiptKnown bool +} + +// batchSettlementStore exposes built-in member transition ownership without +// requiring established custom stores to implement another public method. +type batchSettlementStore interface { + settleBatchOutcome(ctx context.Context, batchID, jobID string, outcome BatchJobOutcome, cause error, claim transitionClaim) (batchSettlementResult, error) +} + +// transitionReceiptStore exposes durable writer identity only to the workflow +// engine; established public stores remain source-compatible. +type transitionReceiptStore interface { + chainTransitionReceipt(ctx context.Context, chainID, nodeID string) (transitionReceipt, bool, error) + batchTransitionReceipt(ctx context.Context, batchID, jobID string) (transitionReceipt, bool, error) +} + +// chainNodePosition resolves persisted order so stale and future deliveries +// cannot mutate the aggregate merely because they carry a valid chain ID. +func chainNodePosition(nodes []ChainNode, nodeID string) (int, bool) { + for index := range nodes { + if nodes[index].NodeID == nodeID { + return index, true + } + } + return 0, false +} + +// validateChainState rejects representations that cannot prove which nodes +// committed before recovery reconstructs any externally visible fact. +func validateChainState(state ChainState) error { + if err := validateChainRecord(ChainRecord{ChainID: state.ChainID, Nodes: state.Nodes}); err != nil { + return err + } + if state.NextIndex < 0 || state.NextIndex > len(state.Nodes) { + return fmt.Errorf("chain %q has invalid next index %d", state.ChainID, state.NextIndex) + } + if state.Completed != (state.NextIndex == len(state.Nodes)) { + return fmt.Errorf("chain %q completion does not match next index %d", state.ChainID, state.NextIndex) + } + return nil +} + +// chainNodeSuccessDisposition reports whether the persisted ordering proves +// that node success won while rejecting future or internally inconsistent deliveries. +func chainNodeSuccessDisposition(state ChainState, nodeID string) (bool, error) { + if err := validateChainState(state); err != nil { + return false, err + } + index, ok := chainNodePosition(state.Nodes, nodeID) + if !ok { + return false, fmt.Errorf("chain %q does not contain node %q", state.ChainID, nodeID) + } + if index > state.NextIndex { + return false, fmt.Errorf("chain %q received node %q before node %q", state.ChainID, nodeID, state.Nodes[state.NextIndex].NodeID) + } + return index < state.NextIndex, nil +} + +// validateChainRecord rejects ambiguous order because duplicate or empty node +// IDs make physical redelivery indistinguishable from a different chain step. +func validateChainRecord(record ChainRecord) error { + if record.ChainID == "" { + return errors.New("chain id is required") + } + if len(record.Nodes) == 0 { + return errors.New("chain requires at least one node") + } + seen := make(map[string]struct{}, len(record.Nodes)) + for _, node := range record.Nodes { + if node.NodeID == "" { + return errors.New("chain node id is required") + } + if _, exists := seen[node.NodeID]; exists { + return fmt.Errorf("chain contains duplicate node id %q", node.NodeID) + } + seen[node.NodeID] = struct{}{} + } + return nil +} + +// validateBatchRecord rejects ambiguous member identity because first-writer +// outcome ownership is keyed by the stable (batchID, jobID) pair. +func validateBatchRecord(record BatchRecord) error { + if record.BatchID == "" { + return errors.New("batch id is required") + } + if len(record.Jobs) == 0 { + return errors.New("batch requires at least one job") + } + seen := make(map[string]struct{}, len(record.Jobs)) + for _, job := range record.Jobs { + if job.JobID == "" { + return errors.New("batch job id is required") + } + if _, exists := seen[job.JobID]; exists { + return fmt.Errorf("batch contains duplicate job id %q", job.JobID) + } + seen[job.JobID] = struct{}{} + } + return nil +} + +// cloneChainNodes isolates immutable order and payload bytes from callers that +// retain either a creation record or a state returned by the memory store. +func cloneChainNodes(nodes []ChainNode) []ChainNode { + if nodes == nil { + return nil + } + cloned := make([]ChainNode, len(nodes)) + for index := range nodes { + cloned[index] = cloneChainNode(nodes[index]) + } + return cloned +} + +// cloneChainNode copies the only reference-bearing field in a persisted node. +func cloneChainNode(node ChainNode) ChainNode { + cloned := node + cloned.Job.Payload = append([]byte(nil), node.Job.Payload...) + return cloned +} + +// chainNodeAdvanceDisposition separates immutable order validation from the +// store-specific compare-and-swap that owns the current node's success. +func chainNodeAdvanceDisposition(state ChainState, nodeID string) (next *ChainNode, done, claimable bool, err error) { + index, ok := chainNodePosition(state.Nodes, nodeID) + if !ok { + return nil, false, false, fmt.Errorf("chain %q does not contain node %q", state.ChainID, nodeID) + } + if state.Completed { + return nil, true, false, nil + } + if state.NextIndex < 0 || state.NextIndex >= len(state.Nodes) { + return nil, false, false, fmt.Errorf("chain %q has invalid next index %d", state.ChainID, state.NextIndex) + } + if state.Failed { + if index > state.NextIndex { + return nil, false, false, fmt.Errorf("chain %q received node %q after failure at node %q", state.ChainID, nodeID, state.Nodes[state.NextIndex].NodeID) + } + return nil, true, false, nil + } + if index > state.NextIndex { + return nil, false, false, fmt.Errorf("chain %q received node %q before node %q", state.ChainID, nodeID, state.Nodes[state.NextIndex].NodeID) + } + if index == state.NextIndex { + return nil, false, true, nil + } + node := state.Nodes[state.NextIndex] + return &node, false, false, nil +} + +// chainNodeFailureDisposition classifies whether failure already owns a node, +// can still claim it, or lost to an earlier successful transition. +func chainNodeFailureDisposition(state ChainState, nodeID string) (owned, claimable bool, err error) { + index, ok := chainNodePosition(state.Nodes, nodeID) + if !ok { + return false, false, fmt.Errorf("chain %q does not contain node %q", state.ChainID, nodeID) + } + // Legacy SQL rows can contain both flags only when completion happened + // first, because the old advancement path never completed a failed chain. + if state.Completed { + return false, false, nil + } + if state.NextIndex < 0 || state.NextIndex >= len(state.Nodes) { + return false, false, fmt.Errorf("chain %q has invalid next index %d", state.ChainID, state.NextIndex) + } + if state.Failed { + if index > state.NextIndex { + return false, false, fmt.Errorf("chain %q received node %q after failure at node %q", state.ChainID, nodeID, state.Nodes[state.NextIndex].NodeID) + } + return index == state.NextIndex, false, nil + } + if index < state.NextIndex { + return false, false, nil + } + if index > state.NextIndex { + return false, false, fmt.Errorf("chain %q received node %q before node %q", state.ChainID, nodeID, state.Nodes[state.NextIndex].NodeID) + } + return false, true, nil +} diff --git a/internal/workflow/store_contract_test.go b/internal/workflow/store_contract_test.go new file mode 100644 index 0000000..9822d42 --- /dev/null +++ b/internal/workflow/store_contract_test.go @@ -0,0 +1,1800 @@ +package workflow + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + _ "modernc.org/sqlite" +) + +func testStoreFactories(t *testing.T) map[string]func(t *testing.T) Store { + t.Helper() + return map[string]func(t *testing.T) Store{ + "memory": func(t *testing.T) Store { + t.Helper() + return NewMemoryStore() + }, + "sql_sqlite": func(t *testing.T) Store { + t.Helper() + dsn := filepath.Join(t.TempDir(), "store-contract.db") + "?_pragma=busy_timeout%3d5000" + store, err := NewSQLStore(SQLStoreConfig{ + DriverName: "sqlite", + DSN: dsn, + }) + if err != nil { + t.Fatalf("new sql store: %v", err) + } + t.Cleanup(func() { _ = store.(*sqlStore).db.Close() }) + return store + }, + } +} + +// waitStoreContractOperations bounds lock-sensitive probes so a regression is +// reported by the focused contract rather than the package-wide test timeout. +func waitStoreContractOperations(t *testing.T, wg *sync.WaitGroup) { + t.Helper() + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for concurrent store operations") + } +} + +// requireOutcomeStore keeps the compatibility Store contract unchanged while +// asserting that every built-in implementation provides stronger arbitration. +func requireOutcomeStore(t *testing.T, store Store) outcomeStore { + t.Helper() + outcomes, ok := store.(outcomeStore) + if !ok { + t.Fatalf("built-in store %T does not implement outcomeStore", store) + } + return outcomes +} + +// requireChainAdvanceStore verifies every built-in can distinguish transition +// ownership without changing the public compatibility contract. +func requireChainAdvanceStore(t *testing.T, store Store) chainAdvanceStore { + t.Helper() + atomic, ok := store.(chainAdvanceStore) + if !ok { + t.Fatalf("built-in store %T does not implement chainAdvanceStore", store) + } + return atomic +} + +// requireChainFailureStore verifies every built-in can persist terminal +// failure provenance without changing the public compatibility contract. +func requireChainFailureStore(t *testing.T, store Store) chainFailureStore { + t.Helper() + atomic, ok := store.(chainFailureStore) + if !ok { + t.Fatalf("built-in store %T does not implement chainFailureStore", store) + } + return atomic +} + +// requireBatchSettlementStore verifies every built-in exposes its exact +// member counter claim for recovery decisions. +func requireBatchSettlementStore(t *testing.T, store Store) batchSettlementStore { + t.Helper() + atomic, ok := store.(batchSettlementStore) + if !ok { + t.Fatalf("built-in store %T does not implement batchSettlementStore", store) + } + return atomic +} + +// requireTransitionReceiptStore verifies built-ins expose durable ownership +// without adding receipt methods to the compatibility-critical public store. +func requireTransitionReceiptStore(t *testing.T, store Store) transitionReceiptStore { + t.Helper() + receipts, ok := store.(transitionReceiptStore) + if !ok { + t.Fatalf("built-in store %T does not implement transitionReceiptStore", store) + } + return receipts +} + +// TestStoreContract_TransitionOwnership distinguishes first claims from +// same-category and contradictory physical replays across every built-in store. +func TestStoreContract_TransitionOwnership(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + ctx := context.Background() + store := factory(t) + chainStore := requireChainAdvanceStore(t, store) + chainClaim := transitionClaim{deliveryID: "generation-chain", attempt: 0, dispatchID: "dispatch-chain", jobID: "job-chain", jobFingerprint: "fingerprint-chain"} + if err := store.CreateChain(ctx, ChainRecord{ + ChainID: "chain-transition-ownership", + DispatchID: chainClaim.dispatchID, + Nodes: []ChainNode{ + {NodeID: "node-first"}, + {NodeID: "node-final"}, + }, + }); err != nil { + t.Fatalf("create chain: %v", err) + } + first, err := chainStore.advanceChainOutcome(ctx, "chain-transition-ownership", "node-first", chainClaim) + if err != nil || !first.claimedNow || !first.successOwned || !first.receiptKnown || first.receipt.owner != chainClaim || first.next == nil || first.next.NodeID != "node-final" { + t.Fatalf("first chain claim = %+v err:%v", first, err) + } + replay, err := chainStore.advanceChainOutcome(ctx, "chain-transition-ownership", "node-first", transitionClaim{deliveryID: "generation-chain-replay", attempt: 0, dispatchID: chainClaim.dispatchID, jobID: chainClaim.jobID, jobFingerprint: chainClaim.jobFingerprint}) + if err != nil || replay.claimedNow || !replay.successOwned || !replay.receiptKnown || replay.receipt.owner != chainClaim || replay.next == nil || replay.next.NodeID != "node-final" { + t.Fatalf("chain replay = %+v err:%v", replay, err) + } + + failureClaim := transitionClaim{deliveryID: "generation-chain-failure", attempt: 1, dispatchID: "dispatch-chain-failure", jobID: "job-chain-failure", jobFingerprint: "fingerprint-chain-failure"} + if err := store.CreateChain(ctx, ChainRecord{ + ChainID: "chain-failure-transition-ownership", + DispatchID: failureClaim.dispatchID, + Nodes: []ChainNode{{NodeID: "node-failure"}}, + }); err != nil { + t.Fatalf("create failure chain: %v", err) + } + failureStore := requireChainFailureStore(t, store) + failed, err := failureStore.failChainOutcome(ctx, "chain-failure-transition-ownership", "node-failure", errors.New("committed failure"), failureClaim) + if err != nil || !failed.claimedNow || !failed.owned || !failed.receiptKnown || failed.receipt.owner != failureClaim || failed.receipt.outcome != BatchJobFailed || failed.receipt.aggregateCompleted || failed.receipt.aggregateCancelled || !failed.state.Failed || failed.state.Completed { + t.Fatalf("first chain failure claim = %+v err:%v", failed, err) + } + failedReplay, err := failureStore.failChainOutcome(ctx, "chain-failure-transition-ownership", "node-failure", errors.New("replacement failure"), transitionClaim{deliveryID: "generation-chain-failure-replay", attempt: 1, dispatchID: failureClaim.dispatchID, jobID: failureClaim.jobID, jobFingerprint: failureClaim.jobFingerprint}) + if err != nil || failedReplay.claimedNow || !failedReplay.owned || !failedReplay.receiptKnown || failedReplay.receipt.owner != failureClaim || failedReplay.state.Failure != "committed failure" { + t.Fatalf("chain failure replay = %+v err:%v", failedReplay, err) + } + + batchStore := requireBatchSettlementStore(t, store) + batchClaim := transitionClaim{deliveryID: "generation-batch", attempt: 0, dispatchID: "dispatch-batch", jobID: "job-first", jobFingerprint: "fingerprint-batch"} + if err := store.CreateBatch(ctx, BatchRecord{ + BatchID: "batch-transition-ownership", + DispatchID: batchClaim.dispatchID, + Jobs: []BatchJob{{JobID: "job-first"}, {JobID: "job-final"}}, + }); err != nil { + t.Fatalf("create batch: %v", err) + } + settled, err := batchStore.settleBatchOutcome(ctx, "batch-transition-ownership", "job-first", BatchJobSucceeded, nil, batchClaim) + if err != nil || !settled.claimedNow || !settled.owned || !settled.receiptKnown || settled.receipt.owner != batchClaim || settled.state.Processed != 1 { + t.Fatalf("first batch claim = %+v err:%v", settled, err) + } + replayed, err := batchStore.settleBatchOutcome(ctx, "batch-transition-ownership", "job-first", BatchJobSucceeded, nil, transitionClaim{deliveryID: "generation-batch-replay", attempt: 0, dispatchID: batchClaim.dispatchID, jobID: batchClaim.jobID, jobFingerprint: batchClaim.jobFingerprint}) + if err != nil || replayed.claimedNow || !replayed.owned || !replayed.receiptKnown || replayed.receipt.owner != batchClaim || replayed.state.Processed != 1 { + t.Fatalf("same batch replay = %+v err:%v", replayed, err) + } + contradictory, err := batchStore.settleBatchOutcome(ctx, "batch-transition-ownership", "job-first", BatchJobFailed, errors.New("contradictory"), transitionClaim{}) + if err != nil || contradictory.claimedNow || contradictory.owned || contradictory.state.Processed != 1 { + t.Fatalf("contradictory batch replay = %+v err:%v", contradictory, err) + } + }) + } +} + +// TestStoreContract_TransitionClaimDispatchMismatch proves built-ins reject a +// complete or transport-only claim for another workflow incarnation without mutation. +func TestStoreContract_TransitionClaimDispatchMismatch(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + ctx := context.Background() + store := factory(t) + const ( + chainID = "chain-claim-dispatch-mismatch" + nodeID = "node-claim-dispatch-mismatch" + batchID = "batch-claim-dispatch-mismatch" + jobID = "job-claim-dispatch-mismatch" + ) + if err := store.CreateChain(ctx, ChainRecord{ChainID: chainID, DispatchID: "dispatch-current-chain", Nodes: []ChainNode{{NodeID: nodeID}}}); err != nil { + t.Fatalf("create chain: %v", err) + } + mismatches := []transitionClaim{ + {deliveryID: "generation-mismatch", attempt: 0, dispatchID: "dispatch-other", jobID: "job-mismatch", jobFingerprint: "fingerprint-mismatch"}, + {dispatchID: "dispatch-other", jobID: "job-mismatch", jobFingerprint: "fingerprint-mismatch"}, + } + for index, mismatch := range mismatches { + if _, err := requireChainAdvanceStore(t, store).advanceChainOutcome(ctx, chainID, nodeID, mismatch); err == nil || !strings.Contains(err.Error(), "dispatch mismatch") { + t.Fatalf("advance mismatch %d error = %v", index, err) + } + if _, err := requireChainFailureStore(t, store).failChainOutcome(ctx, chainID, nodeID, errors.New("must not commit"), mismatch); err == nil || !strings.Contains(err.Error(), "dispatch mismatch") { + t.Fatalf("failure mismatch %d error = %v", index, err) + } + } + chain, err := store.GetChain(ctx, chainID) + if err != nil || chain.NextIndex != 0 || chain.Completed || chain.Failed { + t.Fatalf("chain after mismatches = %+v err:%v", chain, err) + } + if receipt, known, err := requireTransitionReceiptStore(t, store).chainTransitionReceipt(ctx, chainID, nodeID); err != nil || known { + t.Fatalf("chain mismatch receipt = known:%t receipt:%+v err:%v", known, receipt, err) + } + if _, done, err := store.AdvanceChain(ctx, chainID, nodeID); err != nil || !done { + t.Fatalf("complete current chain = done:%t err:%v", done, err) + } + staleSuccess, err := requireChainAdvanceStore(t, store).advanceChainOutcome(ctx, chainID, nodeID, mismatches[1]) + if err != nil || staleSuccess.successOwned || staleSuccess.claimedNow || staleSuccess.receiptKnown || staleSuccess.next != nil || staleSuccess.done { + t.Fatalf("terminal stale chain success = %+v err:%v, want pure non-owner no-op", staleSuccess, err) + } + if err := store.CreateChain(ctx, ChainRecord{ChainID: "chain-failure-terminal-dispatch-mismatch", DispatchID: "dispatch-current-chain", Nodes: []ChainNode{{NodeID: nodeID}}}); err != nil { + t.Fatalf("create terminal failure chain: %v", err) + } + if _, owned, err := requireOutcomeStore(t, store).FailChainNode(ctx, "chain-failure-terminal-dispatch-mismatch", nodeID, errors.New("current failure")); err != nil || !owned { + t.Fatalf("fail current chain = owned:%t err:%v", owned, err) + } + staleFailure, err := requireChainFailureStore(t, store).failChainOutcome(ctx, "chain-failure-terminal-dispatch-mismatch", nodeID, errors.New("stale failure"), mismatches[1]) + if err != nil || staleFailure.owned || staleFailure.claimedNow || staleFailure.receiptKnown { + t.Fatalf("terminal stale chain failure = %+v err:%v, want pure non-owner no-op", staleFailure, err) + } + + if err := store.CreateBatch(ctx, BatchRecord{BatchID: batchID, DispatchID: "dispatch-current-batch", Jobs: []BatchJob{{JobID: jobID}}}); err != nil { + t.Fatalf("create batch: %v", err) + } + for index, mismatch := range mismatches { + if _, err := requireBatchSettlementStore(t, store).settleBatchOutcome(ctx, batchID, jobID, BatchJobSucceeded, nil, mismatch); err == nil || !strings.Contains(err.Error(), "dispatch mismatch") { + t.Fatalf("batch mismatch %d error = %v", index, err) + } + } + batch, err := store.GetBatch(ctx, batchID) + if err != nil || batch.Processed != 0 || batch.Pending != 1 || batch.Completed || batch.Cancelled { + t.Fatalf("batch after mismatch = %+v err:%v", batch, err) + } + if receipt, known, err := requireTransitionReceiptStore(t, store).batchTransitionReceipt(ctx, batchID, jobID); err != nil || known { + t.Fatalf("batch mismatch receipt = known:%t receipt:%+v err:%v", known, receipt, err) + } + if state, done, err := store.MarkBatchJobSucceeded(ctx, batchID, jobID); err != nil || !done || !state.Completed { + t.Fatalf("complete current batch = %+v done:%t err:%v", state, done, err) + } + staleBatch, err := requireBatchSettlementStore(t, store).settleBatchOutcome(ctx, batchID, jobID, BatchJobSucceeded, nil, mismatches[1]) + if err != nil || staleBatch.owned || staleBatch.claimedNow || staleBatch.receiptKnown { + t.Fatalf("terminal stale batch settlement = %+v err:%v, want pure non-owner no-op", staleBatch, err) + } + + legacyClaim := transitionClaim{deliveryID: "generation-legacy-dispatch", attempt: 0, dispatchID: "dispatch-transport-only", jobID: "job-legacy-dispatch", jobFingerprint: "fingerprint-legacy-dispatch"} + if err := store.CreateChain(ctx, ChainRecord{ChainID: "chain-legacy-empty-dispatch", Nodes: []ChainNode{{NodeID: "node-legacy-empty-dispatch"}}}); err != nil { + t.Fatalf("create legacy chain: %v", err) + } + legacyChain, err := requireChainAdvanceStore(t, store).advanceChainOutcome(ctx, "chain-legacy-empty-dispatch", "node-legacy-empty-dispatch", legacyClaim) + if err != nil || !legacyChain.claimedNow || !legacyChain.receiptKnown || legacyChain.receipt.workflowDispatchID != "" || legacyChain.receipt.owner.dispatchID != legacyClaim.dispatchID { + t.Fatalf("legacy chain claim = %+v err:%v", legacyChain, err) + } + if err := store.CreateChain(ctx, ChainRecord{ChainID: "chain-failure-legacy-empty-dispatch", Nodes: []ChainNode{{NodeID: "node-failure-legacy-empty-dispatch"}}}); err != nil { + t.Fatalf("create legacy failure chain: %v", err) + } + legacyFailure, err := requireChainFailureStore(t, store).failChainOutcome(ctx, "chain-failure-legacy-empty-dispatch", "node-failure-legacy-empty-dispatch", errors.New("legacy failure"), legacyClaim) + if err != nil || !legacyFailure.claimedNow || !legacyFailure.receiptKnown || legacyFailure.receipt.workflowDispatchID != "" || legacyFailure.receipt.owner.dispatchID != legacyClaim.dispatchID { + t.Fatalf("legacy chain failure claim = %+v err:%v", legacyFailure, err) + } + if err := store.CreateBatch(ctx, BatchRecord{BatchID: "batch-legacy-empty-dispatch", Jobs: []BatchJob{{JobID: "job-legacy-empty-dispatch"}}}); err != nil { + t.Fatalf("create legacy batch: %v", err) + } + legacyBatchClaim := legacyClaim + legacyBatchClaim.jobID = "job-legacy-empty-dispatch" + legacyBatch, err := requireBatchSettlementStore(t, store).settleBatchOutcome(ctx, "batch-legacy-empty-dispatch", "job-legacy-empty-dispatch", BatchJobSucceeded, nil, legacyBatchClaim) + if err != nil || !legacyBatch.claimedNow || !legacyBatch.receiptKnown || legacyBatch.receipt.workflowDispatchID != "" || legacyBatch.receipt.owner.dispatchID != legacyClaim.dispatchID { + t.Fatalf("legacy batch claim = %+v err:%v", legacyBatch, err) + } + }) + } +} + +// TestStoreContract_FailChainPreservesReceiptBackedCause proves the legacy +// terminal method cannot replace the cause bound to an immutable failed receipt. +func TestStoreContract_FailChainPreservesReceiptBackedCause(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + ctx := context.Background() + store := factory(t) + const ( + chainID = "chain-receipt-backed-first-cause" + nodeID = "node-receipt-backed-first-cause" + dispatchID = "dispatch-receipt-backed-first-cause" + ) + if err := store.CreateChain(ctx, ChainRecord{ChainID: chainID, DispatchID: dispatchID, Nodes: []ChainNode{{NodeID: nodeID}}}); err != nil { + t.Fatalf("create chain: %v", err) + } + claim := transitionClaim{deliveryID: "generation-receipt-backed-first-cause", attempt: 2, dispatchID: dispatchID, jobID: "job-receipt-backed-first-cause", jobFingerprint: "fingerprint-receipt-backed-first-cause"} + result, err := requireChainFailureStore(t, store).failChainOutcome(ctx, chainID, nodeID, errors.New("authoritative first cause"), claim) + if err != nil || !result.claimedNow || !result.receiptKnown { + t.Fatalf("commit receipt-backed failure = %+v err:%v", result, err) + } + if err := store.FailChain(ctx, chainID, errors.New("replacement cause")); err != nil { + t.Fatalf("repeat legacy failure: %v", err) + } + state, err := store.GetChain(ctx, chainID) + if err != nil || !state.Failed || state.Completed || state.Failure != "authoritative first cause" { + t.Fatalf("chain after replacement attempt = %+v err:%v", state, err) + } + receipt, known, err := requireTransitionReceiptStore(t, store).chainTransitionReceipt(ctx, chainID, nodeID) + if err != nil || !known || receipt.owner != claim || receipt.outcome != BatchJobFailed { + t.Fatalf("receipt after replacement attempt = known:%t receipt:%+v err:%v", known, receipt, err) + } + }) + } +} + +// TestStoreContract_TransitionReceiptIncarnationMismatchFailsClosed proves a +// persisted row for another parent incarnation is never collapsed into absence. +func TestStoreContract_TransitionReceiptIncarnationMismatchFailsClosed(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + ctx := context.Background() + store := factory(t) + const ( + chainID = "chain-receipt-incarnation-mismatch" + nodeID = "node-receipt-incarnation-mismatch" + batchID = "batch-receipt-incarnation-mismatch" + jobID = "job-receipt-incarnation-mismatch" + ) + chainClaim := transitionClaim{deliveryID: "generation-chain-incarnation", attempt: 0, dispatchID: "dispatch-chain-incarnation", jobID: "job-chain-incarnation", jobFingerprint: "fingerprint-chain-incarnation"} + if err := store.CreateChain(ctx, ChainRecord{ChainID: chainID, DispatchID: chainClaim.dispatchID, Nodes: []ChainNode{{NodeID: nodeID}}}); err != nil { + t.Fatalf("create chain: %v", err) + } + if result, err := requireChainAdvanceStore(t, store).advanceChainOutcome(ctx, chainID, nodeID, chainClaim); err != nil || !result.receiptKnown { + t.Fatalf("commit chain receipt = %+v err:%v", result, err) + } + corruptTransitionReceiptDispatch(t, store, chainTransitionKind, chainID, nodeID) + if receipt, known, err := requireTransitionReceiptStore(t, store).chainTransitionReceipt(ctx, chainID, nodeID); err == nil || known || !strings.Contains(err.Error(), "incarnation") { + t.Fatalf("mismatched chain receipt = known:%t receipt:%+v err:%v", known, receipt, err) + } + + batchClaim := transitionClaim{deliveryID: "generation-batch-incarnation", attempt: 0, dispatchID: "dispatch-batch-incarnation", jobID: jobID, jobFingerprint: "fingerprint-batch-incarnation"} + if err := store.CreateBatch(ctx, BatchRecord{BatchID: batchID, DispatchID: batchClaim.dispatchID, Jobs: []BatchJob{{JobID: jobID}}}); err != nil { + t.Fatalf("create batch: %v", err) + } + if result, err := requireBatchSettlementStore(t, store).settleBatchOutcome(ctx, batchID, jobID, BatchJobSucceeded, nil, batchClaim); err != nil || !result.receiptKnown { + t.Fatalf("commit batch receipt = %+v err:%v", result, err) + } + corruptTransitionReceiptDispatch(t, store, batchTransitionKind, batchID, jobID) + if receipt, known, err := requireTransitionReceiptStore(t, store).batchTransitionReceipt(ctx, batchID, jobID); err == nil || known || !strings.Contains(err.Error(), "incarnation") { + t.Fatalf("mismatched batch receipt = known:%t receipt:%+v err:%v", known, receipt, err) + } + }) + } +} + +// corruptTransitionReceiptDispatch simulates a retained row whose parent +// identity no longer matches without depending on one store's internals in callers. +func corruptTransitionReceiptDispatch(t *testing.T, store Store, kind, workflowID, memberID string) { + t.Helper() + switch concrete := store.(type) { + case *memoryStore: + concrete.mu.Lock() + key := transitionReceiptKey{workflowKind: kind, workflowID: workflowID, memberID: memberID} + receipt := concrete.transitionReceipts[key] + receipt.workflowDispatchID = "dispatch-corrupt-incarnation" + concrete.transitionReceipts[key] = receipt + concrete.mu.Unlock() + case *sqlStore: + if _, err := concrete.db.Exec(`UPDATE bus_workflow_transition_receipts SET workflow_dispatch_id=? WHERE workflow_kind=? AND workflow_id=? AND member_id=?`, "dispatch-corrupt-incarnation", kind, workflowID, memberID); err != nil { + t.Fatalf("corrupt SQL transition receipt: %v", err) + } + default: + t.Fatalf("unsupported built-in store %T", store) + } +} + +// TestStoreContract_ConcurrentTransitionReceiptOwner proves every competing +// generation observes the same immutable owner and aggregate outcome. +func TestStoreContract_ConcurrentTransitionReceiptOwner(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + store := factory(t) + const deliveries = 16 + + if err := store.CreateChain(ctx, ChainRecord{ + ChainID: "chain-concurrent-receipt-owner", + DispatchID: "dispatch-concurrent-chain-receipt", + Nodes: []ChainNode{{NodeID: "node-concurrent-receipt"}}, + }); err != nil { + t.Fatalf("create chain: %v", err) + } + chainStore := requireChainAdvanceStore(t, store) + type chainObservation struct { + result chainAdvanceResult + err error + } + chainStart := make(chan struct{}) + chainResults := make(chan chainObservation, deliveries) + var chainWait sync.WaitGroup + for delivery := range deliveries { + chainWait.Add(1) + go func(delivery int) { + defer chainWait.Done() + <-chainStart + claim := transitionClaim{ + deliveryID: fmt.Sprintf("generation-chain-%02d", delivery), + attempt: delivery, + dispatchID: "dispatch-concurrent-chain-receipt", + jobID: "job-concurrent-chain-receipt", + jobFingerprint: "fingerprint-concurrent-chain-receipt", + } + result, err := chainStore.advanceChainOutcome(ctx, "chain-concurrent-receipt-owner", "node-concurrent-receipt", claim) + chainResults <- chainObservation{result: result, err: err} + }(delivery) + } + close(chainStart) + waitStoreContractOperations(t, &chainWait) + close(chainResults) + chainOwners := make(map[transitionClaim]struct{}) + chainClaims := 0 + for observation := range chainResults { + if observation.err != nil { + t.Fatalf("concurrent chain claim: %v", observation.err) + } + if observation.result.claimedNow { + chainClaims++ + } + if !observation.result.successOwned || !observation.result.receiptKnown { + t.Fatalf("concurrent chain result = %+v, want owned receipt", observation.result) + } + chainOwners[observation.result.receipt.owner] = struct{}{} + } + if chainClaims != 1 || len(chainOwners) != 1 { + t.Fatalf("chain claims/owners = %d/%d, want 1/1", chainClaims, len(chainOwners)) + } + + if err := store.CreateBatch(ctx, BatchRecord{ + BatchID: "batch-concurrent-receipt-owner", + DispatchID: "dispatch-concurrent-batch-receipt", + AllowFailed: true, + Jobs: []BatchJob{{JobID: "job-concurrent-receipt"}}, + }); err != nil { + t.Fatalf("create batch: %v", err) + } + batchStore := requireBatchSettlementStore(t, store) + type batchObservation struct { + result batchSettlementResult + err error + } + batchStart := make(chan struct{}) + batchResults := make(chan batchObservation, deliveries) + var batchWait sync.WaitGroup + for delivery := range deliveries { + batchWait.Add(1) + go func(delivery int) { + defer batchWait.Done() + <-batchStart + outcome := BatchJobSucceeded + if delivery%2 == 0 { + outcome = BatchJobFailed + } + claim := transitionClaim{ + deliveryID: fmt.Sprintf("generation-batch-%02d", delivery), + attempt: delivery, + dispatchID: "dispatch-concurrent-batch-receipt", + jobID: "job-concurrent-receipt", + jobFingerprint: "fingerprint-concurrent-batch-receipt", + } + result, err := batchStore.settleBatchOutcome(ctx, "batch-concurrent-receipt-owner", "job-concurrent-receipt", outcome, errors.New("raced receipt outcome"), claim) + batchResults <- batchObservation{result: result, err: err} + }(delivery) + } + close(batchStart) + waitStoreContractOperations(t, &batchWait) + close(batchResults) + batchOwners := make(map[transitionClaim]struct{}) + batchOutcomes := make(map[BatchJobOutcome]struct{}) + batchClaims := 0 + for observation := range batchResults { + if observation.err != nil { + t.Fatalf("concurrent batch claim: %v", observation.err) + } + if observation.result.claimedNow { + batchClaims++ + } + if !observation.result.receiptKnown { + t.Fatalf("concurrent batch result = %+v, want member receipt", observation.result) + } + batchOwners[observation.result.receipt.owner] = struct{}{} + batchOutcomes[observation.result.receipt.outcome] = struct{}{} + } + if batchClaims != 1 || len(batchOwners) != 1 || len(batchOutcomes) != 1 { + t.Fatalf("batch claims/owners/outcomes = %d/%d/%d, want 1/1/1", batchClaims, len(batchOwners), len(batchOutcomes)) + } + terminalReceipt, known, err := requireTransitionReceiptStore(t, store).batchTransitionReceipt(ctx, "batch-concurrent-receipt-owner", "job-concurrent-receipt") + if err != nil || !known || !terminalReceipt.aggregateCompleted { + t.Fatalf("terminal batch receipt = known:%t receipt:%+v err:%v", known, terminalReceipt, err) + } + }) + } +} + +// TestStoreContract_ConcurrentDistinctBatchReceiptOwner proves concurrent +// member receipts cannot compete for terminal aggregate ownership after their +// parent updates serialize, including fail-fast completion before pending +// members finish. +func TestStoreContract_ConcurrentDistinctBatchReceiptOwner(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + for _, policy := range []struct { + name string + allowFailures bool + }{ + {name: "allow_failures", allowFailures: true}, + {name: "fail_fast", allowFailures: false}, + } { + t.Run(policy.name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + store := factory(t) + settlements := requireBatchSettlementStore(t, store) + receipts := requireTransitionReceiptStore(t, store) + const memberCount = 16 + batchID := "batch-distinct-receipt-" + policy.name + dispatchID := "dispatch-distinct-receipt-" + policy.name + jobs := make([]BatchJob, memberCount) + claims := make(map[string]transitionClaim, memberCount) + outcomes := make(map[string]BatchJobOutcome, memberCount) + for member := range memberCount { + jobID := fmt.Sprintf("job-distinct-receipt-%02d", member) + jobs[member] = BatchJob{JobID: jobID} + claims[jobID] = transitionClaim{ + deliveryID: fmt.Sprintf("generation-distinct-receipt-%02d", member), + attempt: member, + dispatchID: dispatchID, + jobID: "delivery-" + jobID, + jobFingerprint: "fingerprint-" + jobID, + } + outcomes[jobID] = BatchJobSucceeded + if member%2 == 1 { + outcomes[jobID] = BatchJobFailed + } + } + if err := store.CreateBatch(ctx, BatchRecord{ + BatchID: batchID, + DispatchID: dispatchID, + AllowFailed: policy.allowFailures, + Jobs: jobs, + }); err != nil { + t.Fatalf("create batch: %v", err) + } + + type settlementObservation struct { + jobID string + result batchSettlementResult + err error + } + start := make(chan struct{}) + observations := make(chan settlementObservation, memberCount) + var wg sync.WaitGroup + for _, job := range jobs { + wg.Add(1) + go func(jobID string) { + defer wg.Done() + <-start + outcome := outcomes[jobID] + result, err := settlements.settleBatchOutcome(ctx, batchID, jobID, outcome, errors.New("concurrent member failure"), claims[jobID]) + observations <- settlementObservation{jobID: jobID, result: result, err: err} + }(job.JobID) + } + close(start) + waitStoreContractOperations(t, &wg) + close(observations) + + aggregateOwners := 0 + for observation := range observations { + if observation.err != nil { + t.Fatalf("settle member %q: %v", observation.jobID, observation.err) + } + wantClaim := claims[observation.jobID] + wantOutcome := outcomes[observation.jobID] + if !observation.result.claimedNow || !observation.result.owned || !observation.result.receiptKnown || observation.result.receipt.owner != wantClaim || observation.result.receipt.outcome != wantOutcome { + t.Fatalf("member %q result = %+v, want its exact receipt owner and outcome", observation.jobID, observation.result) + } + if observation.result.receipt.aggregateCompleted { + aggregateOwners++ + } + } + if aggregateOwners != 1 { + t.Fatalf("aggregate owners returned during settlement = %d, want 1", aggregateOwners) + } + + persistedOwners := 0 + for _, job := range jobs { + receipt, known, err := receipts.batchTransitionReceipt(ctx, batchID, job.JobID) + if err != nil || !known || receipt.owner != claims[job.JobID] || receipt.outcome != outcomes[job.JobID] { + t.Fatalf("persisted receipt for %q = known:%t receipt:%+v err:%v", job.JobID, known, receipt, err) + } + if receipt.aggregateCompleted { + persistedOwners++ + if receipt.aggregateCancelled != !policy.allowFailures { + t.Fatalf("terminal receipt for %q cancelled = %t, want %t", job.JobID, receipt.aggregateCancelled, !policy.allowFailures) + } + } + } + if persistedOwners != 1 { + t.Fatalf("persisted aggregate receipt owners = %d, want 1", persistedOwners) + } + state, err := store.GetBatch(ctx, batchID) + if err != nil { + t.Fatalf("get batch: %v", err) + } + if state.Pending != 0 || state.Processed != memberCount || state.Failed != memberCount/2 || !state.Completed || state.Cancelled != !policy.allowFailures { + t.Fatalf("batch state = %+v, want exact counters and cancelled=%t", state, !policy.allowFailures) + } + }) + } + }) + } +} + +// TestStoreContract_LegacyTransitionRemainsReceiptUnknown proves state written +// through the established public API is never retroactively assigned an owner. +func TestStoreContract_LegacyTransitionRemainsReceiptUnknown(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + ctx := context.Background() + store := factory(t) + receipts := requireTransitionReceiptStore(t, store) + if err := store.CreateChain(ctx, ChainRecord{ChainID: "chain-legacy-receipt", DispatchID: "dispatch-legacy-chain", Nodes: []ChainNode{{NodeID: "node-legacy-receipt"}}}); err != nil { + t.Fatalf("create chain: %v", err) + } + if _, done, err := store.AdvanceChain(ctx, "chain-legacy-receipt", "node-legacy-receipt"); err != nil || !done { + t.Fatalf("legacy chain advance = done:%t err:%v", done, err) + } + if receipt, known, err := receipts.chainTransitionReceipt(ctx, "chain-legacy-receipt", "node-legacy-receipt"); err != nil || known { + t.Fatalf("legacy chain receipt = known:%t receipt:%+v err:%v", known, receipt, err) + } + + if err := store.CreateBatch(ctx, BatchRecord{BatchID: "batch-legacy-receipt", DispatchID: "dispatch-legacy-batch", Jobs: []BatchJob{{JobID: "job-legacy-receipt"}}}); err != nil { + t.Fatalf("create batch: %v", err) + } + if _, done, err := store.MarkBatchJobSucceeded(ctx, "batch-legacy-receipt", "job-legacy-receipt"); err != nil || !done { + t.Fatalf("legacy batch settlement = done:%t err:%v", done, err) + } + if receipt, known, err := receipts.batchTransitionReceipt(ctx, "batch-legacy-receipt", "job-legacy-receipt"); err != nil || known { + t.Fatalf("legacy batch receipt = known:%t receipt:%+v err:%v", known, receipt, err) + } + }) + } +} + +// TestStoreContract_RejectsAmbiguousChainRecords protects the immutable order +// required by atomic per-node success and failure compare-and-swap operations. +func TestStoreContract_RejectsAmbiguousChainRecords(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + store := factory(t) + ctx := context.Background() + for _, test := range []struct { + name string + record ChainRecord + }{ + {name: "empty chain id", record: ChainRecord{Nodes: []ChainNode{{NodeID: "node-0"}}}}, + {name: "no nodes", record: ChainRecord{ChainID: "chain-no-nodes"}}, + {name: "empty node id", record: ChainRecord{ChainID: "chain-empty-node", Nodes: []ChainNode{{}}}}, + {name: "duplicate node id", record: ChainRecord{ChainID: "chain-duplicate-node", Nodes: []ChainNode{{NodeID: "node-shared"}, {NodeID: "node-shared"}}}}, + } { + t.Run(test.name, func(t *testing.T) { + if err := store.CreateChain(ctx, test.record); err == nil { + t.Fatal("ambiguous chain record was accepted") + } + if _, err := store.GetChain(ctx, test.record.ChainID); !errors.Is(err, ErrNotFound) { + t.Fatalf("invalid chain persisted: %v", err) + } + }) + } + }) + } +} + +// TestStoreContract_RejectsAmbiguousBatchRecords protects the stable member +// identity required by first-writer outcome arbitration. +func TestStoreContract_RejectsAmbiguousBatchRecords(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + store := factory(t) + ctx := context.Background() + for _, test := range []struct { + name string + record BatchRecord + }{ + {name: "empty batch id", record: BatchRecord{Jobs: []BatchJob{{JobID: "job-0"}}}}, + {name: "no jobs", record: BatchRecord{BatchID: "batch-no-jobs"}}, + {name: "empty job id", record: BatchRecord{BatchID: "batch-empty-job", Jobs: []BatchJob{{}}}}, + {name: "duplicate job id", record: BatchRecord{BatchID: "batch-duplicate-job", Jobs: []BatchJob{{JobID: "job-shared"}, {JobID: "job-shared"}}}}, + } { + t.Run(test.name, func(t *testing.T) { + if err := store.CreateBatch(ctx, test.record); err == nil { + t.Fatal("ambiguous batch record was accepted") + } + if test.record.BatchID == "" { + return + } + if _, err := store.GetBatch(ctx, test.record.BatchID); !errors.Is(err, ErrNotFound) { + t.Fatalf("invalid batch persisted: %v", err) + } + }) + } + }) + } +} + +// TestStoreContract_BatchStartRejectsUnknownMember prevents a malformed +// delivery from creating a synthetic member before outcome settlement. +func TestStoreContract_BatchStartRejectsUnknownMember(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + store := factory(t) + ctx := context.Background() + const batchID = "batch-start-membership" + if err := store.CreateBatch(ctx, BatchRecord{BatchID: batchID, Jobs: []BatchJob{{JobID: "job-known"}}}); err != nil { + t.Fatalf("create batch: %v", err) + } + before, err := store.GetBatch(ctx, batchID) + if err != nil { + t.Fatalf("get batch before unknown start: %v", err) + } + if err := store.MarkBatchJobStarted(ctx, batchID, "job-missing"); !errors.Is(err, ErrNotFound) { + t.Fatalf("unknown member start error = %v, want ErrNotFound", err) + } + after, err := store.GetBatch(ctx, batchID) + if err != nil { + t.Fatalf("get batch after unknown start: %v", err) + } + if after.Pending != before.Pending || after.Processed != before.Processed || after.Failed != before.Failed || after.Completed != before.Completed || !after.UpdatedAt.Equal(before.UpdatedAt) { + t.Fatalf("unknown member start changed batch: before=%+v after=%+v", before, after) + } + if err := store.MarkBatchJobStarted(ctx, batchID, "job-known"); err != nil { + t.Fatalf("start known member: %v", err) + } + if err := store.MarkBatchJobStarted(ctx, batchID, "job-known"); err != nil { + t.Fatalf("replay known member start: %v", err) + } + }) + } +} + +// TestStoreContract_ChainRecordOwnership prevents callers from changing the +// node identity or payload that outcome arbitration treats as immutable. +func TestStoreContract_ChainRecordOwnership(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + store := factory(t) + ctx := context.Background() + record := ChainRecord{ + ChainID: "chain-record-ownership", + Nodes: []ChainNode{ + {NodeID: "node-owned", Job: StoredJob{Payload: []byte("owned")}}, + {NodeID: "node-successor", Job: StoredJob{Payload: []byte("successor")}}, + }, + } + if err := store.CreateChain(ctx, record); err != nil { + t.Fatalf("create chain: %v", err) + } + record.Nodes[0].NodeID = "node-mutated" + record.Nodes[0].Job.Payload[0] = '!' + state, err := store.GetChain(ctx, record.ChainID) + if err != nil { + t.Fatalf("get chain: %v", err) + } + if state.Nodes[0].NodeID != "node-owned" || string(state.Nodes[0].Job.Payload) != "owned" { + t.Fatalf("creation record aliases state: %+v", state.Nodes[0]) + } + state.Nodes[0].NodeID = "node-return-mutated" + state.Nodes[0].Job.Payload[0] = '?' + state, err = store.GetChain(ctx, record.ChainID) + if err != nil { + t.Fatalf("get chain again: %v", err) + } + if state.Nodes[0].NodeID != "node-owned" || string(state.Nodes[0].Job.Payload) != "owned" { + t.Fatalf("returned state aliases store: %+v", state.Nodes[0]) + } + next, done, err := store.AdvanceChain(ctx, record.ChainID, "node-owned") + if err != nil || done || next == nil { + t.Fatalf("advance to successor = next:%+v done:%t err:%v", next, done, err) + } + next.NodeID = "node-successor-mutated" + next.Job.Payload[0] = '!' + state, err = store.GetChain(ctx, record.ChainID) + if err != nil { + t.Fatalf("get chain after successor mutation: %v", err) + } + if state.Nodes[1].NodeID != "node-successor" || string(state.Nodes[1].Job.Payload) != "successor" { + t.Fatalf("returned successor aliases store: %+v", state.Nodes[1]) + } + }) + } +} + +// TestChainNodeDispositionsRejectInvalidPersistedIndex covers corrupt state +// that no valid creation or transition path can produce intentionally. +func TestChainNodeDispositionsRejectInvalidPersistedIndex(t *testing.T) { + for _, nextIndex := range []int{-1, 1} { + state := ChainState{ChainID: "chain-invalid-index", Nodes: []ChainNode{{NodeID: "node-0"}}, NextIndex: nextIndex} + if _, _, _, err := chainNodeAdvanceDisposition(state, "node-0"); err == nil { + t.Fatalf("advance accepted next index %d", nextIndex) + } + if _, _, err := chainNodeFailureDisposition(state, "node-0"); err == nil { + t.Fatalf("failure accepted next index %d", nextIndex) + } + } +} + +// TestStoreContract_ChainNodeOutcomeOwnership proves a physical redelivery +// cannot replace the first result committed for a sequential node. +func TestStoreContract_ChainNodeOutcomeOwnership(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 7*time.Second) + defer cancel() + + t.Run("success first", func(t *testing.T) { + store := factory(t) + outcomes := requireOutcomeStore(t, store) + const chainID = "chain-success-first" + if err := store.CreateChain(ctx, ChainRecord{ChainID: chainID, Nodes: []ChainNode{{NodeID: "node-0"}, {NodeID: "node-1"}}}); err != nil { + t.Fatalf("create chain: %v", err) + } + if _, done, err := store.AdvanceChain(ctx, chainID, "node-0"); err != nil || done { + t.Fatalf("advance first node = done:%t err:%v", done, err) + } + state, owned, err := outcomes.FailChainNode(ctx, chainID, "node-0", errors.New("late failure")) + if err != nil || owned { + t.Fatalf("late failure = owned:%t err:%v", owned, err) + } + if state.NextIndex != 1 || state.Completed || state.Failed { + t.Fatalf("success-first state = %+v", state) + } + }) + + t.Run("failure first and replay", func(t *testing.T) { + store := factory(t) + outcomes := requireOutcomeStore(t, store) + const chainID = "chain-failure-first" + if err := store.CreateChain(ctx, ChainRecord{ChainID: chainID, Nodes: []ChainNode{{NodeID: "node-0"}, {NodeID: "node-1"}}}); err != nil { + t.Fatalf("create chain: %v", err) + } + firstCause := errors.New("first failure") + state, owned, err := outcomes.FailChainNode(ctx, chainID, "node-0", firstCause) + if err != nil || !owned || !state.Failed || state.NextIndex != 0 { + t.Fatalf("first failure = owned:%t state:%+v err:%v", owned, state, err) + } + state, owned, err = outcomes.FailChainNode(ctx, chainID, "node-0", errors.New("replacement failure")) + if err != nil || !owned || state.Failure != firstCause.Error() { + t.Fatalf("failure replay = owned:%t state:%+v err:%v", owned, state, err) + } + if _, done, err := store.AdvanceChain(ctx, chainID, "node-0"); err != nil || !done { + t.Fatalf("late success = done:%t err:%v", done, err) + } + }) + + t.Run("stale and invalid nodes", func(t *testing.T) { + store := factory(t) + outcomes := requireOutcomeStore(t, store) + const chainID = "chain-node-validation" + if err := store.CreateChain(ctx, ChainRecord{ChainID: chainID, Nodes: []ChainNode{{NodeID: "node-0"}, {NodeID: "node-1"}, {NodeID: "node-2"}}}); err != nil { + t.Fatalf("create chain: %v", err) + } + if _, _, err := outcomes.FailChainNode(ctx, chainID, "node-1", errors.New("future failure")); err == nil { + t.Fatal("future failure was accepted") + } + if _, _, err := store.AdvanceChain(ctx, chainID, "node-1"); err == nil { + t.Fatal("future success was accepted") + } + if _, _, err := store.AdvanceChain(ctx, chainID, "missing-node"); err == nil { + t.Fatal("unknown success was accepted") + } + if _, _, err := store.AdvanceChain(ctx, chainID, "node-0"); err != nil { + t.Fatalf("advance current node: %v", err) + } + laterCause := errors.New("later node failed") + if _, owned, err := outcomes.FailChainNode(ctx, chainID, "node-1", laterCause); err != nil || !owned { + t.Fatalf("fail current node = owned:%t err:%v", owned, err) + } + state, owned, err := outcomes.FailChainNode(ctx, chainID, "node-0", errors.New("stale failure")) + if err != nil || owned || !state.Failed || state.Failure != laterCause.Error() || state.NextIndex != 1 { + t.Fatalf("stale failure = owned:%t state:%+v err:%v", owned, state, err) + } + if _, _, err := store.AdvanceChain(ctx, chainID, "node-2"); err == nil { + t.Fatal("future success after failure was accepted") + } + if _, _, err := outcomes.FailChainNode(ctx, chainID, "node-2", errors.New("future failure")); err == nil { + t.Fatal("future failure after failure was accepted") + } + }) + }) + } +} + +// TestStoreContract_TerminalChainRejectsUnknownNodes keeps malformed +// deliveries from inheriting the idempotent result of a real terminal node. +func TestStoreContract_TerminalChainRejectsUnknownNodes(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + for _, terminal := range []string{"completed", "failed"} { + t.Run(terminal, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 7*time.Second) + defer cancel() + store := factory(t) + outcomes := requireOutcomeStore(t, store) + chainID := "chain-terminal-unknown-" + terminal + if err := store.CreateChain(ctx, ChainRecord{ + ChainID: chainID, + Nodes: []ChainNode{ + {NodeID: "node-0"}, + {NodeID: "node-1"}, + }, + }); err != nil { + t.Fatalf("create chain: %v", err) + } + if _, _, err := store.AdvanceChain(ctx, chainID, "node-0"); err != nil { + t.Fatalf("advance first node: %v", err) + } + if terminal == "completed" { + if _, done, err := store.AdvanceChain(ctx, chainID, "node-1"); err != nil || !done { + t.Fatalf("complete chain = done:%t err:%v", done, err) + } + } else { + if _, owned, err := outcomes.FailChainNode(ctx, chainID, "node-1", errors.New("terminal failure")); err != nil || !owned { + t.Fatalf("fail chain = owned:%t err:%v", owned, err) + } + } + before, err := store.GetChain(ctx, chainID) + if err != nil { + t.Fatalf("get terminal chain: %v", err) + } + if _, _, err := store.AdvanceChain(ctx, chainID, "node-missing"); err == nil { + t.Fatal("unknown success inherited terminal state") + } + if _, _, err := outcomes.FailChainNode(ctx, chainID, "node-missing", errors.New("unknown failure")); err == nil { + t.Fatal("unknown failure inherited terminal state") + } + after, err := store.GetChain(ctx, chainID) + if err != nil { + t.Fatalf("get chain after unknown deliveries: %v", err) + } + if after.NextIndex != before.NextIndex || after.Completed != before.Completed || after.Failed != before.Failed || after.Failure != before.Failure || !after.UpdatedAt.Equal(before.UpdatedAt) { + t.Fatalf("unknown delivery changed terminal chain: before=%+v after=%+v", before, after) + } + }) + } + }) + } +} + +// TestStoreContract_ConcurrentChainNodeOutcomeOwnership repeatedly races both +// outcomes and accepts only one of the two valid linearized states. +func TestStoreContract_ConcurrentChainNodeOutcomeOwnership(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + for iteration := range 12 { + store := factory(t) + outcomes := requireOutcomeStore(t, store) + chainID := fmt.Sprintf("chain-outcome-race-%02d", iteration) + if err := store.CreateChain(ctx, ChainRecord{ChainID: chainID, Nodes: []ChainNode{{NodeID: "node-0"}, {NodeID: "node-1"}}}); err != nil { + t.Fatalf("create chain: %v", err) + } + start := make(chan struct{}) + errs := make(chan error, 32) + var wg sync.WaitGroup + for delivery := range 32 { + wg.Add(1) + go func(fail bool) { + defer wg.Done() + <-start + if fail { + _, _, err := outcomes.FailChainNode(ctx, chainID, "node-0", errors.New("raced failure")) + errs <- err + return + } + _, _, err := store.AdvanceChain(ctx, chainID, "node-0") + errs <- err + }(delivery%2 == 0) + } + close(start) + waitStoreContractOperations(t, &wg) + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("race operation: %v", err) + } + } + state, err := store.GetChain(ctx, chainID) + if err != nil { + t.Fatalf("get raced chain: %v", err) + } + successWon := state.NextIndex == 1 && !state.Failed && !state.Completed + failureWon := state.NextIndex == 0 && state.Failed && !state.Completed + if !successWon && !failureWon { + t.Fatalf("non-linearized raced state = %+v", state) + } + } + }) + } +} + +// TestStoreContract_ConcurrentFinalChainNodeOutcomeOwnership races failure +// against the two-step SQL advancement that also marks the chain completed. +func TestStoreContract_ConcurrentFinalChainNodeOutcomeOwnership(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + for iteration := range 8 { + store := factory(t) + outcomes := requireOutcomeStore(t, store) + chainID := fmt.Sprintf("chain-final-outcome-race-%02d", iteration) + if err := store.CreateChain(ctx, ChainRecord{ChainID: chainID, Nodes: []ChainNode{{NodeID: "node-final"}}}); err != nil { + t.Fatalf("create chain: %v", err) + } + start := make(chan struct{}) + errs := make(chan error, 32) + var wg sync.WaitGroup + for delivery := range 32 { + wg.Add(1) + go func(fail bool) { + defer wg.Done() + <-start + if fail { + _, _, err := outcomes.FailChainNode(ctx, chainID, "node-final", errors.New("raced final failure")) + errs <- err + return + } + _, _, err := store.AdvanceChain(ctx, chainID, "node-final") + errs <- err + }(delivery%2 == 0) + } + close(start) + waitStoreContractOperations(t, &wg) + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("race final chain outcome: %v", err) + } + } + state, err := store.GetChain(ctx, chainID) + if err != nil { + t.Fatalf("get raced final chain: %v", err) + } + successWon := state.NextIndex == 1 && state.Completed && !state.Failed + failureWon := state.NextIndex == 0 && !state.Completed && state.Failed + if !successWon && !failureWon { + t.Fatalf("non-linearized final chain state = %+v", state) + } + } + }) + } +} + +// TestStoreContract_BatchJobOutcomeOwnership proves contradictory redelivery +// cannot change either member state or the aggregate's logical winner. +func TestStoreContract_BatchJobOutcomeOwnership(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 7*time.Second) + defer cancel() + for _, first := range []BatchJobOutcome{BatchJobSucceeded, BatchJobFailed} { + t.Run(string(first), func(t *testing.T) { + store := factory(t) + outcomes := requireOutcomeStore(t, store) + batchID := "batch-outcome-" + string(first) + if err := store.CreateBatch(ctx, BatchRecord{BatchID: batchID, AllowFailed: true, Jobs: []BatchJob{{JobID: "job-0"}, {JobID: "job-1"}}}); err != nil { + t.Fatalf("create batch: %v", err) + } + state, owned, err := outcomes.SettleBatchJob(ctx, batchID, "job-0", first, errors.New("first cause")) + if err != nil || !owned { + t.Fatalf("first outcome = owned:%t err:%v", owned, err) + } + if state.Pending != 1 || state.Processed != 1 || state.Failed != boolInt(first == BatchJobFailed) { + t.Fatalf("first outcome state = %+v", state) + } + if _, owned, err := outcomes.SettleBatchJob(ctx, batchID, "job-0", first, nil); err != nil || !owned { + t.Fatalf("same-outcome replay = owned:%t err:%v", owned, err) + } + opposite := BatchJobFailed + if first == BatchJobFailed { + opposite = BatchJobSucceeded + } + state, owned, err = outcomes.SettleBatchJob(ctx, batchID, "job-0", opposite, errors.New("opposite cause")) + if err != nil || owned || state.Pending != 1 || state.Processed != 1 || state.Failed != boolInt(first == BatchJobFailed) { + t.Fatalf("opposite replay = owned:%t state:%+v err:%v", owned, state, err) + } + }) + } + t.Run("invalid outcome", func(t *testing.T) { + store := factory(t) + outcomes := requireOutcomeStore(t, store) + const batchID = "batch-invalid-outcome" + if err := store.CreateBatch(ctx, BatchRecord{BatchID: batchID, Jobs: []BatchJob{{JobID: "job-0"}}}); err != nil { + t.Fatalf("create batch: %v", err) + } + if _, _, err := outcomes.SettleBatchJob(ctx, batchID, "job-0", BatchJobOutcome("unknown"), nil); err == nil { + t.Fatal("invalid batch outcome was accepted") + } + state, err := store.GetBatch(ctx, batchID) + if err != nil { + t.Fatalf("get batch: %v", err) + } + if state.Pending != 1 || state.Processed != 0 || state.Failed != 0 || state.Completed { + t.Fatalf("invalid outcome changed state: %+v", state) + } + }) + t.Run("missing member", func(t *testing.T) { + store := factory(t) + outcomes := requireOutcomeStore(t, store) + const batchID = "batch-missing-member" + if err := store.CreateBatch(ctx, BatchRecord{BatchID: batchID, Jobs: []BatchJob{{JobID: "job-known"}}}); err != nil { + t.Fatalf("create batch: %v", err) + } + if _, _, err := outcomes.SettleBatchJob(ctx, batchID, "job-missing", BatchJobSucceeded, nil); !errors.Is(err, ErrNotFound) { + t.Fatalf("missing member outcome error = %v, want ErrNotFound", err) + } + state, err := store.GetBatch(ctx, batchID) + if err != nil { + t.Fatalf("get batch: %v", err) + } + if state.Pending != 1 || state.Processed != 0 || state.Failed != 0 || state.Completed { + t.Fatalf("missing member outcome changed state: %+v", state) + } + }) + }) + } +} + +// TestStoreContract_ConcurrentBatchJobOutcomeOwnership makes every delivery +// observe one immutable member winner while aggregate counters advance once. +func TestStoreContract_ConcurrentBatchJobOutcomeOwnership(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + store := factory(t) + outcomes := requireOutcomeStore(t, store) + ctx, cancel := context.WithTimeout(context.Background(), 7*time.Second) + defer cancel() + const batchID = "batch-concurrent-outcome" + if err := store.CreateBatch(ctx, BatchRecord{BatchID: batchID, AllowFailed: true, Jobs: []BatchJob{{JobID: "job-shared"}, {JobID: "job-pending"}}}); err != nil { + t.Fatalf("create batch: %v", err) + } + start := make(chan struct{}) + errs := make(chan error, 32) + var wg sync.WaitGroup + for delivery := range 32 { + outcome := BatchJobSucceeded + if delivery%2 == 0 { + outcome = BatchJobFailed + } + wg.Add(1) + go func(outcome BatchJobOutcome) { + defer wg.Done() + <-start + _, _, err := outcomes.SettleBatchJob(ctx, batchID, "job-shared", outcome, errors.New("raced outcome")) + errs <- err + }(outcome) + } + close(start) + waitStoreContractOperations(t, &wg) + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("concurrent settlement: %v", err) + } + } + state, err := store.GetBatch(ctx, batchID) + if err != nil { + t.Fatalf("get batch: %v", err) + } + if state.Pending != 1 || state.Processed != 1 || (state.Failed != 0 && state.Failed != 1) || state.Completed { + t.Fatalf("concurrent outcome state = %+v", state) + } + _, successOwned, err := outcomes.SettleBatchJob(ctx, batchID, "job-shared", BatchJobSucceeded, nil) + if err != nil { + t.Fatalf("replay success: %v", err) + } + _, failureOwned, err := outcomes.SettleBatchJob(ctx, batchID, "job-shared", BatchJobFailed, errors.New("replayed failure")) + if err != nil { + t.Fatalf("replay failure: %v", err) + } + if successOwned == failureOwned || successOwned != (state.Failed == 0) { + t.Fatalf("replay ownership = success:%t failure:%t state:%+v", successOwned, failureOwned, state) + } + }) + } +} + +// TestStoreContract_ConcurrentTerminalBatchJobOutcomeOwnership races both +// categories through final completion and fail-fast cancellation branches. +func TestStoreContract_ConcurrentTerminalBatchJobOutcomeOwnership(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + for _, policy := range []struct { + name string + allowFailures bool + }{ + {name: "allow_failures", allowFailures: true}, + {name: "fail_fast", allowFailures: false}, + } { + t.Run(policy.name, func(t *testing.T) { + for iteration := range 6 { + store := factory(t) + outcomes := requireOutcomeStore(t, store) + batchID := fmt.Sprintf("batch-terminal-outcome-%s-%02d", policy.name, iteration) + if err := store.CreateBatch(ctx, BatchRecord{ + BatchID: batchID, + AllowFailed: policy.allowFailures, + Jobs: []BatchJob{{JobID: "job-final"}}, + }); err != nil { + t.Fatalf("create batch: %v", err) + } + start := make(chan struct{}) + errs := make(chan error, 32) + var wg sync.WaitGroup + for delivery := range 32 { + outcome := BatchJobSucceeded + if delivery%2 == 0 { + outcome = BatchJobFailed + } + wg.Add(1) + go func(outcome BatchJobOutcome) { + defer wg.Done() + <-start + _, _, err := outcomes.SettleBatchJob(ctx, batchID, "job-final", outcome, errors.New("raced terminal outcome")) + errs <- err + }(outcome) + } + close(start) + waitStoreContractOperations(t, &wg) + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("race terminal batch outcome: %v", err) + } + } + state, err := store.GetBatch(ctx, batchID) + if err != nil { + t.Fatalf("get terminal batch: %v", err) + } + if state.Pending != 0 || state.Processed != 1 || !state.Completed || (state.Failed != 0 && state.Failed != 1) { + t.Fatalf("terminal batch state = %+v", state) + } + wantCancelled := state.Failed == 1 && !policy.allowFailures + if state.Cancelled != wantCancelled { + t.Fatalf("terminal batch cancellation = %t, want %t for state %+v", state.Cancelled, wantCancelled, state) + } + } + }) + } + }) + } +} + +// boolInt keeps aggregate expectations readable without obscuring the outcome +// condition inside test tables. +func boolInt(value bool) int { + if value { + return 1 + } + return 0 +} + +// TestStoreContract_ConcurrentDuplicateBatchSettlement proves redelivery can +// claim one member only once even when every delivery observes it concurrently. +func TestStoreContract_ConcurrentDuplicateBatchSettlement(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + s := factory(t) + ctx, cancel := context.WithTimeout(context.Background(), 7*time.Second) + defer cancel() + const batchID = "batch-concurrent-duplicate" + if err := s.CreateBatch(ctx, BatchRecord{ + BatchID: batchID, + DispatchID: "dispatch-concurrent-duplicate", + AllowFailed: true, + Jobs: []BatchJob{ + {JobID: "job-shared", Job: StoredJob{Type: "reports:shared"}}, + {JobID: "job-final", Job: StoredJob{Type: "reports:final"}}, + }, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create batch: %v", err) + } + + const deliveries = 32 + start := make(chan struct{}) + errs := make(chan error, deliveries) + var wg sync.WaitGroup + for range deliveries { + wg.Add(1) + go func() { + defer wg.Done() + <-start + _, _, err := s.MarkBatchJobSucceeded(ctx, batchID, "job-shared") + errs <- err + }() + } + close(start) + waitStoreContractOperations(t, &wg) + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("concurrent duplicate settlement: %v", err) + } + } + + state, err := s.GetBatch(ctx, batchID) + if err != nil { + t.Fatalf("get batch: %v", err) + } + if state.Pending != 1 || state.Processed != 1 || state.Failed != 0 || state.Completed { + t.Fatalf("duplicate settlement state = %+v, want one processed and one pending", state) + } + }) + } +} + +// TestStoreContract_ConcurrentDistinctBatchSettlement proves aggregate +// counters cannot overwrite one another when independent members finish. +func TestStoreContract_ConcurrentDistinctBatchSettlement(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + s := factory(t) + ctx, cancel := context.WithTimeout(context.Background(), 7*time.Second) + defer cancel() + const jobCount = 32 + jobs := make([]BatchJob, jobCount) + for i := range jobs { + jobs[i] = BatchJob{ + JobID: fmt.Sprintf("job-%02d", i), + Job: StoredJob{Type: "reports:member"}, + } + } + for _, policy := range []struct { + name string + allowFailures bool + }{ + {name: "allow_failures", allowFailures: true}, + {name: "fail_fast", allowFailures: false}, + } { + t.Run(policy.name, func(t *testing.T) { + batchID := "batch-concurrent-distinct-" + policy.name + if err := s.CreateBatch(ctx, BatchRecord{ + BatchID: batchID, + DispatchID: "dispatch-concurrent-distinct-" + policy.name, + AllowFailed: policy.allowFailures, + Jobs: jobs, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create batch: %v", err) + } + + start := make(chan struct{}) + errs := make(chan error, jobCount) + var wg sync.WaitGroup + for i, job := range jobs { + wg.Add(1) + go func(index int, member BatchJob) { + defer wg.Done() + <-start + var err error + if index%2 == 0 { + _, _, err = s.MarkBatchJobSucceeded(ctx, batchID, member.JobID) + } else { + _, _, err = s.MarkBatchJobFailed(ctx, batchID, member.JobID, errors.New("member failed")) + } + errs <- err + }(i, job) + } + close(start) + waitStoreContractOperations(t, &wg) + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("concurrent distinct settlement: %v", err) + } + } + + state, err := s.GetBatch(ctx, batchID) + if err != nil { + t.Fatalf("get batch: %v", err) + } + wantCancelled := !policy.allowFailures + if state.Pending != 0 || state.Processed != jobCount || state.Failed != jobCount/2 || !state.Completed || state.Cancelled != wantCancelled { + t.Fatalf("concurrent settlement state = %+v, want exact aggregate counters and cancelled=%t", state, wantCancelled) + } + }) + } + }) + } +} + +// TestStoreContract_DuplicateSuccessCannotBecomeFailure keeps the first +// committed member outcome authoritative across inconsistent redelivery. +func TestStoreContract_DuplicateSuccessCannotBecomeFailure(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + s := factory(t) + ctx, cancel := context.WithTimeout(context.Background(), 7*time.Second) + defer cancel() + const batchID = "batch-immutable-outcome" + if err := s.CreateBatch(ctx, BatchRecord{ + BatchID: batchID, + DispatchID: "dispatch-immutable-outcome", + AllowFailed: false, + Jobs: []BatchJob{ + {JobID: "job-first", Job: StoredJob{Type: "reports:first"}}, + {JobID: "job-second", Job: StoredJob{Type: "reports:second"}}, + }, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create batch: %v", err) + } + if _, done, err := s.MarkBatchJobSucceeded(ctx, batchID, "job-first"); err != nil || done { + t.Fatalf("mark first success = done:%t err:%v, want active batch", done, err) + } + + state, done, err := s.MarkBatchJobFailed(ctx, batchID, "job-first", errors.New("inconsistent duplicate")) + if err != nil { + t.Fatalf("mark inconsistent duplicate: %v", err) + } + if done || state.Pending != 1 || state.Processed != 1 || state.Failed != 0 || state.Cancelled || state.Completed { + t.Fatalf("inconsistent duplicate state = %+v done:%t, want original success retained", state, done) + } + }) + } +} + +// TestStoreContract_ConcurrentDuplicateChainAdvance proves every redelivery +// observes the same current successor after one node claim wins. +func TestStoreContract_ConcurrentDuplicateChainAdvance(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + s := factory(t) + ctx, cancel := context.WithTimeout(context.Background(), 7*time.Second) + defer cancel() + const chainID = "chain-concurrent-duplicate" + if err := s.CreateChain(ctx, ChainRecord{ + ChainID: chainID, + DispatchID: "dispatch-concurrent-chain", + Nodes: []ChainNode{ + {NodeID: "node-first", Job: StoredJob{Type: "reports:first"}}, + {NodeID: "node-second", Job: StoredJob{Type: "reports:second"}}, + }, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create chain: %v", err) + } + + const deliveries = 32 + start := make(chan struct{}) + errs := make(chan error, deliveries) + var wg sync.WaitGroup + for range deliveries { + wg.Add(1) + go func() { + defer wg.Done() + <-start + next, done, err := s.AdvanceChain(ctx, chainID, "node-first") + if err == nil && (done || next == nil || next.NodeID != "node-second") { + err = fmt.Errorf("next = %+v done:%t, want node-second", next, done) + } + errs <- err + }() + } + close(start) + waitStoreContractOperations(t, &wg) + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("concurrent duplicate advance: %v", err) + } + } + + state, err := s.GetChain(ctx, chainID) + if err != nil { + t.Fatalf("get chain: %v", err) + } + if state.NextIndex != 1 || state.Completed || state.Failed { + t.Fatalf("concurrent chain state = %+v, want one committed node", state) + } + }) + } +} + +func TestStoreContract_NotFound(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + s := factory(t) + ctx := context.Background() + + if _, err := s.GetChain(ctx, "missing"); !errors.Is(err, ErrNotFound) { + t.Fatalf("expected chain ErrNotFound, got %v", err) + } + if _, err := s.GetBatch(ctx, "missing"); !errors.Is(err, ErrNotFound) { + t.Fatalf("expected batch ErrNotFound, got %v", err) + } + }) + } +} + +func TestStoreContract_ChainAdvanceIdempotent(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + s := factory(t) + ctx := context.Background() + chainID := "chain-contract" + + if err := s.CreateChain(ctx, ChainRecord{ + ChainID: chainID, + DispatchID: "d1", + Queue: "default", + Nodes: []ChainNode{ + {NodeID: "n1", Job: StoredJob{Type: "monitor:poll"}}, + {NodeID: "n2", Job: StoredJob{Type: "monitor:downsample"}}, + }, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create chain: %v", err) + } + + next, done, err := s.AdvanceChain(ctx, chainID, "n1") + if err != nil { + t.Fatalf("advance first: %v", err) + } + if done || next == nil || next.NodeID != "n2" { + t.Fatalf("expected next n2 on first advance, done=%v next=%+v", done, next) + } + + next, done, err = s.AdvanceChain(ctx, chainID, "n1") + if err != nil { + t.Fatalf("advance duplicate: %v", err) + } + if done || next == nil || next.NodeID != "n2" { + t.Fatalf("expected idempotent duplicate advance, done=%v next=%+v", done, next) + } + + next, done, err = s.AdvanceChain(ctx, chainID, "n2") + if err != nil { + t.Fatalf("advance final: %v", err) + } + if !done || next != nil { + t.Fatalf("expected chain done with nil next, done=%v next=%+v", done, next) + } + }) + } +} + +// TestStoreContract_CompletedChainRejectsLateFailure keeps the first terminal +// outcome authoritative when a competing delivery reports failure too late. +func TestStoreContract_CompletedChainRejectsLateFailure(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + store := factory(t) + ctx := context.Background() + const chainID = "chain-completed-before-failure" + if err := store.CreateChain(ctx, ChainRecord{ + ChainID: chainID, + DispatchID: "dispatch-completed-before-failure", + Nodes: []ChainNode{{NodeID: "node-only", Job: StoredJob{Type: "reports:only"}}}, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create chain: %v", err) + } + if next, done, err := store.AdvanceChain(ctx, chainID, "node-only"); err != nil || !done || next != nil { + t.Fatalf("complete chain = next:%+v done:%t err:%v", next, done, err) + } + if err := store.FailChain(ctx, chainID, errors.New("late competing failure")); err != nil { + t.Fatalf("fail completed chain: %v", err) + } + state, err := store.GetChain(ctx, chainID) + if err != nil { + t.Fatalf("get chain: %v", err) + } + if !state.Completed || state.Failed || state.Failure != "" { + t.Fatalf("late failure changed completed chain: %+v", state) + } + }) + } +} + +func TestStoreContract_BatchTerminalBehavior(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + s := factory(t) + ctx := context.Background() + batchID := "batch-contract" + + if err := s.CreateBatch(ctx, BatchRecord{ + BatchID: batchID, + DispatchID: "d1", + Name: "contract", + Queue: "default", + AllowFailed: false, + Jobs: []BatchJob{ + {JobID: "j1", Job: StoredJob{Type: "monitor:poll"}}, + {JobID: "j2", Job: StoredJob{Type: "monitor:downsample"}}, + }, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create batch: %v", err) + } + + st, done, err := s.MarkBatchJobSucceeded(ctx, batchID, "j1") + if err != nil { + t.Fatalf("mark success: %v", err) + } + if done { + t.Fatal("expected batch not done after first success") + } + if st.Pending != 1 || st.Processed != 1 || st.Failed != 0 { + t.Fatalf("unexpected mid state: %+v", st) + } + + st, done, err = s.MarkBatchJobFailed(ctx, batchID, "j2", errors.New("boom")) + if err != nil { + t.Fatalf("mark failed: %v", err) + } + if !done { + t.Fatal("expected batch done on failure when allow_failed=false") + } + if !st.Completed || !st.Cancelled || st.Failed != 1 { + t.Fatalf("unexpected terminal state: %+v", st) + } + }) + } +} + +func TestStoreContract_CallbackMarkerIdempotent(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + s := factory(t) + ctx := context.Background() + key := "batch_finally:contract" + + first, err := s.MarkCallbackInvoked(ctx, key) + if err != nil { + t.Fatalf("first callback marker: %v", err) + } + if !first { + t.Fatal("expected first callback marker=true") + } + + second, err := s.MarkCallbackInvoked(ctx, key) + if err != nil { + t.Fatalf("second callback marker: %v", err) + } + if second { + t.Fatal("expected second callback marker=false") + } + }) + } +} + +func TestStoreContract_PruneClearsOldCallbackMarkers(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + s := factory(t) + ctx := context.Background() + key := "batch_finally:contract-prune" + + first, err := s.MarkCallbackInvoked(ctx, key) + if err != nil { + t.Fatalf("first callback marker: %v", err) + } + if !first { + t.Fatal("expected first callback marker=true") + } + + // Future cutoff ensures just-inserted marker is considered old. + if err := s.Prune(ctx, time.Now().Add(1*time.Minute)); err != nil { + t.Fatalf("prune markers: %v", err) + } + + again, err := s.MarkCallbackInvoked(ctx, key) + if err != nil { + t.Fatalf("callback marker after prune: %v", err) + } + if !again { + t.Fatal("expected callback marker to be insertable again after prune") + } + }) + } +} + +func TestStoreContract_BatchAllowFailuresContinues(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + s := factory(t) + ctx := context.Background() + batchID := "batch-allow-fail-contract" + + if err := s.CreateBatch(ctx, BatchRecord{ + BatchID: batchID, + DispatchID: "d1", + Name: "allow-fail", + Queue: "default", + AllowFailed: true, + Jobs: []BatchJob{ + {JobID: "j1", Job: StoredJob{Type: "monitor:poll"}}, + {JobID: "j2", Job: StoredJob{Type: "monitor:downsample"}}, + {JobID: "j3", Job: StoredJob{Type: "monitor:alert"}}, + }, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create batch: %v", err) + } + + st, done, err := s.MarkBatchJobFailed(ctx, batchID, "j1", errors.New("boom")) + if err != nil { + t.Fatalf("mark first failed: %v", err) + } + if done { + t.Fatal("expected batch to continue when allow_failed=true") + } + if st.Cancelled { + t.Fatal("expected batch not cancelled when allow_failed=true") + } + + st, done, err = s.MarkBatchJobSucceeded(ctx, batchID, "j2") + if err != nil { + t.Fatalf("mark second success: %v", err) + } + if done { + t.Fatal("expected batch still not done after second job") + } + + st, done, err = s.MarkBatchJobSucceeded(ctx, batchID, "j3") + if err != nil { + t.Fatalf("mark third success: %v", err) + } + if !done || !st.Completed { + t.Fatalf("expected batch completed, done=%v state=%+v", done, st) + } + if st.Failed != 1 || st.Processed != 3 || st.Pending != 0 { + t.Fatalf("unexpected final counters: %+v", st) + } + }) + } +} + +func TestStoreContract_BatchDuplicateTerminalUpdateDoesNotDoubleCount(t *testing.T) { + for name, factory := range testStoreFactories(t) { + t.Run(name, func(t *testing.T) { + s := factory(t) + ctx := context.Background() + batchID := "batch-dup-contract" + + if err := s.CreateBatch(ctx, BatchRecord{ + BatchID: batchID, + DispatchID: "d1", + Name: "dup", + Queue: "default", + AllowFailed: true, + Jobs: []BatchJob{ + {JobID: "j1", Job: StoredJob{Type: "monitor:poll"}}, + }, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create batch: %v", err) + } + + st, done, err := s.MarkBatchJobSucceeded(ctx, batchID, "j1") + if err != nil { + t.Fatalf("mark success first: %v", err) + } + if !done || !st.Completed { + t.Fatalf("expected completed after first success, done=%v state=%+v", done, st) + } + + st, done, err = s.MarkBatchJobSucceeded(ctx, batchID, "j1") + if err != nil { + t.Fatalf("mark success duplicate: %v", err) + } + if !done || !st.Completed { + t.Fatalf("expected completed after duplicate success, done=%v state=%+v", done, st) + } + if st.Processed != 1 || st.Pending != 0 || st.Failed != 0 { + t.Fatalf("expected counters unchanged after duplicate terminal update, got %+v", st) + } + }) + } +} diff --git a/internal/workflow/store_memory.go b/internal/workflow/store_memory.go new file mode 100644 index 0000000..4843a89 --- /dev/null +++ b/internal/workflow/store_memory.go @@ -0,0 +1,483 @@ +package workflow + +import ( + "context" + "fmt" + "sync" + "time" +) + +// NewMemoryStore creates an in-memory orchestration store implementation. +func NewMemoryStore() Store { + return &memoryStore{ + chains: make(map[string]*memoryChain), + batch: make(map[string]*memoryBatch), + callbacks: make(map[string]time.Time), + transitionReceipts: make(map[transitionReceiptKey]transitionReceipt), + } +} + +type memoryStore struct { + mu sync.Mutex + chains map[string]*memoryChain + batch map[string]*memoryBatch + callbacks map[string]time.Time + transitionReceipts map[transitionReceiptKey]transitionReceipt +} + +var _ Store = (*memoryStore)(nil) +var _ chainAdvanceStore = (*memoryStore)(nil) +var _ chainFailureStore = (*memoryStore)(nil) +var _ batchSettlementStore = (*memoryStore)(nil) +var _ transitionReceiptStore = (*memoryStore)(nil) + +type memoryChain struct { + state ChainState + completedNode map[string]bool +} + +type batchJobStatus struct { + started bool + done bool + failed bool +} + +type memoryBatch struct { + state BatchState + jobs map[string]batchJobStatus +} + +// CreateChain installs the complete chain under one mutex so readers never observe partial state. +func (m *memoryStore) CreateChain(_ context.Context, rec ChainRecord) error { + if err := validateChainRecord(rec); err != nil { + return err + } + m.mu.Lock() + defer m.mu.Unlock() + now := time.Now() + m.deleteTransitionReceipts(chainTransitionKind, rec.ChainID) + m.chains[rec.ChainID] = &memoryChain{ + state: ChainState{ + ChainID: rec.ChainID, + DispatchID: rec.DispatchID, + Queue: rec.Queue, + Nodes: cloneChainNodes(rec.Nodes), + NextIndex: 0, + CreatedAt: rec.CreatedAt, + UpdatedAt: now, + }, + completedNode: make(map[string]bool), + } + return nil +} + +// AdvanceChain serializes node deduplication and index advancement so retries cannot skip work. +func (m *memoryStore) AdvanceChain(ctx context.Context, chainID string, completedNode string) (next *ChainNode, done bool, err error) { + result, err := m.advanceChainOutcome(ctx, chainID, completedNode, transitionClaim{}) + return result.next, result.done, err +} + +// advanceChainOutcome retains transition ownership under the same mutex used +// for node advancement so a racing delivery cannot repeat continuation effects. +func (m *memoryStore) advanceChainOutcome(_ context.Context, chainID string, completedNode string, claim transitionClaim) (chainAdvanceResult, error) { + m.mu.Lock() + defer m.mu.Unlock() + ch, ok := m.chains[chainID] + if !ok { + return chainAdvanceResult{}, ErrNotFound + } + successOwned, err := chainNodeSuccessDisposition(ch.state, completedNode) + if err != nil { + return chainAdvanceResult{}, err + } + next, done, claimable, err := chainNodeAdvanceDisposition(ch.state, completedNode) + if err != nil { + return chainAdvanceResult{}, err + } + if !claimable { + if next != nil { + cloned := cloneChainNode(*next) + next = &cloned + } + state := ch.state + state.Nodes = cloneChainNodes(state.Nodes) + if state.DispatchID != "" && claim.dispatchID != "" && state.DispatchID != claim.dispatchID { + return chainAdvanceResult{state: state}, nil + } + receipt, receiptKnown := m.transitionReceipt(chainTransitionKind, chainID, completedNode) + return chainAdvanceResult{state: state, next: next, done: done, successOwned: successOwned, receipt: receipt, receiptKnown: receiptKnown}, nil + } + if ch.state.DispatchID != "" && claim.dispatchID != "" && ch.state.DispatchID != claim.dispatchID { + return chainAdvanceResult{}, fmt.Errorf("chain %q dispatch mismatch", chainID) + } + ch.completedNode[completedNode] = true + ch.state.NextIndex++ + ch.state.UpdatedAt = time.Now() + if ch.state.NextIndex >= len(ch.state.Nodes) { + ch.state.Completed = true + state := ch.state + state.Nodes = cloneChainNodes(state.Nodes) + receipt, receiptKnown := m.recordTransitionReceipt(chainTransitionKind, chainID, state.DispatchID, state.CreatedAt, completedNode, BatchJobSucceeded, claim, true, false) + return chainAdvanceResult{state: state, done: true, successOwned: true, claimedNow: true, receipt: receipt, receiptKnown: receiptKnown}, nil + } + n := cloneChainNode(ch.state.Nodes[ch.state.NextIndex]) + state := ch.state + state.Nodes = cloneChainNodes(state.Nodes) + receipt, receiptKnown := m.recordTransitionReceipt(chainTransitionKind, chainID, state.DispatchID, state.CreatedAt, completedNode, BatchJobSucceeded, claim, false, false) + return chainAdvanceResult{state: state, next: &n, successOwned: true, claimedNow: true, receipt: receipt, receiptKnown: receiptKnown}, nil +} + +// FailChainNode serializes failure against advancement so the first outcome +// for a sequential node remains authoritative across physical redelivery. +func (m *memoryStore) FailChainNode(ctx context.Context, chainID, nodeID string, cause error) (ChainState, bool, error) { + result, err := m.failChainOutcome(ctx, chainID, nodeID, cause, transitionClaim{}) + return result.state, result.owned, err +} + +// failChainOutcome records terminal failure and its exact delivery generation +// under one mutex so recovery never needs to replay application code. +func (m *memoryStore) failChainOutcome(_ context.Context, chainID, nodeID string, cause error, claim transitionClaim) (chainFailureResult, error) { + m.mu.Lock() + defer m.mu.Unlock() + ch, ok := m.chains[chainID] + if !ok { + return chainFailureResult{}, ErrNotFound + } + owned, claimable, err := chainNodeFailureDisposition(ch.state, nodeID) + if err != nil { + return chainFailureResult{}, err + } + if !claimable { + state := ch.state + state.Nodes = cloneChainNodes(state.Nodes) + if state.DispatchID != "" && claim.dispatchID != "" && state.DispatchID != claim.dispatchID { + return chainFailureResult{state: state}, nil + } + receipt, receiptKnown := m.transitionReceipt(chainTransitionKind, chainID, nodeID) + return chainFailureResult{state: state, owned: owned, receipt: receipt, receiptKnown: receiptKnown}, nil + } + if ch.state.DispatchID != "" && claim.dispatchID != "" && ch.state.DispatchID != claim.dispatchID { + return chainFailureResult{}, fmt.Errorf("chain %q dispatch mismatch", chainID) + } + ch.state.Failed = true + if cause != nil { + ch.state.Failure = cause.Error() + } + ch.state.UpdatedAt = time.Now() + state := ch.state + state.Nodes = cloneChainNodes(state.Nodes) + receipt, receiptKnown := m.recordTransitionReceipt(chainTransitionKind, chainID, state.DispatchID, state.CreatedAt, nodeID, BatchJobFailed, claim, false, false) + return chainFailureResult{state: state, owned: true, claimedNow: true, receipt: receipt, receiptKnown: receiptKnown}, nil +} + +// FailChain leaves completed chains successful while recording a terminal cause for unfinished work. +func (m *memoryStore) FailChain(_ context.Context, chainID string, cause error) error { + m.mu.Lock() + defer m.mu.Unlock() + ch, ok := m.chains[chainID] + if !ok { + return ErrNotFound + } + if !ch.state.Completed && !ch.state.Failed { + ch.state.Failed = true + if cause != nil { + ch.state.Failure = cause.Error() + } + ch.state.UpdatedAt = time.Now() + } + return nil +} + +// GetChain reads chain state under the same mutex used for every mutation. +func (m *memoryStore) GetChain(_ context.Context, chainID string) (ChainState, error) { + m.mu.Lock() + defer m.mu.Unlock() + ch, ok := m.chains[chainID] + if !ok { + return ChainState{}, ErrNotFound + } + state := ch.state + state.Nodes = cloneChainNodes(state.Nodes) + return state, nil +} + +// DiscardChain removes exactly one transient recording state. It intentionally +// remains outside Store because production retention continues to use Prune. +func (m *memoryStore) DiscardChain(chainID string) { + m.mu.Lock() + delete(m.chains, chainID) + m.deleteTransitionReceipts(chainTransitionKind, chainID) + m.mu.Unlock() +} + +// CreateBatch installs aggregate and per-job state together so readers cannot observe a partial batch. +func (m *memoryStore) CreateBatch(_ context.Context, rec BatchRecord) error { + if err := validateBatchRecord(rec); err != nil { + return err + } + m.mu.Lock() + defer m.mu.Unlock() + now := time.Now() + m.deleteTransitionReceipts(batchTransitionKind, rec.BatchID) + st := BatchState{ + BatchID: rec.BatchID, + DispatchID: rec.DispatchID, + Name: rec.Name, + Queue: rec.Queue, + AllowFailed: rec.AllowFailed, + Total: len(rec.Jobs), + Pending: len(rec.Jobs), + CreatedAt: rec.CreatedAt, + UpdatedAt: now, + } + jm := make(map[string]batchJobStatus, len(rec.Jobs)) + for _, job := range rec.Jobs { + jm[job.JobID] = batchJobStatus{} + } + m.batch[rec.BatchID] = &memoryBatch{ + state: st, + jobs: jm, + } + return nil +} + +// MarkBatchJobStarted records a retry-safe started marker without changing aggregate counters. +func (m *memoryStore) MarkBatchJobStarted(_ context.Context, batchID, jobID string) error { + m.mu.Lock() + defer m.mu.Unlock() + b, ok := m.batch[batchID] + if !ok { + return ErrNotFound + } + js, ok := b.jobs[jobID] + if !ok { + return ErrNotFound + } + js.started = true + b.jobs[jobID] = js + b.state.UpdatedAt = time.Now() + return nil +} + +// MarkBatchJobSucceeded applies completion counters at most once while holding the aggregate lock. +func (m *memoryStore) MarkBatchJobSucceeded(ctx context.Context, batchID, jobID string) (BatchState, bool, error) { + state, _, err := m.SettleBatchJob(ctx, batchID, jobID, BatchJobSucceeded, nil) + return state, state.Completed, err +} + +// MarkBatchJobFailed counts each failure once and applies fail-fast cancellation atomically. +func (m *memoryStore) MarkBatchJobFailed(ctx context.Context, batchID, jobID string, cause error) (BatchState, bool, error) { + state, _, err := m.SettleBatchJob(ctx, batchID, jobID, BatchJobFailed, cause) + return state, state.Completed, err +} + +// SettleBatchJob serializes aggregate counters with per-member outcome +// ownership so inconsistent redelivery cannot publish a different result. +func (m *memoryStore) SettleBatchJob(ctx context.Context, batchID, jobID string, outcome BatchJobOutcome, cause error) (BatchState, bool, error) { + result, err := m.settleBatchOutcome(ctx, batchID, jobID, outcome, cause, transitionClaim{}) + return result.state, result.owned, err +} + +// settleBatchOutcome retains the first counter claim alongside category +// ownership so fact recovery cannot impersonate a later terminal member. +func (m *memoryStore) settleBatchOutcome(_ context.Context, batchID, jobID string, outcome BatchJobOutcome, _ error, claim transitionClaim) (batchSettlementResult, error) { + if outcome != BatchJobSucceeded && outcome != BatchJobFailed { + return batchSettlementResult{}, fmt.Errorf("unsupported batch job outcome %q", outcome) + } + m.mu.Lock() + defer m.mu.Unlock() + b, ok := m.batch[batchID] + if !ok { + return batchSettlementResult{}, ErrNotFound + } + js, ok := b.jobs[jobID] + if !ok { + return batchSettlementResult{}, ErrNotFound + } + requestedFailure := outcome == BatchJobFailed + if js.done { + if b.state.DispatchID != "" && claim.dispatchID != "" && b.state.DispatchID != claim.dispatchID { + return batchSettlementResult{state: b.state}, nil + } + b.state.UpdatedAt = time.Now() + receipt, receiptKnown := m.transitionReceipt(batchTransitionKind, batchID, jobID) + return batchSettlementResult{state: b.state, owned: js.failed == requestedFailure, receipt: receipt, receiptKnown: receiptKnown}, nil + } + if b.state.DispatchID != "" && claim.dispatchID != "" && b.state.DispatchID != claim.dispatchID { + return batchSettlementResult{}, fmt.Errorf("batch %q dispatch mismatch", batchID) + } + js.done = true + js.failed = requestedFailure + b.jobs[jobID] = js + wasCompleted := b.state.Completed + b.state.Pending-- + b.state.Processed++ + if requestedFailure { + b.state.Failed++ + } + if requestedFailure && !b.state.AllowFailed { + b.state.Cancelled = true + b.state.Completed = true + } else if b.state.Pending <= 0 { + b.state.Completed = true + } + b.state.UpdatedAt = time.Now() + aggregateCompleted := !wasCompleted && b.state.Completed + receipt, receiptKnown := m.recordTransitionReceipt(batchTransitionKind, batchID, b.state.DispatchID, b.state.CreatedAt, jobID, outcome, claim, aggregateCompleted, aggregateCompleted && b.state.Cancelled) + return batchSettlementResult{state: b.state, owned: true, claimedNow: true, receipt: receipt, receiptKnown: receiptKnown}, nil +} + +// recordTransitionReceipt persists immutable generation provenance only when a +// settlement owner supplied a complete transition claim. +func (m *memoryStore) recordTransitionReceipt(kind, workflowID, workflowDispatchID string, workflowCreatedAt time.Time, memberID string, outcome BatchJobOutcome, claim transitionClaim, aggregateCompleted, aggregateCancelled bool) (transitionReceipt, bool) { + if !claim.valid() { + return transitionReceipt{}, false + } + receipt := transitionReceipt{ + version: transitionReceiptVersion, + eventSchemaVersion: eventSchemaVersion, + workflowKind: kind, + workflowID: workflowID, + workflowDispatchID: workflowDispatchID, + workflowCreatedAt: workflowCreatedAt, + memberID: memberID, + outcome: outcome, + owner: claim, + aggregateCompleted: aggregateCompleted, + aggregateCancelled: aggregateCancelled, + createdAt: time.Now(), + } + m.transitionReceipts[transitionReceiptKey{workflowKind: kind, workflowID: workflowID, memberID: memberID}] = receipt + return receipt, true +} + +// transitionReceipt returns one immutable receipt while the caller holds the +// store mutex used for the corresponding state transition. +func (m *memoryStore) transitionReceipt(kind, workflowID, memberID string) (transitionReceipt, bool) { + receipt, ok := m.transitionReceipts[transitionReceiptKey{workflowKind: kind, workflowID: workflowID, memberID: memberID}] + return receipt, ok +} + +// chainTransitionReceipt distinguishes corrupt cross-incarnation provenance +// from a genuinely absent receipt so recovery always fails closed. +func (m *memoryStore) chainTransitionReceipt(_ context.Context, chainID, nodeID string) (transitionReceipt, bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + chain, ok := m.chains[chainID] + if !ok { + return transitionReceipt{}, false, ErrNotFound + } + receipt, ok := m.transitionReceipt(chainTransitionKind, chainID, nodeID) + if !ok { + return transitionReceipt{}, false, nil + } + if receipt.workflowDispatchID != chain.state.DispatchID || !receipt.workflowCreatedAt.Equal(chain.state.CreatedAt) { + return transitionReceipt{}, false, fmt.Errorf("chain %q transition receipt does not match current workflow incarnation", chainID) + } + if err := validateTransitionReceiptSupport(receipt); err != nil { + return transitionReceipt{}, false, err + } + return receipt, true, nil +} + +// batchTransitionReceipt returns provenance only when it still belongs to the +// current batch incarnation. +func (m *memoryStore) batchTransitionReceipt(_ context.Context, batchID, jobID string) (transitionReceipt, bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + batch, ok := m.batch[batchID] + if !ok { + return transitionReceipt{}, false, ErrNotFound + } + receipt, ok := m.transitionReceipt(batchTransitionKind, batchID, jobID) + if !ok { + return transitionReceipt{}, false, nil + } + if receipt.workflowDispatchID != batch.state.DispatchID || !receipt.workflowCreatedAt.Equal(batch.state.CreatedAt) { + return transitionReceipt{}, false, fmt.Errorf("batch %q transition receipt does not match current workflow incarnation", batchID) + } + if err := validateTransitionReceiptSupport(receipt); err != nil { + return transitionReceipt{}, false, err + } + return receipt, true, nil +} + +// deleteTransitionReceipts removes stale in-memory provenance before a test or +// local caller intentionally reuses a workflow identifier. +func (m *memoryStore) deleteTransitionReceipts(kind, workflowID string) { + for key := range m.transitionReceipts { + if key.workflowKind == kind && key.workflowID == workflowID { + delete(m.transitionReceipts, key) + } + } +} + +// CancelBatch marks the batch terminal under the mutation lock so observers see a consistent state. +func (m *memoryStore) CancelBatch(_ context.Context, batchID string) error { + m.mu.Lock() + defer m.mu.Unlock() + b, ok := m.batch[batchID] + if !ok { + return ErrNotFound + } + b.state.Cancelled = true + b.state.Completed = true + b.state.UpdatedAt = time.Now() + return nil +} + +// GetBatch reads aggregate batch state under the same mutex used for settlement. +func (m *memoryStore) GetBatch(_ context.Context, batchID string) (BatchState, error) { + m.mu.Lock() + defer m.mu.Unlock() + b, ok := m.batch[batchID] + if !ok { + return BatchState{}, ErrNotFound + } + return b.state, nil +} + +// DiscardBatch removes exactly one transient recording state. It intentionally +// remains outside Store because production retention continues to use Prune. +func (m *memoryStore) DiscardBatch(batchID string) { + m.mu.Lock() + delete(m.batch, batchID) + m.deleteTransitionReceipts(batchTransitionKind, batchID) + m.mu.Unlock() +} + +// MarkCallbackInvoked atomically reserves a callback key so retries cannot invoke it twice. +func (m *memoryStore) MarkCallbackInvoked(_ context.Context, key string) (bool, error) { + m.mu.Lock() + defer m.mu.Unlock() + if _, exists := m.callbacks[key]; exists { + return false, nil + } + m.callbacks[key] = time.Now() + return true, nil +} + +// Prune removes only expired terminal workflows and callback markers while mutations are excluded. +func (m *memoryStore) Prune(_ context.Context, before time.Time) error { + m.mu.Lock() + defer m.mu.Unlock() + + for chainID, ch := range m.chains { + if (ch.state.Completed || ch.state.Failed) && ch.state.UpdatedAt.Before(before) { + delete(m.chains, chainID) + m.deleteTransitionReceipts(chainTransitionKind, chainID) + } + } + for batchID, b := range m.batch { + if b.state.Completed && b.state.UpdatedAt.Before(before) { + delete(m.batch, batchID) + m.deleteTransitionReceipts(batchTransitionKind, batchID) + } + } + for key, createdAt := range m.callbacks { + if createdAt.Before(before) { + delete(m.callbacks, key) + } + } + return nil +} diff --git a/internal/workflow/store_memory_test.go b/internal/workflow/store_memory_test.go new file mode 100644 index 0000000..f84e710 --- /dev/null +++ b/internal/workflow/store_memory_test.go @@ -0,0 +1,143 @@ +package workflow + +import ( + "context" + "errors" + "testing" + "time" +) + +// TestMemoryStoreDiscardIsExactAndIdempotent pins the recording-only cleanup +// capability without making it part of the durable Store contract. +func TestMemoryStoreDiscardIsExactAndIdempotent(t *testing.T) { + store := NewMemoryStore() + discarder, ok := store.(interface { + DiscardChain(string) + DiscardBatch(string) + }) + if !ok { + t.Fatal("memory store does not expose exact discard capability") + } + ctx := context.Background() + for _, chainID := range []string{"chain-discard", "chain-keep"} { + if err := store.CreateChain(ctx, ChainRecord{ + ChainID: chainID, + Nodes: []ChainNode{{NodeID: chainID + "-node", Job: StoredJob{Type: "chain:job"}}}, + }); err != nil { + t.Fatalf("create chain %q: %v", chainID, err) + } + } + for _, batchID := range []string{"batch-discard", "batch-keep"} { + if err := store.CreateBatch(ctx, BatchRecord{ + BatchID: batchID, + Jobs: []BatchJob{{JobID: batchID + "-job", Job: StoredJob{Type: "batch:job"}}}, + }); err != nil { + t.Fatalf("create batch %q: %v", batchID, err) + } + } + for _, chainID := range []string{"chain-discard", "chain-keep"} { + if err := store.FailChain(ctx, chainID, errors.New("rejected")); err != nil { + t.Fatalf("fail chain %q: %v", chainID, err) + } + } + for _, batchID := range []string{"batch-discard", "batch-keep"} { + if err := store.CancelBatch(ctx, batchID); err != nil { + t.Fatalf("cancel batch %q: %v", batchID, err) + } + } + + discarder.DiscardChain("chain-discard") + discarder.DiscardChain("chain-discard") + discarder.DiscardBatch("batch-discard") + discarder.DiscardBatch("batch-discard") + if _, err := store.GetChain(ctx, "chain-discard"); !errors.Is(err, ErrNotFound) { + t.Fatalf("discarded chain error = %v, want ErrNotFound", err) + } + if _, err := store.GetBatch(ctx, "batch-discard"); !errors.Is(err, ErrNotFound) { + t.Fatalf("discarded batch error = %v, want ErrNotFound", err) + } + if _, err := store.GetChain(ctx, "chain-keep"); err != nil { + t.Fatalf("unrelated chain was discarded: %v", err) + } + if _, err := store.GetBatch(ctx, "batch-keep"); err != nil { + t.Fatalf("unrelated batch was discarded: %v", err) + } +} + +func TestMemoryStorePruneRemovesTerminalRecordsOnly(t *testing.T) { + s := NewMemoryStore() + ctx := context.Background() + old := time.Now().Add(-2 * time.Hour) + cutoff := time.Now().Add(1 * time.Minute) + + if err := s.CreateChain(ctx, ChainRecord{ + ChainID: "chain-old-done", + DispatchID: "d1", + Queue: "default", + Nodes: []ChainNode{{NodeID: "n1", Job: StoredJob{Type: "monitor:poll"}}}, + CreatedAt: old, + }); err != nil { + t.Fatalf("create chain old done: %v", err) + } + if _, _, err := s.AdvanceChain(ctx, "chain-old-done", "n1"); err != nil { + t.Fatalf("advance old done chain: %v", err) + } + + if err := s.CreateChain(ctx, ChainRecord{ + ChainID: "chain-old-failed", + DispatchID: "d2", + Queue: "default", + Nodes: []ChainNode{{NodeID: "n1", Job: StoredJob{Type: "monitor:downsample"}}}, + CreatedAt: old, + }); err != nil { + t.Fatalf("create chain old failed: %v", err) + } + if err := s.FailChain(ctx, "chain-old-failed", errors.New("boom")); err != nil { + t.Fatalf("fail old chain: %v", err) + } + + if err := s.CreateBatch(ctx, BatchRecord{ + BatchID: "batch-old-done", + DispatchID: "d3", + Name: "old", + Queue: "default", + AllowFailed: true, + Jobs: []BatchJob{{JobID: "j1", Job: StoredJob{Type: "monitor:poll"}}}, + CreatedAt: old, + }); err != nil { + t.Fatalf("create batch old done: %v", err) + } + if _, _, err := s.MarkBatchJobSucceeded(ctx, "batch-old-done", "j1"); err != nil { + t.Fatalf("mark batch old done: %v", err) + } + + if err := s.CreateChain(ctx, ChainRecord{ + ChainID: "chain-active", + DispatchID: "d4", + Queue: "default", + Nodes: []ChainNode{ + {NodeID: "n1", Job: StoredJob{Type: "monitor:poll"}}, + {NodeID: "n2", Job: StoredJob{Type: "monitor:alert"}}, + }, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create active chain: %v", err) + } + + if err := s.Prune(ctx, cutoff); err != nil { + t.Fatalf("prune: %v", err) + } + + if _, err := s.GetChain(ctx, "chain-old-done"); !errors.Is(err, ErrNotFound) { + t.Fatalf("expected old completed chain pruned, got err=%v", err) + } + if _, err := s.GetChain(ctx, "chain-old-failed"); !errors.Is(err, ErrNotFound) { + t.Fatalf("expected old failed chain pruned, got err=%v", err) + } + if _, err := s.GetBatch(ctx, "batch-old-done"); !errors.Is(err, ErrNotFound) { + t.Fatalf("expected old completed batch pruned, got err=%v", err) + } + if _, err := s.GetChain(ctx, "chain-active"); err != nil { + t.Fatalf("expected active chain retained, got err=%v", err) + } +} diff --git a/internal/workflow/store_sql.go b/internal/workflow/store_sql.go new file mode 100644 index 0000000..9a539df --- /dev/null +++ b/internal/workflow/store_sql.go @@ -0,0 +1,1679 @@ +package workflow + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "strings" + "sync" + "time" +) + +// mysqlWorkflowIdentifierLimit keeps both columns of each composite primary +// key below MySQL's portable InnoDB index budget while retaining exact bytes. +const mysqlWorkflowIdentifierLimit = 255 + +// mysqlWorkflowCallbackKeyLimit accommodates a callback-kind prefix plus the +// maximum workflow identifier without weakening exact-key comparison. +const mysqlWorkflowCallbackKeyLimit = 512 + +// SQLStoreConfig configures connection ownership, dialect binding, and schema setup for a SQL store. +type SQLStoreConfig struct { + DB *sql.DB + DriverName string + DSN string + // AutoMigrate is retained for compatibility with the established config + // shape. NewSQLStore keeps startup schema creation enabled by default; use + // NewSQLStoreWithManagedSchema when deployment tooling owns the schema. + AutoMigrate bool +} + +// NewSQLStore creates a SQL-backed orchestration store. +func NewSQLStore(cfg SQLStoreConfig) (Store, error) { + return newSQLStore(cfg, true) +} + +// NewSQLStoreWithManagedSchema creates a SQL-backed orchestration store that +// executes no schema DDL because deployment tooling owns the required tables. +func NewSQLStoreWithManagedSchema(cfg SQLStoreConfig) (Store, error) { + return newSQLStore(cfg, false) +} + +// newSQLStore centralizes connection setup while keeping migration policy an +// explicit constructor choice instead of overloading a false zero value. +func newSQLStore(cfg SQLStoreConfig, autoMigrate bool) (Store, error) { + if cfg.DB == nil { + if cfg.DriverName == "" { + return nil, fmt.Errorf("sql store driver name is required") + } + if cfg.DSN == "" { + return nil, fmt.Errorf("sql store dsn is required") + } + db, err := sql.Open(cfg.DriverName, cfg.DSN) + if err != nil { + return nil, err + } + cfg.DB = db + } + if cfg.DriverName == "" { + cfg.DriverName = "sqlite" + } + return &sqlStore{ + db: cfg.DB, + driverName: cfg.DriverName, + autoMigrate: autoMigrate, + }, nil +} + +// sqlStore persists workflow state and transition ownership in one database. +type sqlStore struct { + db *sql.DB + driverName string + autoMigrate bool + mysqlKeyLimit mysqlWorkflowKeyLimits + + ensureMu sync.Mutex + schemaReady bool +} + +// transitionReceiptQueryer lets receipt reads share one implementation across +// committed database state and an in-flight workflow transaction. +type transitionReceiptQueryer interface { + QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row +} + +var ( + _ Store = (*sqlStore)(nil) + _ chainAdvanceStore = (*sqlStore)(nil) + _ chainFailureStore = (*sqlStore)(nil) + _ batchSettlementStore = (*sqlStore)(nil) + _ transitionReceiptStore = (*sqlStore)(nil) +) + +// mysqlColumnCapacity retains both limits because multibyte managed schemas may +// constrain exact identifiers by characters before they run out of bytes. +type mysqlColumnCapacity struct { + dataType string + characters int64 + bytes int64 +} + +// mysqlWorkflowKeyLimits records each workflow model's effective capacity +// across every table that persists its identifiers. +type mysqlWorkflowKeyLimits struct { + chainID mysqlColumnCapacity + chainNode mysqlColumnCapacity + batchID mysqlColumnCapacity + batchJob mysqlColumnCapacity + callback mysqlColumnCapacity +} + +// mysqlTransitionReceiptWidths records the binary widths needed for the one +// receipt table shared by chain and batch workflows. +type mysqlTransitionReceiptWidths struct { + workflowID int64 + memberID int64 +} + +// ensureSchema serializes schema creation and caches only a successful result +// so transient first-use failures remain retryable without racing the DDL sequence. +func (s *sqlStore) ensureSchema(ctx context.Context) error { + s.ensureMu.Lock() + defer s.ensureMu.Unlock() + if s.schemaReady { + return nil + } + + var limits mysqlWorkflowKeyLimits + if !s.autoMigrate { + if s.driverName == "mysql" { + loaded, err := s.loadMySQLWorkflowKeyLimits(ctx) + if err != nil { + return err + } + limits = loaded + } + } else if s.driverName == "mysql" { + loaded, err := s.ensureMySQLSchema(ctx) + if err != nil { + return err + } + limits = loaded + } else { + for _, stmt := range s.schemaStatements() { + if _, err := s.db.ExecContext(ctx, s.rebind(stmt)); err != nil { + return err + } + } + } + + s.mysqlKeyLimit = limits + s.schemaReady = true + return nil +} + +// ensureMySQLSchema creates legacy state tables before deriving a missing +// receipt table from their live widths, leaving every existing table unchanged. +func (s *sqlStore) ensureMySQLSchema(ctx context.Context) (mysqlWorkflowKeyLimits, error) { + for _, stmt := range s.workflowStateSchemaStatements() { + if _, err := s.db.ExecContext(ctx, stmt); err != nil { + return mysqlWorkflowKeyLimits{}, err + } + } + + receiptExists, err := s.mysqlTransitionReceiptTableExists(ctx) + if err != nil { + return mysqlWorkflowKeyLimits{}, err + } + if !receiptExists { + columns, columnsErr := s.loadMySQLWorkflowColumns(ctx) + if columnsErr != nil { + return mysqlWorkflowKeyLimits{}, columnsErr + } + widths, widthsErr := mysqlTransitionReceiptWidthsFromColumns(columns) + if widthsErr != nil { + return mysqlWorkflowKeyLimits{}, widthsErr + } + statement := s.transitionReceiptSchemaStatement( + "VARBINARY(16)", + fmt.Sprintf("VARBINARY(%d)", widths.workflowID), + fmt.Sprintf("VARBINARY(%d)", widths.memberID), + ) + if _, createErr := s.db.ExecContext(ctx, statement); createErr != nil { + return mysqlWorkflowKeyLimits{}, fmt.Errorf( + "create missing MySQL workflow transition receipt table with derived workflow_id VARBINARY(%d) and member_id VARBINARY(%d): %w; existing legacy tables were not altered, so pre-create bus_workflow_transition_receipts with compatible indexed widths before restarting", + widths.workflowID, + widths.memberID, + createErr, + ) + } + } + return s.loadMySQLWorkflowKeyLimits(ctx) +} + +// mysqlTransitionReceiptTableExists distinguishes a missing receipt table from +// an existing malformed table that automatic startup must never alter. +func (s *sqlStore) mysqlTransitionReceiptTableExists(ctx context.Context) (bool, error) { + var count int + if err := s.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM information_schema.tables + WHERE table_schema=DATABASE() AND table_name='bus_workflow_transition_receipts'`).Scan(&count); err != nil { + return false, err + } + return count > 0, nil +} + +// schemaStatements selects only the physical SQL types that differ by +// dialect while keeping one legacy table and column contract. +func (s *sqlStore) schemaStatements() []string { + statements := s.workflowStateSchemaStatements() + idType := "TEXT" + receiptKindType := "TEXT" + switch s.driverName { + case "mysql": + idType = fmt.Sprintf("VARBINARY(%d)", mysqlWorkflowIdentifierLimit) + receiptKindType = "VARBINARY(16)" + } + return append(statements, s.transitionReceiptSchemaStatement(receiptKindType, idType, idType)) +} + +// workflowStateSchemaStatements returns the five established state and +// callback tables whose live MySQL widths govern a newly introduced receipt. +func (s *sqlStore) workflowStateSchemaStatements() []string { + idType := "TEXT" + callbackKeyType := "TEXT" + payloadType := "BLOB" + switch s.driverName { + case "pgx", "postgres": + payloadType = "BYTEA" + case "mysql": + idType = fmt.Sprintf("VARBINARY(%d)", mysqlWorkflowIdentifierLimit) + callbackKeyType = fmt.Sprintf("VARBINARY(%d)", mysqlWorkflowCallbackKeyLimit) + payloadType = "LONGBLOB" + } + return []string{ + fmt.Sprintf(`CREATE TABLE IF NOT EXISTS bus_chains ( + chain_id %s PRIMARY KEY, + dispatch_id TEXT NOT NULL, + queue_name TEXT NOT NULL, + nodes_json %s NOT NULL, + next_index INTEGER NOT NULL, + completed INTEGER NOT NULL, + failed INTEGER NOT NULL, + failure TEXT NOT NULL, + created_at_ms BIGINT NOT NULL, + updated_at_ms BIGINT NOT NULL + )`, idType, payloadType), + fmt.Sprintf(`CREATE TABLE IF NOT EXISTS bus_chain_completed_nodes ( + chain_id %s NOT NULL, + node_id %s NOT NULL, + created_at_ms BIGINT NOT NULL, + PRIMARY KEY (chain_id, node_id) + )`, idType, idType), + fmt.Sprintf(`CREATE TABLE IF NOT EXISTS bus_batches ( + batch_id %s PRIMARY KEY, + dispatch_id TEXT NOT NULL, + name TEXT NOT NULL, + queue_name TEXT NOT NULL, + allow_failed INTEGER NOT NULL, + total_jobs INTEGER NOT NULL, + pending_jobs INTEGER NOT NULL, + processed_jobs INTEGER NOT NULL, + failed_jobs INTEGER NOT NULL, + cancelled INTEGER NOT NULL, + completed INTEGER NOT NULL, + created_at_ms BIGINT NOT NULL, + updated_at_ms BIGINT NOT NULL + )`, idType), + fmt.Sprintf(`CREATE TABLE IF NOT EXISTS bus_batch_jobs ( + batch_id %s NOT NULL, + job_id %s NOT NULL, + started INTEGER NOT NULL, + done INTEGER NOT NULL, + failed INTEGER NOT NULL, + PRIMARY KEY (batch_id, job_id) + )`, idType, idType), + fmt.Sprintf(`CREATE TABLE IF NOT EXISTS bus_callback_invocations ( + callback_key %s PRIMARY KEY, + created_at_ms BIGINT NOT NULL + )`, callbackKeyType), + } +} + +// transitionReceiptSchemaStatement builds the shared immutable-receipt table +// with caller-selected key types while retaining one dialect-neutral layout. +func (s *sqlStore) transitionReceiptSchemaStatement(kindType, workflowIDType, memberIDType string) string { + return fmt.Sprintf(`CREATE TABLE IF NOT EXISTS bus_workflow_transition_receipts ( + workflow_kind %s NOT NULL, + receipt_version INTEGER NOT NULL, + event_schema_version INTEGER NOT NULL, + workflow_id %s NOT NULL, + member_id %s NOT NULL, + workflow_dispatch_id TEXT NOT NULL, + workflow_created_at_ms BIGINT NOT NULL, + outcome VARCHAR(16) NOT NULL, + owner_delivery_id TEXT NOT NULL, + owner_attempt BIGINT NOT NULL, + job_dispatch_id TEXT NOT NULL, + job_id TEXT NOT NULL, + job_fingerprint TEXT NOT NULL, + aggregate_completed INTEGER NOT NULL, + aggregate_cancelled INTEGER NOT NULL, + created_at_ms BIGINT NOT NULL, + PRIMARY KEY (workflow_kind, workflow_id, member_id) + )`, kindType, workflowIDType, memberIDType) +} + +// loadMySQLWorkflowKeyLimits derives validation from the connected schema so +// wider caller-managed binary columns retain their established capacity. +func (s *sqlStore) loadMySQLWorkflowKeyLimits(ctx context.Context) (mysqlWorkflowKeyLimits, error) { + columns, err := s.loadMySQLWorkflowColumns(ctx) + if err != nil { + return mysqlWorkflowKeyLimits{}, err + } + return mysqlWorkflowKeyLimitsFromColumns(columns) +} + +// loadMySQLWorkflowColumns reads every key-bearing workflow column so startup +// can validate existing schemas and derive only genuinely missing structures. +func (s *sqlStore) loadMySQLWorkflowColumns(ctx context.Context) (map[string]mysqlColumnCapacity, error) { + rows, err := s.db.QueryContext(ctx, `SELECT table_name, column_name, data_type, character_maximum_length, character_octet_length + FROM information_schema.columns + WHERE table_schema=DATABASE() AND ( + (table_name='bus_chains' AND column_name='chain_id') OR + (table_name='bus_chain_completed_nodes' AND column_name IN ('chain_id', 'node_id')) OR + (table_name='bus_batches' AND column_name='batch_id') OR + (table_name='bus_batch_jobs' AND column_name IN ('batch_id', 'job_id')) OR + (table_name='bus_callback_invocations' AND column_name='callback_key') OR + (table_name='bus_workflow_transition_receipts' AND column_name IN ('workflow_id', 'member_id')) + )`) + if err != nil { + return nil, err + } + defer rows.Close() + columns := make(map[string]mysqlColumnCapacity, 9) + for rows.Next() { + var ( + tableName, columnName, dataType string + characters, bytes sql.NullInt64 + ) + if err := rows.Scan(&tableName, &columnName, &dataType, &characters, &bytes); err != nil { + return nil, err + } + if !characters.Valid || !bytes.Valid || characters.Int64 <= 0 || bytes.Int64 <= 0 { + return nil, fmt.Errorf("MySQL workflow column %s.%s has no character capacity", tableName, columnName) + } + columns[tableName+"."+columnName] = mysqlColumnCapacity{dataType: dataType, characters: characters.Int64, bytes: bytes.Int64} + } + if err := rows.Err(); err != nil { + return nil, err + } + return columns, nil +} + +// mysqlWorkflowKeyLimitsFromColumns intersects IDs stored in multiple tables +// and rejects schemas whose comparison semantics can conflate identities. +func mysqlWorkflowKeyLimitsFromColumns(columns map[string]mysqlColumnCapacity) (mysqlWorkflowKeyLimits, error) { + required := []string{ + "bus_chains.chain_id", + "bus_chain_completed_nodes.chain_id", + "bus_chain_completed_nodes.node_id", + "bus_batches.batch_id", + "bus_batch_jobs.batch_id", + "bus_batch_jobs.job_id", + "bus_callback_invocations.callback_key", + "bus_workflow_transition_receipts.workflow_id", + "bus_workflow_transition_receipts.member_id", + } + if err := validateMySQLWorkflowColumns(columns, required); err != nil { + return mysqlWorkflowKeyLimits{}, err + } + return mysqlWorkflowKeyLimits{ + chainID: intersectMySQLColumnCapacity( + intersectMySQLColumnCapacity(columns["bus_chains.chain_id"], columns["bus_chain_completed_nodes.chain_id"]), + columns["bus_workflow_transition_receipts.workflow_id"], + ), + chainNode: intersectMySQLColumnCapacity(columns["bus_chain_completed_nodes.node_id"], columns["bus_workflow_transition_receipts.member_id"]), + batchID: intersectMySQLColumnCapacity( + intersectMySQLColumnCapacity(columns["bus_batches.batch_id"], columns["bus_batch_jobs.batch_id"]), + columns["bus_workflow_transition_receipts.workflow_id"], + ), + batchJob: intersectMySQLColumnCapacity(columns["bus_batch_jobs.job_id"], columns["bus_workflow_transition_receipts.member_id"]), + callback: columns["bus_callback_invocations.callback_key"], + }, nil +} + +// mysqlTransitionReceiptWidthsFromColumns derives one shared table wide enough +// for the narrowest effective chain and batch identifiers in the legacy state. +func mysqlTransitionReceiptWidthsFromColumns(columns map[string]mysqlColumnCapacity) (mysqlTransitionReceiptWidths, error) { + required := []string{ + "bus_chains.chain_id", + "bus_chain_completed_nodes.chain_id", + "bus_chain_completed_nodes.node_id", + "bus_batches.batch_id", + "bus_batch_jobs.batch_id", + "bus_batch_jobs.job_id", + "bus_callback_invocations.callback_key", + } + if err := validateMySQLWorkflowColumns(columns, required); err != nil { + return mysqlTransitionReceiptWidths{}, err + } + chainID := intersectMySQLColumnCapacity(columns["bus_chains.chain_id"], columns["bus_chain_completed_nodes.chain_id"]) + batchID := intersectMySQLColumnCapacity(columns["bus_batches.batch_id"], columns["bus_batch_jobs.batch_id"]) + return mysqlTransitionReceiptWidths{ + workflowID: maxMySQLVARBINARYWidth(chainID, batchID), + memberID: maxMySQLVARBINARYWidth(columns["bus_chain_completed_nodes.node_id"], columns["bus_batch_jobs.job_id"]), + }, nil +} + +// validateMySQLWorkflowColumns rejects incomplete or comparison-unsafe key +// columns before automatic startup creates a dependent receipt table. +func validateMySQLWorkflowColumns(columns map[string]mysqlColumnCapacity, required []string) error { + for _, column := range required { + capacity, ok := columns[column] + if !ok || capacity.characters <= 0 || capacity.bytes <= 0 { + return fmt.Errorf("MySQL workflow schema is missing key capacity for %s", column) + } + if !strings.EqualFold(capacity.dataType, "varbinary") { + return fmt.Errorf("MySQL workflow column %s must use VARBINARY for byte-exact identity; found %s", column, capacity.dataType) + } + } + return nil +} + +// maxMySQLVARBINARYWidth returns the physical width that can represent either +// effective capacity in the receipt table's byte-exact VARBINARY column. +func maxMySQLVARBINARYWidth(left, right mysqlColumnCapacity) int64 { + width := left.bytes + if left.characters > width { + width = left.characters + } + if right.bytes > width { + width = right.bytes + } + if right.characters > width { + width = right.characters + } + return width +} + +// intersectMySQLColumnCapacity uses the narrowest representation because one +// logical ID must round-trip through every column in which it participates. +func intersectMySQLColumnCapacity(left, right mysqlColumnCapacity) mysqlColumnCapacity { + capacity := left + if right.characters < capacity.characters { + capacity.characters = right.characters + } + if right.bytes < capacity.bytes { + capacity.bytes = right.bytes + } + return capacity +} + +// validateMySQLKey rejects values the connected schema could truncate into a +// false duplicate while allowing wider established schemas to keep working. +func (s *sqlStore) validateMySQLKey(label, value string, capacity mysqlColumnCapacity) error { + if s.driverName != "mysql" { + return nil + } + if int64(len(value)) > capacity.bytes { + return fmt.Errorf("%s exceeds MySQL schema limit of %d bytes", label, capacity.bytes) + } + if int64(len([]rune(value))) > capacity.characters { + return fmt.Errorf("%s exceeds MySQL schema limit of %d characters", label, capacity.characters) + } + return nil +} + +// validateTransitionReceiptKeys applies the capacity of the workflow model +// that owns the receipt so a wider batch schema is not constrained by chain +// columns, or vice versa. +func (s *sqlStore) validateTransitionReceiptKeys(receipt transitionReceipt) error { + var workflowCapacity, memberCapacity mysqlColumnCapacity + var workflowLabel, memberLabel string + switch receipt.workflowKind { + case chainTransitionKind: + workflowCapacity = s.mysqlKeyLimit.chainID + memberCapacity = s.mysqlKeyLimit.chainNode + workflowLabel = "chain receipt id" + memberLabel = "chain receipt node id" + case batchTransitionKind: + workflowCapacity = s.mysqlKeyLimit.batchID + memberCapacity = s.mysqlKeyLimit.batchJob + workflowLabel = "batch receipt id" + memberLabel = "batch receipt job id" + default: + return fmt.Errorf("unsupported workflow transition receipt kind %q", receipt.workflowKind) + } + if err := s.validateMySQLKey(workflowLabel, receipt.workflowID, workflowCapacity); err != nil { + return err + } + return s.validateMySQLKey(memberLabel, receipt.memberID, memberCapacity) +} + +// CreateChain persists the complete encoded chain as one durable initial state. +func (s *sqlStore) CreateChain(ctx context.Context, rec ChainRecord) error { + if err := validateChainRecord(rec); err != nil { + return err + } + if err := s.ensureSchema(ctx); err != nil { + return err + } + if err := s.validateMySQLKey("chain id", rec.ChainID, s.mysqlKeyLimit.chainID); err != nil { + return err + } + for _, node := range rec.Nodes { + if err := s.validateMySQLKey("chain node id", node.NodeID, s.mysqlKeyLimit.chainNode); err != nil { + return err + } + } + nodesJSON, err := json.Marshal(rec.Nodes) + if err != nil { + return err + } + if rec.CreatedAt.IsZero() { + rec.CreatedAt = time.Now() + } + now := rec.CreatedAt.UnixMilli() + _, err = s.db.ExecContext(ctx, s.rebind(`INSERT INTO bus_chains + (chain_id, dispatch_id, queue_name, nodes_json, next_index, completed, failed, failure, created_at_ms, updated_at_ms) + VALUES (?, ?, ?, ?, 0, 0, 0, '', ?, ?)`), + rec.ChainID, rec.DispatchID, rec.Queue, nodesJSON, now, now, + ) + return err +} + +// AdvanceChain claims the completed node before atomically incrementing the +// parent so concurrent redelivery cannot return a stale successor. +func (s *sqlStore) AdvanceChain(ctx context.Context, chainID string, completedNode string) (next *ChainNode, done bool, err error) { + result, err := s.advanceChainOutcome(ctx, chainID, completedNode, transitionClaim{}) + return result.next, result.done, err +} + +// advanceChainOutcome couples the compare-and-swap result with the state that +// won it so a recovered delivery cannot mistake an earlier success for its own. +func (s *sqlStore) advanceChainOutcome(ctx context.Context, chainID string, completedNode string, claim transitionClaim) (chainAdvanceResult, error) { + state, err := s.GetChain(ctx, chainID) + if err != nil { + return chainAdvanceResult{}, err + } + successOwned, err := chainNodeSuccessDisposition(state, completedNode) + if err != nil { + return chainAdvanceResult{}, err + } + next, done, claimable, err := chainNodeAdvanceDisposition(state, completedNode) + if err != nil { + return chainAdvanceResult{}, err + } + if !claimable { + if state.DispatchID != "" && claim.dispatchID != "" && state.DispatchID != claim.dispatchID { + return chainAdvanceResult{state: state}, nil + } + receipt, receiptKnown, receiptErr := s.chainTransitionReceipt(ctx, chainID, completedNode) + if receiptErr != nil { + return chainAdvanceResult{}, receiptErr + } + return chainAdvanceResult{state: state, next: next, done: done, successOwned: successOwned, receipt: receipt, receiptKnown: receiptKnown}, nil + } + if state.DispatchID != "" && claim.dispatchID != "" && state.DispatchID != claim.dispatchID { + return chainAdvanceResult{}, fmt.Errorf("chain %q dispatch mismatch", chainID) + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return chainAdvanceResult{}, err + } + defer func() { _ = tx.Rollback() }() + + if _, err := s.insertChainCompletedNode(ctx, tx, chainID, completedNode); err != nil { + return chainAdvanceResult{}, err + } + advancedAt := time.Now() + result, err := tx.ExecContext(ctx, s.rebind(`UPDATE bus_chains + SET next_index=next_index+1, updated_at_ms=? + WHERE chain_id=? AND next_index=? AND completed=0 AND failed=0`), advancedAt.UnixMilli(), chainID, state.NextIndex) + if err != nil { + return chainAdvanceResult{}, err + } + updated, err := result.RowsAffected() + if err != nil { + return chainAdvanceResult{}, err + } + if updated > 1 { + return chainAdvanceResult{}, fmt.Errorf("chain %q advancement updated %d rows", chainID, updated) + } + if updated == 0 { + if err := tx.Rollback(); err != nil { + return chainAdvanceResult{}, err + } + state, err := s.GetChain(ctx, chainID) + if err != nil { + return chainAdvanceResult{}, err + } + successOwned, err := chainNodeSuccessDisposition(state, completedNode) + if err != nil { + return chainAdvanceResult{}, err + } + next, done, claimable, err := chainNodeAdvanceDisposition(state, completedNode) + if err != nil { + return chainAdvanceResult{}, err + } + if !claimable { + if state.DispatchID != "" && claim.dispatchID != "" && state.DispatchID != claim.dispatchID { + return chainAdvanceResult{state: state}, nil + } + receipt, receiptKnown, receiptErr := s.chainTransitionReceipt(ctx, chainID, completedNode) + if receiptErr != nil { + return chainAdvanceResult{}, receiptErr + } + return chainAdvanceResult{state: state, next: next, done: done, successOwned: successOwned, receipt: receipt, receiptKnown: receiptKnown}, nil + } + if state.DispatchID != "" && claim.dispatchID != "" && state.DispatchID != claim.dispatchID { + return chainAdvanceResult{}, fmt.Errorf("chain %q dispatch mismatch", chainID) + } + return chainAdvanceResult{}, fmt.Errorf("chain %q node %q could not claim success", chainID, completedNode) + } + + state.NextIndex++ + state.UpdatedAt = advancedAt + if state.NextIndex >= len(state.Nodes) { + completedAt := time.Now() + result, err := tx.ExecContext(ctx, s.rebind(`UPDATE bus_chains + SET completed=1, updated_at_ms=? + WHERE chain_id=? AND next_index=? AND completed=0 AND failed=0`), completedAt.UnixMilli(), chainID, state.NextIndex) + if err != nil { + return chainAdvanceResult{}, err + } + completed, err := result.RowsAffected() + if err != nil { + return chainAdvanceResult{}, err + } + if completed != 1 { + return chainAdvanceResult{}, fmt.Errorf("chain %q completion updated %d rows", chainID, completed) + } + state.Completed = true + state.UpdatedAt = completedAt + } + receipt, receiptKnown, err := s.insertTransitionReceipt(ctx, tx, transitionReceipt{ + workflowKind: chainTransitionKind, + workflowID: chainID, + workflowDispatchID: state.DispatchID, + workflowCreatedAt: state.CreatedAt, + memberID: completedNode, + outcome: BatchJobSucceeded, + owner: claim, + aggregateCompleted: state.Completed, + createdAt: state.UpdatedAt, + }) + if err != nil { + return chainAdvanceResult{}, err + } + if err := tx.Commit(); err != nil { + return s.readCommittedChainAdvance(ctx, chainID, completedNode, claim, err) + } + if state.Completed { + return chainAdvanceResult{state: state, done: true, successOwned: true, claimedNow: true, receipt: receipt, receiptKnown: receiptKnown}, nil + } + n := state.Nodes[state.NextIndex] + return chainAdvanceResult{state: state, next: &n, successOwned: true, claimedNow: true, receipt: receipt, receiptKnown: receiptKnown}, nil +} + +// FailChainNode conditionally fails only the current node so a success that +// already advanced the chain cannot be reclassified by a late redelivery. +func (s *sqlStore) FailChainNode(ctx context.Context, chainID, nodeID string, cause error) (ChainState, bool, error) { + result, err := s.failChainOutcome(ctx, chainID, nodeID, cause, transitionClaim{}) + return result.state, result.owned, err +} + +// failChainOutcome commits terminal state and immutable delivery provenance in +// one transaction so a later settlement recovery can trust the failure owner. +func (s *sqlStore) failChainOutcome(ctx context.Context, chainID, nodeID string, cause error, claim transitionClaim) (chainFailureResult, error) { + state, err := s.GetChain(ctx, chainID) + if err != nil { + return chainFailureResult{}, err + } + owned, claimable, err := chainNodeFailureDisposition(state, nodeID) + if err != nil { + return chainFailureResult{}, err + } + if !claimable { + if state.DispatchID != "" && claim.dispatchID != "" && state.DispatchID != claim.dispatchID { + return chainFailureResult{state: state}, nil + } + receipt, receiptKnown, receiptErr := s.chainTransitionReceipt(ctx, chainID, nodeID) + if receiptErr != nil { + return chainFailureResult{}, receiptErr + } + return chainFailureResult{state: state, owned: owned, receipt: receipt, receiptKnown: receiptKnown}, nil + } + if state.DispatchID != "" && claim.dispatchID != "" && state.DispatchID != claim.dispatchID { + return chainFailureResult{}, fmt.Errorf("chain %q dispatch mismatch", chainID) + } + + message := "" + if cause != nil { + message = cause.Error() + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return chainFailureResult{}, err + } + defer func() { _ = tx.Rollback() }() + + failedAt := time.Now() + result, err := tx.ExecContext(ctx, s.rebind(`UPDATE bus_chains + SET failed=1, failure=?, updated_at_ms=? + WHERE chain_id=? AND next_index=? AND completed=0 AND failed=0`), message, failedAt.UnixMilli(), chainID, state.NextIndex) + if err != nil { + return chainFailureResult{}, err + } + updated, err := result.RowsAffected() + if err != nil { + return chainFailureResult{}, err + } + if updated == 1 { + state.Failed = true + state.Failure = message + state.UpdatedAt = failedAt + receipt, receiptKnown, receiptErr := s.insertTransitionReceipt(ctx, tx, transitionReceipt{ + workflowKind: chainTransitionKind, + workflowID: chainID, + workflowDispatchID: state.DispatchID, + workflowCreatedAt: state.CreatedAt, + memberID: nodeID, + outcome: BatchJobFailed, + owner: claim, + createdAt: failedAt, + }) + if receiptErr != nil { + return chainFailureResult{}, receiptErr + } + if err := tx.Commit(); err != nil { + return s.readCommittedChainFailure(ctx, chainID, nodeID, claim, err) + } + return chainFailureResult{state: state, owned: true, claimedNow: true, receipt: receipt, receiptKnown: receiptKnown}, nil + } + if updated > 1 { + return chainFailureResult{}, fmt.Errorf("chain %q failure updated %d rows", chainID, updated) + } + if err := tx.Rollback(); err != nil { + return chainFailureResult{}, err + } + + state, err = s.GetChain(ctx, chainID) + if err != nil { + return chainFailureResult{}, err + } + owned, claimable, err = chainNodeFailureDisposition(state, nodeID) + if err != nil { + return chainFailureResult{}, err + } + if state.DispatchID != "" && claim.dispatchID != "" && state.DispatchID != claim.dispatchID { + if claimable { + return chainFailureResult{}, fmt.Errorf("chain %q dispatch mismatch", chainID) + } + return chainFailureResult{state: state}, nil + } + if claimable { + return chainFailureResult{}, fmt.Errorf("chain %q node %q could not claim failure", chainID, nodeID) + } + receipt, receiptKnown, receiptErr := s.chainTransitionReceipt(ctx, chainID, nodeID) + if receiptErr != nil { + return chainFailureResult{}, receiptErr + } + return chainFailureResult{state: state, owned: owned, receipt: receipt, receiptKnown: receiptKnown}, nil +} + +// FailChain preserves an already committed completion while recording a +// terminal cause only for an unfinished chain. +func (s *sqlStore) FailChain(ctx context.Context, chainID string, cause error) error { + if err := s.ensureSchema(ctx); err != nil { + return err + } + msg := "" + if cause != nil { + msg = cause.Error() + } + _, err := s.db.ExecContext(ctx, s.rebind(`UPDATE bus_chains SET failed=1, failure=?, updated_at_ms=? WHERE chain_id=? AND completed=0 AND failed=0`), msg, time.Now().UnixMilli(), chainID) + return err +} + +// GetChain decodes the stored node payload and normalizes missing rows to ErrNotFound. +func (s *sqlStore) GetChain(ctx context.Context, chainID string) (ChainState, error) { + if err := s.ensureSchema(ctx); err != nil { + return ChainState{}, err + } + row := s.db.QueryRowContext(ctx, s.rebind(`SELECT dispatch_id, queue_name, nodes_json, next_index, completed, failed, failure, created_at_ms, updated_at_ms + FROM bus_chains WHERE chain_id=?`), chainID) + var ( + dispatchID, queueName, failure string + nodesJSON []byte + nextIndex, completed, failed int + createdMS, updatedMS int64 + ) + if err := row.Scan(&dispatchID, &queueName, &nodesJSON, &nextIndex, &completed, &failed, &failure, &createdMS, &updatedMS); err != nil { + if err == sql.ErrNoRows { + return ChainState{}, ErrNotFound + } + return ChainState{}, err + } + var nodes []ChainNode + if err := json.Unmarshal(nodesJSON, &nodes); err != nil { + return ChainState{}, err + } + return ChainState{ + ChainID: chainID, + DispatchID: dispatchID, + Queue: queueName, + Nodes: nodes, + NextIndex: nextIndex, + Completed: completed == 1, + Failed: failed == 1, + Failure: failure, + CreatedAt: time.UnixMilli(createdMS), + UpdatedAt: time.UnixMilli(updatedMS), + }, nil +} + +// CreateBatch inserts aggregate and member rows in one transaction to prevent partial batches. +func (s *sqlStore) CreateBatch(ctx context.Context, rec BatchRecord) error { + if err := validateBatchRecord(rec); err != nil { + return err + } + if err := s.ensureSchema(ctx); err != nil { + return err + } + if err := s.validateMySQLKey("batch id", rec.BatchID, s.mysqlKeyLimit.batchID); err != nil { + return err + } + for _, job := range rec.Jobs { + if err := s.validateMySQLKey("batch job id", job.JobID, s.mysqlKeyLimit.batchJob); err != nil { + return err + } + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + + if rec.CreatedAt.IsZero() { + rec.CreatedAt = time.Now() + } + now := rec.CreatedAt.UnixMilli() + allow := 0 + if rec.AllowFailed { + allow = 1 + } + _, err = tx.ExecContext(ctx, s.rebind(`INSERT INTO bus_batches + (batch_id, dispatch_id, name, queue_name, allow_failed, total_jobs, pending_jobs, processed_jobs, failed_jobs, cancelled, completed, created_at_ms, updated_at_ms) + VALUES (?, ?, ?, ?, ?, ?, ?, 0, 0, 0, 0, ?, ?)`), + rec.BatchID, rec.DispatchID, rec.Name, rec.Queue, allow, len(rec.Jobs), len(rec.Jobs), now, now, + ) + if err != nil { + return err + } + for _, job := range rec.Jobs { + if _, err := tx.ExecContext(ctx, s.rebind(`INSERT INTO bus_batch_jobs (batch_id, job_id, started, done, failed) VALUES (?, ?, 0, 0, 0)`), rec.BatchID, job.JobID); err != nil { + return err + } + } + return tx.Commit() +} + +// MarkBatchJobStarted idempotently records that a member has begun without changing settlement counters. +func (s *sqlStore) MarkBatchJobStarted(ctx context.Context, batchID, jobID string) error { + if err := s.ensureSchema(ctx); err != nil { + return err + } + result, err := s.db.ExecContext(ctx, s.rebind(`UPDATE bus_batch_jobs SET started=1 WHERE batch_id=? AND job_id=?`), batchID, jobID) + if err != nil { + return err + } + updated, err := result.RowsAffected() + if err != nil { + return err + } + if updated > 0 { + return nil + } + // MySQL reports changed rows by default, so an already-started member and + // a missing member both return zero until existence is checked explicitly. + var exists int + err = s.db.QueryRowContext(ctx, s.rebind(`SELECT 1 FROM bus_batch_jobs WHERE batch_id=? AND job_id=?`), batchID, jobID).Scan(&exists) + if err == sql.ErrNoRows { + return ErrNotFound + } + return err +} + +// MarkBatchJobSucceeded delegates successful settlement to the shared transactional counter path. +func (s *sqlStore) MarkBatchJobSucceeded(ctx context.Context, batchID, jobID string) (BatchState, bool, error) { + state, done, _, _, _, _, err := s.markBatchTerminal(ctx, batchID, jobID, false, transitionClaim{}) + return state, done, err +} + +// MarkBatchJobFailed delegates failed settlement to the shared transactional counter path. +func (s *sqlStore) MarkBatchJobFailed(ctx context.Context, batchID, jobID string, _ error) (BatchState, bool, error) { + state, done, _, _, _, _, err := s.markBatchTerminal(ctx, batchID, jobID, true, transitionClaim{}) + return state, done, err +} + +// SettleBatchJob returns whether the requested outcome owns the member while +// preserving the established aggregate state returned by compatibility APIs. +func (s *sqlStore) SettleBatchJob(ctx context.Context, batchID, jobID string, outcome BatchJobOutcome, _ error) (BatchState, bool, error) { + result, err := s.settleBatchOutcome(ctx, batchID, jobID, outcome, nil, transitionClaim{}) + return result.state, result.owned, err +} + +// settleBatchOutcome returns both durable category ownership and the exact +// transaction's counter claim so recovery can keep aggregate ownership honest. +func (s *sqlStore) settleBatchOutcome(ctx context.Context, batchID, jobID string, outcome BatchJobOutcome, _ error, claim transitionClaim) (batchSettlementResult, error) { + switch outcome { + case BatchJobSucceeded: + state, _, owned, claimed, receipt, receiptKnown, err := s.markBatchTerminal(ctx, batchID, jobID, false, claim) + return batchSettlementResult{state: state, owned: owned, claimedNow: claimed, receipt: receipt, receiptKnown: receiptKnown}, err + case BatchJobFailed: + state, _, owned, claimed, receipt, receiptKnown, err := s.markBatchTerminal(ctx, batchID, jobID, true, claim) + return batchSettlementResult{state: state, owned: owned, claimedNow: claimed, receipt: receipt, receiptKnown: receiptKnown}, err + default: + return batchSettlementResult{}, fmt.Errorf("unsupported batch job outcome %q", outcome) + } +} + +// CancelBatch persists cancellation and completion together so readers cannot observe an intermediate state. +func (s *sqlStore) CancelBatch(ctx context.Context, batchID string) error { + if err := s.ensureSchema(ctx); err != nil { + return err + } + _, err := s.db.ExecContext(ctx, s.rebind(`UPDATE bus_batches SET cancelled=1, completed=1, updated_at_ms=? WHERE batch_id=?`), time.Now().UnixMilli(), batchID) + return err +} + +// GetBatch reconstructs aggregate state and normalizes missing rows to ErrNotFound. +func (s *sqlStore) GetBatch(ctx context.Context, batchID string) (BatchState, error) { + if err := s.ensureSchema(ctx); err != nil { + return BatchState{}, err + } + row := s.db.QueryRowContext(ctx, s.rebind(`SELECT dispatch_id, name, queue_name, allow_failed, total_jobs, pending_jobs, processed_jobs, failed_jobs, cancelled, completed, created_at_ms, updated_at_ms + FROM bus_batches WHERE batch_id=?`), batchID) + var ( + dispatchID, name, queueName string + allow, total, pending, processed, failed int + cancelled, completed int + createdMS, updatedMS int64 + ) + if err := row.Scan(&dispatchID, &name, &queueName, &allow, &total, &pending, &processed, &failed, &cancelled, &completed, &createdMS, &updatedMS); err != nil { + if err == sql.ErrNoRows { + return BatchState{}, ErrNotFound + } + return BatchState{}, err + } + return BatchState{ + BatchID: batchID, + DispatchID: dispatchID, + Name: name, + Queue: queueName, + AllowFailed: allow == 1, + Total: total, + Pending: pending, + Processed: processed, + Failed: failed, + Cancelled: cancelled == 1, + Completed: completed == 1, + CreatedAt: time.UnixMilli(createdMS), + UpdatedAt: time.UnixMilli(updatedMS), + }, nil +} + +// MarkCallbackInvoked uses dialect-specific conflict suppression to claim each callback key once. +func (s *sqlStore) MarkCallbackInvoked(ctx context.Context, key string) (bool, error) { + if err := s.ensureSchema(ctx); err != nil { + return false, err + } + if err := s.validateMySQLKey("callback key", key, s.mysqlKeyLimit.callback); err != nil { + return false, err + } + now := time.Now().UnixMilli() + switch s.driverName { + case "pgx", "postgres": + res, err := s.db.ExecContext(ctx, `INSERT INTO bus_callback_invocations (callback_key, created_at_ms) VALUES ($1, $2) ON CONFLICT (callback_key) DO NOTHING`, key, now) + if err != nil { + return false, err + } + n, _ := res.RowsAffected() + return n > 0, nil + case "mysql": + res, err := s.db.ExecContext(ctx, `INSERT IGNORE INTO bus_callback_invocations (callback_key, created_at_ms) VALUES (?, ?)`, key, now) + if err != nil { + return false, err + } + n, _ := res.RowsAffected() + return n > 0, nil + default: + res, err := s.db.ExecContext(ctx, `INSERT INTO bus_callback_invocations (callback_key, created_at_ms) VALUES (?, ?) ON CONFLICT(callback_key) DO NOTHING`, key, now) + if err != nil { + return false, err + } + n, _ := res.RowsAffected() + return n > 0, nil + } +} + +// Prune deletes dependent rows and terminal parents in one transaction to avoid orphaned state. +func (s *sqlStore) Prune(ctx context.Context, before time.Time) error { + if err := s.ensureSchema(ctx); err != nil { + return err + } + cutoff := before.UnixMilli() + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer func() { _ = tx.Rollback() }() + + if _, err := tx.ExecContext(ctx, s.rebind(`DELETE FROM bus_workflow_transition_receipts WHERE workflow_kind=? AND workflow_id IN ( + SELECT chain_id FROM bus_chains WHERE updated_at_ms < ? AND (completed=1 OR failed=1) + )`), chainTransitionKind, cutoff); err != nil { + return err + } + + // Remove chain node-idempotency rows for terminal chains before pruning chains. + if _, err := tx.ExecContext(ctx, s.rebind(`DELETE FROM bus_chain_completed_nodes WHERE chain_id IN ( + SELECT chain_id FROM bus_chains WHERE updated_at_ms < ? AND (completed=1 OR failed=1) + )`), cutoff); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, s.rebind(`DELETE FROM bus_chains WHERE updated_at_ms < ? AND (completed=1 OR failed=1)`), cutoff); err != nil { + return err + } + + if _, err := tx.ExecContext(ctx, s.rebind(`DELETE FROM bus_workflow_transition_receipts WHERE workflow_kind=? AND workflow_id IN ( + SELECT batch_id FROM bus_batches WHERE updated_at_ms < ? AND completed=1 + )`), batchTransitionKind, cutoff); err != nil { + return err + } + + // Remove per-job state for terminal batches before pruning batches. + if _, err := tx.ExecContext(ctx, s.rebind(`DELETE FROM bus_batch_jobs WHERE batch_id IN ( + SELECT batch_id FROM bus_batches WHERE updated_at_ms < ? AND completed=1 + )`), cutoff); err != nil { + return err + } + if _, err := tx.ExecContext(ctx, s.rebind(`DELETE FROM bus_batches WHERE updated_at_ms < ? AND completed=1`), cutoff); err != nil { + return err + } + + // Callback markers are safe to prune independently. + if _, err := tx.ExecContext(ctx, s.rebind(`DELETE FROM bus_callback_invocations WHERE created_at_ms < ?`), cutoff); err != nil { + return err + } + + return tx.Commit() +} + +// markBatchTerminal conditionally claims one member and updates aggregate +// counters arithmetically so concurrent settlements cannot overwrite state. +func (s *sqlStore) markBatchTerminal(ctx context.Context, batchID, jobID string, isFailure bool, claim transitionClaim) (BatchState, bool, bool, bool, transitionReceipt, bool, error) { + if err := s.ensureSchema(ctx); err != nil { + return BatchState{}, false, false, false, transitionReceipt{}, false, err + } + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return BatchState{}, false, false, false, transitionReceipt{}, false, err + } + defer func() { _ = tx.Rollback() }() + + failed := 0 + if isFailure { + failed = 1 + } + result, err := tx.ExecContext(ctx, s.rebind(`UPDATE bus_batch_jobs + SET done=1, failed=?, started=1 + WHERE batch_id=? AND job_id=? AND done=0`), failed, batchID, jobID) + if err != nil { + return BatchState{}, false, false, false, transitionReceipt{}, false, err + } + claimedRows, err := result.RowsAffected() + if err != nil { + return BatchState{}, false, false, false, transitionReceipt{}, false, err + } + claimed := claimedRows > 0 + owned := claimed + // Reading the parent after the member CAS avoids two SQLite deferred + // transactions deadlocking while both try to upgrade a shared read lock. + initialState, err := s.getBatchTx(ctx, tx, batchID) + if err != nil { + return BatchState{}, false, false, false, transitionReceipt{}, false, err + } + if !claimed { + var committedFailure int + row := tx.QueryRowContext(ctx, s.rebind(`SELECT failed FROM bus_batch_jobs WHERE batch_id=? AND job_id=?`), batchID, jobID) + if err := row.Scan(&committedFailure); err != nil { + if err == sql.ErrNoRows { + return BatchState{}, false, false, false, transitionReceipt{}, false, ErrNotFound + } + return BatchState{}, false, false, false, transitionReceipt{}, false, err + } + owned = (committedFailure == 1) == isFailure + } + if initialState.DispatchID != "" && claim.dispatchID != "" && initialState.DispatchID != claim.dispatchID { + if claimed { + return BatchState{}, false, false, false, transitionReceipt{}, false, fmt.Errorf("batch %q dispatch mismatch", batchID) + } + return initialState, initialState.Completed, false, false, transitionReceipt{}, false, nil + } + + now := time.Now().UnixMilli() + if claimed { + // MySQL evaluates assignments left-to-right, so completion must read + // the pre-settlement pending count before that count is decremented. + result, err = tx.ExecContext(ctx, s.rebind(`UPDATE bus_batches SET + cancelled=CASE WHEN ?=1 AND allow_failed=0 THEN 1 ELSE cancelled END, + completed=CASE WHEN pending_jobs <= 1 OR (?=1 AND allow_failed=0) THEN 1 ELSE completed END, + pending_jobs=CASE WHEN pending_jobs > 0 THEN pending_jobs-1 ELSE 0 END, + processed_jobs=processed_jobs+1, + failed_jobs=failed_jobs+?, + updated_at_ms=? + WHERE batch_id=?`), failed, failed, failed, now, batchID) + if err != nil { + return BatchState{}, false, false, false, transitionReceipt{}, false, err + } + updated, rowsErr := result.RowsAffected() + if rowsErr != nil { + return BatchState{}, false, false, false, transitionReceipt{}, false, rowsErr + } + if updated == 0 { + return BatchState{}, false, false, false, transitionReceipt{}, false, ErrNotFound + } + } else if _, err := tx.ExecContext(ctx, s.rebind(`UPDATE bus_batches SET updated_at_ms=? WHERE batch_id=?`), now, batchID); err != nil { + return BatchState{}, false, false, false, transitionReceipt{}, false, err + } + st, err := s.getBatchTx(ctx, tx, batchID) + if err != nil { + return BatchState{}, false, false, false, transitionReceipt{}, false, err + } + receipt, receiptKnown := transitionReceipt{}, false + if claimed { + outcome := BatchJobSucceeded + if isFailure { + outcome = BatchJobFailed + } + receipt, receiptKnown, err = s.insertTransitionReceipt(ctx, tx, transitionReceipt{ + workflowKind: batchTransitionKind, + workflowID: batchID, + workflowDispatchID: st.DispatchID, + workflowCreatedAt: st.CreatedAt, + memberID: jobID, + outcome: outcome, + owner: claim, + createdAt: time.UnixMilli(now), + }) + if err != nil { + return BatchState{}, false, false, false, transitionReceipt{}, false, err + } + if !initialState.Completed && st.Completed && receiptKnown { + aggregate, aggregateKnown, aggregateErr := s.insertTransitionReceipt(ctx, tx, transitionReceipt{ + workflowKind: batchTransitionKind, + workflowID: batchID, + workflowDispatchID: st.DispatchID, + workflowCreatedAt: st.CreatedAt, + memberID: "", + outcome: outcome, + owner: claim, + aggregateCompleted: true, + aggregateCancelled: st.Cancelled, + createdAt: time.UnixMilli(now), + }) + if aggregateErr != nil { + return BatchState{}, false, false, false, transitionReceipt{}, false, aggregateErr + } + if aggregateKnown && aggregate.owner == claim { + receipt.aggregateCompleted = true + receipt.aggregateCancelled = st.Cancelled + } + } + } else { + receipt, receiptKnown, err = s.getBatchTransitionReceipt(ctx, tx, batchID, jobID) + if err != nil { + return BatchState{}, false, false, false, transitionReceipt{}, false, err + } + } + if err := tx.Commit(); err != nil { + return s.readCommittedBatchSettlement(ctx, batchID, jobID, isFailure, claim, err) + } + return st, st.Completed, owned, claimed, receipt, receiptKnown, nil +} + +// getChainTx reads chain state through the caller's transaction so advancement uses one consistent view. +func (s *sqlStore) getChainTx(ctx context.Context, tx *sql.Tx, chainID string) (ChainState, error) { + row := tx.QueryRowContext(ctx, s.rebind(`SELECT dispatch_id, queue_name, nodes_json, next_index, completed, failed, failure, created_at_ms, updated_at_ms FROM bus_chains WHERE chain_id=?`), chainID) + var ( + dispatchID, queueName, failure string + nodesJSON []byte + nextIndex, completed, failed int + createdMS, updatedMS int64 + ) + if err := row.Scan(&dispatchID, &queueName, &nodesJSON, &nextIndex, &completed, &failed, &failure, &createdMS, &updatedMS); err != nil { + if err == sql.ErrNoRows { + return ChainState{}, ErrNotFound + } + return ChainState{}, err + } + var nodes []ChainNode + if err := json.Unmarshal(nodesJSON, &nodes); err != nil { + return ChainState{}, err + } + return ChainState{ + ChainID: chainID, + DispatchID: dispatchID, + Queue: queueName, + Nodes: nodes, + NextIndex: nextIndex, + Completed: completed == 1, + Failed: failed == 1, + Failure: failure, + CreatedAt: time.UnixMilli(createdMS), + UpdatedAt: time.UnixMilli(updatedMS), + }, nil +} + +// getBatchTx locks the aggregate row before reading its current state so only +// the member whose parent update crosses into completion can own the terminal +// receipt. SQLite already holds its writer lock after the member claim, while +// PostgreSQL and MySQL need an explicit current read rather than an MVCC +// snapshot that may predate a concurrent settlement. +func (s *sqlStore) getBatchTx(ctx context.Context, tx *sql.Tx, batchID string) (BatchState, error) { + query := `SELECT dispatch_id, name, queue_name, allow_failed, total_jobs, pending_jobs, processed_jobs, failed_jobs, cancelled, completed, created_at_ms, updated_at_ms FROM bus_batches WHERE batch_id=?` + if s.driverName == "mysql" || s.driverName == "pgx" || s.driverName == "postgres" { + query += ` FOR UPDATE` + } + row := tx.QueryRowContext(ctx, s.rebind(query), batchID) + var ( + dispatchID, name, queueName string + allow, total, pending, processed, failed int + cancelled, completed int + createdMS, updatedMS int64 + ) + if err := row.Scan(&dispatchID, &name, &queueName, &allow, &total, &pending, &processed, &failed, &cancelled, &completed, &createdMS, &updatedMS); err != nil { + if err == sql.ErrNoRows { + return BatchState{}, ErrNotFound + } + return BatchState{}, err + } + return BatchState{ + BatchID: batchID, + DispatchID: dispatchID, + Name: name, + Queue: queueName, + AllowFailed: allow == 1, + Total: total, + Pending: pending, + Processed: processed, + Failed: failed, + Cancelled: cancelled == 1, + Completed: completed == 1, + CreatedAt: time.UnixMilli(createdMS), + UpdatedAt: time.UnixMilli(updatedMS), + }, nil +} + +// insertChainCompletedNode uses dialect-specific conflict suppression to detect the first completion atomically. +func (s *sqlStore) insertChainCompletedNode(ctx context.Context, tx *sql.Tx, chainID, nodeID string) (bool, error) { + if err := s.validateMySQLKey("chain id", chainID, s.mysqlKeyLimit.chainID); err != nil { + return false, err + } + if err := s.validateMySQLKey("chain node id", nodeID, s.mysqlKeyLimit.chainNode); err != nil { + return false, err + } + now := time.Now().UnixMilli() + switch s.driverName { + case "pgx", "postgres": + res, err := tx.ExecContext(ctx, `INSERT INTO bus_chain_completed_nodes (chain_id, node_id, created_at_ms) VALUES ($1, $2, $3) ON CONFLICT (chain_id, node_id) DO NOTHING`, chainID, nodeID, now) + if err != nil { + return false, err + } + n, _ := res.RowsAffected() + return n > 0, nil + case "mysql": + res, err := tx.ExecContext(ctx, `INSERT IGNORE INTO bus_chain_completed_nodes (chain_id, node_id, created_at_ms) VALUES (?, ?, ?)`, chainID, nodeID, now) + if err != nil { + return false, err + } + n, _ := res.RowsAffected() + return n > 0, nil + default: + res, err := tx.ExecContext(ctx, `INSERT INTO bus_chain_completed_nodes (chain_id, node_id, created_at_ms) VALUES (?, ?, ?) ON CONFLICT(chain_id, node_id) DO NOTHING`, chainID, nodeID, now) + if err != nil { + return false, err + } + n, _ := res.RowsAffected() + return n > 0, nil + } +} + +// insertTransitionReceipt writes immutable owner identity in the same +// transaction as its workflow state mutation. +func (s *sqlStore) insertTransitionReceipt(ctx context.Context, tx *sql.Tx, receipt transitionReceipt) (transitionReceipt, bool, error) { + if !receipt.owner.valid() { + return transitionReceipt{}, false, nil + } + if receipt.version == 0 { + receipt.version = transitionReceiptVersion + } + if receipt.eventSchemaVersion == 0 { + receipt.eventSchemaVersion = eventSchemaVersion + } + if err := validateTransitionReceiptSupport(receipt); err != nil { + return transitionReceipt{}, false, err + } + if err := s.validateTransitionReceiptKeys(receipt); err != nil { + return transitionReceipt{}, false, err + } + if receipt.createdAt.IsZero() { + receipt.createdAt = time.Now() + } + if _, err := tx.ExecContext(ctx, s.rebind(`DELETE FROM bus_workflow_transition_receipts + WHERE workflow_kind=? AND workflow_id=? AND member_id=? + AND (workflow_dispatch_id<>? OR workflow_created_at_ms<>?)`), + receipt.workflowKind, + receipt.workflowID, + receipt.memberID, + receipt.workflowDispatchID, + receipt.workflowCreatedAt.UnixMilli(), + ); err != nil { + return transitionReceipt{}, false, err + } + completed := 0 + if receipt.aggregateCompleted { + completed = 1 + } + cancelled := 0 + if receipt.aggregateCancelled { + cancelled = 1 + } + query := `INSERT INTO bus_workflow_transition_receipts + (workflow_kind, receipt_version, event_schema_version, workflow_id, member_id, workflow_dispatch_id, workflow_created_at_ms, outcome, + owner_delivery_id, owner_attempt, job_dispatch_id, job_id, job_fingerprint, + aggregate_completed, aggregate_cancelled, created_at_ms) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(workflow_kind, workflow_id, member_id) DO NOTHING` + if s.driverName == "mysql" { + query = `INSERT INTO bus_workflow_transition_receipts + (workflow_kind, receipt_version, event_schema_version, workflow_id, member_id, workflow_dispatch_id, workflow_created_at_ms, outcome, + owner_delivery_id, owner_attempt, job_dispatch_id, job_id, job_fingerprint, + aggregate_completed, aggregate_cancelled, created_at_ms) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON DUPLICATE KEY UPDATE workflow_kind=workflow_kind` + } + _, err := tx.ExecContext(ctx, s.rebind(query), + receipt.workflowKind, + receipt.version, + receipt.eventSchemaVersion, + receipt.workflowID, + receipt.memberID, + receipt.workflowDispatchID, + receipt.workflowCreatedAt.UnixMilli(), + string(receipt.outcome), + receipt.owner.deliveryID, + receipt.owner.attempt, + receipt.owner.dispatchID, + receipt.owner.jobID, + receipt.owner.jobFingerprint, + completed, + cancelled, + receipt.createdAt.UnixMilli(), + ) + if err != nil { + return transitionReceipt{}, false, err + } + persisted, known, err := s.getTransitionReceipt(ctx, tx, receipt.workflowKind, receipt.workflowID, receipt.memberID) + if err != nil { + return transitionReceipt{}, false, err + } + if !known { + return transitionReceipt{}, false, errors.New("workflow transition receipt insert was not readable") + } + if !sameTransitionReceipt(receipt, persisted) { + return transitionReceipt{}, false, fmt.Errorf("workflow transition receipt for %s %q member %q conflicts with its persisted owner", receipt.workflowKind, receipt.workflowID, receipt.memberID) + } + return persisted, true, nil +} + +// sameTransitionReceipt compares every immutable ownership field while +// allowing storage to canonicalize the receipt timestamp to milliseconds. +func sameTransitionReceipt(want, persisted transitionReceipt) bool { + return want.version == persisted.version && + want.eventSchemaVersion == persisted.eventSchemaVersion && + want.workflowKind == persisted.workflowKind && + want.workflowID == persisted.workflowID && + want.workflowDispatchID == persisted.workflowDispatchID && + want.workflowCreatedAt.Equal(persisted.workflowCreatedAt) && + want.memberID == persisted.memberID && + want.outcome == persisted.outcome && + want.owner == persisted.owner && + want.aggregateCompleted == persisted.aggregateCompleted && + want.aggregateCancelled == persisted.aggregateCancelled +} + +// getTransitionReceipt reads one immutable receipt through either a database +// or the transaction currently mutating its parent workflow. +func (s *sqlStore) getTransitionReceipt(ctx context.Context, queryer transitionReceiptQueryer, kind, workflowID, memberID string) (transitionReceipt, bool, error) { + row := queryer.QueryRowContext(ctx, s.rebind(`SELECT receipt_version, event_schema_version, workflow_dispatch_id, workflow_created_at_ms, outcome, + owner_delivery_id, owner_attempt, job_dispatch_id, job_id, job_fingerprint, + aggregate_completed, aggregate_cancelled, created_at_ms + FROM bus_workflow_transition_receipts + WHERE workflow_kind=? AND workflow_id=? AND member_id=?`), kind, workflowID, memberID) + var ( + workflowDispatchID, outcome string + ownerDeliveryID, jobDispatchID, jobID, jobHash string + receiptVersion, storedEventSchemaVersion int64 + workflowCreatedMS, ownerAttempt, createdMS int64 + aggregateCompleted, aggregateCancelled int + ) + if err := row.Scan( + &receiptVersion, + &storedEventSchemaVersion, + &workflowDispatchID, + &workflowCreatedMS, + &outcome, + &ownerDeliveryID, + &ownerAttempt, + &jobDispatchID, + &jobID, + &jobHash, + &aggregateCompleted, + &aggregateCancelled, + &createdMS, + ); err != nil { + if err == sql.ErrNoRows { + return transitionReceipt{}, false, nil + } + return transitionReceipt{}, false, err + } + if receiptVersion != int64(transitionReceiptVersion) || storedEventSchemaVersion != int64(eventSchemaVersion) { + return transitionReceipt{}, false, fmt.Errorf("%w: receipt version %d, event schema %d", errUnsupportedTransitionReceipt, receiptVersion, storedEventSchemaVersion) + } + if ownerAttempt < 0 || int64(int(ownerAttempt)) != ownerAttempt { + return transitionReceipt{}, false, fmt.Errorf("workflow transition receipt has invalid attempt %d", ownerAttempt) + } + if aggregateCompleted < 0 || aggregateCompleted > 1 || aggregateCancelled < 0 || aggregateCancelled > 1 { + return transitionReceipt{}, false, fmt.Errorf("workflow transition receipt has invalid aggregate flags completed=%d cancelled=%d", aggregateCompleted, aggregateCancelled) + } + if aggregateCancelled == 1 && aggregateCompleted == 0 { + return transitionReceipt{}, false, errors.New("workflow transition receipt cancellation is not completed") + } + receiptOutcome := BatchJobOutcome(outcome) + if receiptOutcome != BatchJobSucceeded && receiptOutcome != BatchJobFailed { + return transitionReceipt{}, false, fmt.Errorf("workflow transition receipt has invalid outcome %q", outcome) + } + owner := transitionClaim{ + deliveryID: ownerDeliveryID, + attempt: int(ownerAttempt), + dispatchID: jobDispatchID, + jobID: jobID, + jobFingerprint: jobHash, + } + if !owner.valid() { + return transitionReceipt{}, false, errors.New("workflow transition receipt has incomplete owner identity") + } + return transitionReceipt{ + version: int(receiptVersion), + eventSchemaVersion: int(storedEventSchemaVersion), + workflowKind: kind, + workflowID: workflowID, + workflowDispatchID: workflowDispatchID, + workflowCreatedAt: time.UnixMilli(workflowCreatedMS), + memberID: memberID, + outcome: receiptOutcome, + owner: owner, + aggregateCompleted: aggregateCompleted == 1, + aggregateCancelled: aggregateCancelled == 1, + createdAt: time.UnixMilli(createdMS), + }, true, nil +} + +// chainTransitionReceipt distinguishes corrupt cross-incarnation provenance +// from a genuinely absent receipt so recovery always fails closed. +func (s *sqlStore) chainTransitionReceipt(ctx context.Context, chainID, nodeID string) (transitionReceipt, bool, error) { + if err := s.ensureSchema(ctx); err != nil { + return transitionReceipt{}, false, err + } + state, err := s.GetChain(ctx, chainID) + if err != nil { + return transitionReceipt{}, false, err + } + receipt, known, err := s.getTransitionReceipt(ctx, s.db, chainTransitionKind, chainID, nodeID) + if err != nil || !known { + return receipt, known, err + } + if receipt.workflowDispatchID != state.DispatchID || !receipt.workflowCreatedAt.Equal(state.CreatedAt) { + return transitionReceipt{}, false, fmt.Errorf("chain %q transition receipt does not match current workflow incarnation", chainID) + } + return receipt, true, nil +} + +// batchTransitionReceipt returns member and aggregate writer identity only for +// the current durable batch incarnation. +func (s *sqlStore) batchTransitionReceipt(ctx context.Context, batchID, jobID string) (transitionReceipt, bool, error) { + if err := s.ensureSchema(ctx); err != nil { + return transitionReceipt{}, false, err + } + state, err := s.GetBatch(ctx, batchID) + if err != nil { + return transitionReceipt{}, false, err + } + receipt, known, err := s.getBatchTransitionReceipt(ctx, s.db, batchID, jobID) + if err != nil || !known { + return receipt, known, err + } + if receipt.workflowDispatchID != state.DispatchID || !receipt.workflowCreatedAt.Equal(state.CreatedAt) { + return transitionReceipt{}, false, fmt.Errorf("batch %q transition receipt does not match current workflow incarnation", batchID) + } + return receipt, true, nil +} + +// getBatchTransitionReceipt merges aggregate ownership into the immutable +// member receipt so first claims, replays, and recovery expose one store-neutral +// result even though SQL keeps the aggregate owner in a separate row. +func (s *sqlStore) getBatchTransitionReceipt(ctx context.Context, queryer transitionReceiptQueryer, batchID, jobID string) (transitionReceipt, bool, error) { + receipt, known, err := s.getTransitionReceipt(ctx, queryer, batchTransitionKind, batchID, jobID) + if err != nil { + return receipt, known, err + } + aggregate, aggregateKnown, err := s.getTransitionReceipt(ctx, queryer, batchTransitionKind, batchID, "") + if err != nil { + return transitionReceipt{}, false, err + } + if aggregateKnown && !aggregate.aggregateCompleted { + return transitionReceipt{}, false, fmt.Errorf("batch %q aggregate transition receipt does not own completion", batchID) + } + if aggregateKnown && aggregate.aggregateCancelled && aggregate.outcome != BatchJobFailed { + return transitionReceipt{}, false, fmt.Errorf("batch %q aggregate transition receipt cancellation does not own failure", batchID) + } + if aggregateKnown { + if err := s.validateBatchAggregateReceiptOwner(ctx, queryer, aggregate); err != nil { + return transitionReceipt{}, false, err + } + } + if !known { + return receipt, false, nil + } + if aggregateKnown && (aggregate.workflowDispatchID != receipt.workflowDispatchID || !aggregate.workflowCreatedAt.Equal(receipt.workflowCreatedAt)) { + return transitionReceipt{}, false, fmt.Errorf("batch %q aggregate transition receipt does not match member workflow incarnation", batchID) + } + if aggregateKnown && aggregate.owner == receipt.owner { + receipt.aggregateCompleted = aggregate.aggregateCompleted + receipt.aggregateCancelled = aggregate.aggregateCancelled + } + return receipt, true, nil +} + +// validateBatchAggregateReceiptOwner proves the separate aggregate row maps to +// exactly one member receipt written by the same physical settlement claim. +func (s *sqlStore) validateBatchAggregateReceiptOwner(ctx context.Context, queryer transitionReceiptQueryer, aggregate transitionReceipt) error { + row := queryer.QueryRowContext(ctx, s.rebind(`SELECT COUNT(*), COALESCE(MAX(outcome), '') + FROM bus_workflow_transition_receipts + WHERE workflow_kind=? AND workflow_id=? AND member_id<>? + AND workflow_dispatch_id=? AND workflow_created_at_ms=? + AND owner_delivery_id=? AND owner_attempt=? AND job_dispatch_id=? AND job_id=? AND job_fingerprint=?`), + aggregate.workflowKind, + aggregate.workflowID, + "", + aggregate.workflowDispatchID, + aggregate.workflowCreatedAt.UnixMilli(), + aggregate.owner.deliveryID, + aggregate.owner.attempt, + aggregate.owner.dispatchID, + aggregate.owner.jobID, + aggregate.owner.jobFingerprint, + ) + var ( + matches int + outcome string + ) + if err := row.Scan(&matches, &outcome); err != nil { + return err + } + if matches != 1 { + return fmt.Errorf("batch %q aggregate transition receipt owner matches %d member receipts, want exactly one", aggregate.workflowID, matches) + } + if BatchJobOutcome(outcome) != aggregate.outcome { + return fmt.Errorf("batch %q aggregate transition receipt does not match member outcome", aggregate.workflowID) + } + return nil +} + +// readCommittedChainAdvance resolves an ambiguous commit response only when +// the durable receipt names the same physical generation. +func (s *sqlStore) readCommittedChainAdvance(ctx context.Context, chainID, nodeID string, claim transitionClaim, commitErr error) (chainAdvanceResult, error) { + if !claim.valid() { + return chainAdvanceResult{}, commitErr + } + receipt, known, err := s.chainTransitionReceipt(ctx, chainID, nodeID) + if err != nil || !known || receipt.owner != claim || receipt.outcome != BatchJobSucceeded { + return chainAdvanceResult{}, commitErr + } + state, err := s.GetChain(ctx, chainID) + if err != nil { + return chainAdvanceResult{}, commitErr + } + if receipt.workflowDispatchID != state.DispatchID || !receipt.workflowCreatedAt.Equal(state.CreatedAt) { + return chainAdvanceResult{}, commitErr + } + successOwned, err := chainNodeSuccessDisposition(state, nodeID) + if err != nil || !successOwned { + return chainAdvanceResult{}, commitErr + } + next, done, _, err := chainNodeAdvanceDisposition(state, nodeID) + if err != nil { + return chainAdvanceResult{}, commitErr + } + return chainAdvanceResult{state: state, next: next, done: done, successOwned: true, claimedNow: true, receipt: receipt, receiptKnown: true}, nil +} + +// readCommittedChainFailure resolves an ambiguous commit response only when +// terminal state and its immutable failed receipt name the same generation. +func (s *sqlStore) readCommittedChainFailure(ctx context.Context, chainID, nodeID string, claim transitionClaim, commitErr error) (chainFailureResult, error) { + if !claim.valid() { + return chainFailureResult{}, commitErr + } + receipt, known, err := s.chainTransitionReceipt(ctx, chainID, nodeID) + if err != nil || !known || receipt.owner != claim || receipt.outcome != BatchJobFailed || receipt.aggregateCompleted || receipt.aggregateCancelled { + return chainFailureResult{}, commitErr + } + state, err := s.GetChain(ctx, chainID) + if err != nil { + return chainFailureResult{}, commitErr + } + if receipt.workflowDispatchID != state.DispatchID || !receipt.workflowCreatedAt.Equal(state.CreatedAt) { + return chainFailureResult{}, commitErr + } + owned, _, err := chainNodeFailureDisposition(state, nodeID) + if err != nil || !owned || !state.Failed || state.Completed { + return chainFailureResult{}, commitErr + } + return chainFailureResult{state: state, owned: true, claimedNow: true, receipt: receipt, receiptKnown: true}, nil +} + +// readCommittedBatchSettlement resolves an ambiguous commit response only when +// the durable member receipt names the same physical generation and outcome. +// The reloaded state may include a later member's completion, so callers must +// use the receipt's aggregate flags when attributing terminal effects. +func (s *sqlStore) readCommittedBatchSettlement(ctx context.Context, batchID, jobID string, isFailure bool, claim transitionClaim, commitErr error) (BatchState, bool, bool, bool, transitionReceipt, bool, error) { + if !claim.valid() { + return BatchState{}, false, false, false, transitionReceipt{}, false, commitErr + } + receipt, known, err := s.batchTransitionReceipt(ctx, batchID, jobID) + wantOutcome := BatchJobSucceeded + if isFailure { + wantOutcome = BatchJobFailed + } + if err != nil || !known || receipt.owner != claim || receipt.outcome != wantOutcome { + return BatchState{}, false, false, false, transitionReceipt{}, false, commitErr + } + state, err := s.GetBatch(ctx, batchID) + if err != nil { + return BatchState{}, false, false, false, transitionReceipt{}, false, commitErr + } + return state, state.Completed, true, true, receipt, true, nil +} + +// rebind converts portable question-mark placeholders to PostgreSQL positional parameters when required. +func (s *sqlStore) rebind(query string) string { + if s.driverName != "pgx" && s.driverName != "postgres" { + return query + } + var b strings.Builder + b.Grow(len(query) + 8) + i := 1 + for _, r := range query { + if r == '?' { + b.WriteString(fmt.Sprintf("$%d", i)) + i++ + continue + } + b.WriteRune(r) + } + return b.String() +} diff --git a/internal/workflow/store_sql_test.go b/internal/workflow/store_sql_test.go new file mode 100644 index 0000000..576a572 --- /dev/null +++ b/internal/workflow/store_sql_test.go @@ -0,0 +1,1097 @@ +package workflow + +import ( + "context" + "database/sql" + "errors" + "path/filepath" + "strings" + "testing" + "time" + + _ "modernc.org/sqlite" +) + +func newSQLiteStore(t *testing.T) Store { + t.Helper() + dsn := filepath.Join(t.TempDir(), "bus-store.db") + store, err := NewSQLStore(SQLStoreConfig{ + DriverName: "sqlite", + DSN: dsn, + }) + if err != nil { + t.Fatalf("new sql store: %v", err) + } + t.Cleanup(func() { _ = store.(*sqlStore).db.Close() }) + return store +} + +func TestSQLStoreChainAdvanceIdempotent(t *testing.T) { + s := newSQLiteStore(t) + ctx := context.Background() + chainID := "chain-1" + if err := s.CreateChain(ctx, ChainRecord{ + ChainID: chainID, + DispatchID: "d1", + Queue: "default", + Nodes: []ChainNode{ + {NodeID: "n1", Job: StoredJob{Type: "a"}}, + {NodeID: "n2", Job: StoredJob{Type: "b"}}, + }, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create chain: %v", err) + } + + next, done, err := s.AdvanceChain(ctx, chainID, "n1") + if err != nil { + t.Fatalf("advance chain first: %v", err) + } + if done { + t.Fatal("expected not done after first advance") + } + if next == nil || next.NodeID != "n2" { + t.Fatalf("expected next n2, got %+v", next) + } + + // duplicate completion should not double-advance + next, done, err = s.AdvanceChain(ctx, chainID, "n1") + if err != nil { + t.Fatalf("advance chain duplicate: %v", err) + } + if done { + t.Fatal("expected not done after duplicate completion") + } + if next == nil || next.NodeID != "n2" { + t.Fatalf("expected next n2 on duplicate, got %+v", next) + } + + next, done, err = s.AdvanceChain(ctx, chainID, "n2") + if err != nil { + t.Fatalf("advance chain final: %v", err) + } + if !done || next != nil { + t.Fatalf("expected done with nil next, got done=%v next=%+v", done, next) + } +} + +func TestSQLStoreBatchLifecycle(t *testing.T) { + s := newSQLiteStore(t) + ctx := context.Background() + batchID := "batch-1" + + if err := s.CreateBatch(ctx, BatchRecord{ + BatchID: batchID, + DispatchID: "d1", + Name: "monitor sweep", + Queue: "default", + AllowFailed: false, + Jobs: []BatchJob{ + {JobID: "j1", Job: StoredJob{Type: "monitor:poll"}}, + {JobID: "j2", Job: StoredJob{Type: "monitor:downsample"}}, + }, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create batch: %v", err) + } + + if err := s.MarkBatchJobStarted(ctx, batchID, "j1"); err != nil { + t.Fatalf("mark started: %v", err) + } + st, done, err := s.MarkBatchJobSucceeded(ctx, batchID, "j1") + if err != nil { + t.Fatalf("mark succeeded: %v", err) + } + if done { + t.Fatal("expected not done after first success") + } + if st.Processed != 1 || st.Pending != 1 || st.Failed != 0 { + t.Fatalf("unexpected state after success: %+v", st) + } + + st, done, err = s.MarkBatchJobFailed(ctx, batchID, "j2", nil) + if err != nil { + t.Fatalf("mark failed: %v", err) + } + if !done { + t.Fatal("expected done after failure when allow_failed=false") + } + if !st.Completed || !st.Cancelled || st.Failed != 1 { + t.Fatalf("unexpected terminal state: %+v", st) + } +} + +func TestSQLStoreCallbackMarkerIdempotent(t *testing.T) { + s := newSQLiteStore(t) + ctx := context.Background() + key := "chain_finally:chain-1" + + first, err := s.MarkCallbackInvoked(ctx, key) + if err != nil { + t.Fatalf("mark callback first: %v", err) + } + if !first { + t.Fatal("expected first callback marker insert to be true") + } + + second, err := s.MarkCallbackInvoked(ctx, key) + if err != nil { + t.Fatalf("mark callback second: %v", err) + } + if second { + t.Fatal("expected duplicate callback marker insert to be false") + } +} + +// TestSQLStoreSchemaInitializationRetriesAfterTransientFailure proves one +// canceled first use cannot poison an otherwise healthy store instance. +func TestSQLStoreSchemaInitializationRetriesAfterTransientFailure(t *testing.T) { + s := newSQLiteStore(t) + canceled, cancel := context.WithCancel(context.Background()) + cancel() + + if _, err := s.MarkCallbackInvoked(canceled, "first-attempt"); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled schema initialization error = %v, want context canceled", err) + } + + inserted, err := s.MarkCallbackInvoked(context.Background(), "retry-attempt") + if err != nil { + t.Fatalf("retry schema initialization: %v", err) + } + if !inserted { + t.Fatal("retry schema initialization did not persist callback marker") + } +} + +func TestSQLStorePruneRemovesOldTerminalState(t *testing.T) { + s := newSQLiteStore(t) + ctx := context.Background() + old := time.Now().Add(-2 * time.Hour) + cutoff := time.Now().Add(1 * time.Minute) + + if err := s.CreateChain(ctx, ChainRecord{ + ChainID: "chain-old-done", + DispatchID: "d1", + Queue: "default", + Nodes: []ChainNode{{NodeID: "n1", Job: StoredJob{Type: "monitor:poll"}}}, + CreatedAt: old, + }); err != nil { + t.Fatalf("create chain old done: %v", err) + } + if _, _, err := s.AdvanceChain(ctx, "chain-old-done", "n1"); err != nil { + t.Fatalf("advance old chain: %v", err) + } + + if err := s.CreateBatch(ctx, BatchRecord{ + BatchID: "batch-old-done", + DispatchID: "d2", + Name: "old-batch", + Queue: "default", + AllowFailed: true, + Jobs: []BatchJob{{JobID: "j1", Job: StoredJob{Type: "monitor:poll"}}}, + CreatedAt: old, + }); err != nil { + t.Fatalf("create batch old done: %v", err) + } + if _, _, err := s.MarkBatchJobSucceeded(ctx, "batch-old-done", "j1"); err != nil { + t.Fatalf("mark old batch done: %v", err) + } + + if _, err := s.MarkCallbackInvoked(ctx, "batch_then:batch-old-done"); err != nil { + t.Fatalf("mark callback marker: %v", err) + } + + if err := s.CreateChain(ctx, ChainRecord{ + ChainID: "chain-active", + DispatchID: "d3", + Queue: "default", + Nodes: []ChainNode{ + {NodeID: "n1", Job: StoredJob{Type: "monitor:poll"}}, + {NodeID: "n2", Job: StoredJob{Type: "monitor:alert"}}, + }, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create chain active: %v", err) + } + + if err := s.Prune(ctx, cutoff); err != nil { + t.Fatalf("prune: %v", err) + } + + if _, err := s.GetChain(ctx, "chain-old-done"); !errors.Is(err, ErrNotFound) { + t.Fatalf("expected old chain pruned, got err=%v", err) + } + if _, err := s.GetBatch(ctx, "batch-old-done"); !errors.Is(err, ErrNotFound) { + t.Fatalf("expected old batch pruned, got err=%v", err) + } + if _, err := s.GetChain(ctx, "chain-active"); err != nil { + t.Fatalf("expected active chain retained, got err=%v", err) + } +} + +// TestSQLStorePruneRemovesTransitionReceiptsBeforeIdentifierReuse proves +// retention cannot let an old physical owner claim a new workflow incarnation. +func TestSQLStorePruneRemovesTransitionReceiptsBeforeIdentifierReuse(t *testing.T) { + store := newSQLiteStore(t).(*sqlStore) + ctx := context.Background() + const ( + chainID = "reused-chain" + nodeID = "reused-node" + batchID = "reused-batch" + jobID = "reused-job" + ) + oldChainClaim := transitionClaim{deliveryID: "old-chain-owner", attempt: 0, dispatchID: "old-chain-dispatch", jobID: "old-chain-job", jobFingerprint: "old-chain-fingerprint"} + if err := store.CreateChain(ctx, ChainRecord{ + ChainID: chainID, + DispatchID: oldChainClaim.dispatchID, + Nodes: []ChainNode{{NodeID: nodeID}}, + CreatedAt: time.Now().Add(-2 * time.Hour), + }); err != nil { + t.Fatalf("create old chain: %v", err) + } + oldChain, err := store.advanceChainOutcome(ctx, chainID, nodeID, oldChainClaim) + if err != nil || !oldChain.done || !oldChain.receiptKnown { + t.Fatalf("complete old chain = %+v err:%v", oldChain, err) + } + + oldBatchClaim := transitionClaim{deliveryID: "old-batch-owner", attempt: 0, dispatchID: "old-batch-dispatch", jobID: jobID, jobFingerprint: "old-batch-fingerprint"} + if err := store.CreateBatch(ctx, BatchRecord{ + BatchID: batchID, + DispatchID: oldBatchClaim.dispatchID, + Jobs: []BatchJob{{JobID: jobID}}, + CreatedAt: time.Now().Add(-2 * time.Hour), + }); err != nil { + t.Fatalf("create old batch: %v", err) + } + oldBatch, err := store.settleBatchOutcome(ctx, batchID, jobID, BatchJobSucceeded, nil, oldBatchClaim) + if err != nil || !oldBatch.state.Completed || !oldBatch.receiptKnown || !oldBatch.receipt.aggregateCompleted { + t.Fatalf("complete old batch = %+v err:%v", oldBatch, err) + } + oldBatchReplay, err := store.settleBatchOutcome(ctx, batchID, jobID, BatchJobSucceeded, nil, transitionClaim{ + deliveryID: "old-batch-replay", + attempt: 1, + dispatchID: oldBatchClaim.dispatchID, + jobID: oldBatchClaim.jobID, + jobFingerprint: oldBatchClaim.jobFingerprint, + }) + if err != nil || oldBatchReplay.claimedNow || !oldBatchReplay.receiptKnown || !oldBatchReplay.receipt.aggregateCompleted || oldBatchReplay.receipt.owner != oldBatchClaim { + t.Fatalf("replay old terminal batch = %+v err:%v", oldBatchReplay, err) + } + + var receiptCount int + if err := store.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM bus_workflow_transition_receipts`).Scan(&receiptCount); err != nil { + t.Fatalf("count old receipts: %v", err) + } + if receiptCount != 3 { + t.Fatalf("old receipt count = %d, want chain, batch member, and batch aggregate", receiptCount) + } + if err := store.Prune(ctx, time.Now().Add(time.Minute)); err != nil { + t.Fatalf("prune old workflows: %v", err) + } + if err := store.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM bus_workflow_transition_receipts`).Scan(&receiptCount); err != nil { + t.Fatalf("count pruned receipts: %v", err) + } + if receiptCount != 0 { + t.Fatalf("pruned receipt count = %d, want 0", receiptCount) + } + + newChainClaim := transitionClaim{deliveryID: "new-chain-owner", attempt: 1, dispatchID: "new-chain-dispatch", jobID: "new-chain-job", jobFingerprint: "new-chain-fingerprint"} + if err := store.CreateChain(ctx, ChainRecord{ChainID: chainID, DispatchID: newChainClaim.dispatchID, Nodes: []ChainNode{{NodeID: nodeID}}}); err != nil { + t.Fatalf("recreate chain: %v", err) + } + newChain, err := store.advanceChainOutcome(ctx, chainID, nodeID, newChainClaim) + if err != nil || !newChain.receiptKnown || newChain.receipt.owner != newChainClaim || newChain.receipt.workflowDispatchID != newChainClaim.dispatchID { + t.Fatalf("complete recreated chain = %+v err:%v", newChain, err) + } + + newBatchClaim := transitionClaim{deliveryID: "new-batch-owner", attempt: 1, dispatchID: "new-batch-dispatch", jobID: jobID, jobFingerprint: "new-batch-fingerprint"} + if err := store.CreateBatch(ctx, BatchRecord{BatchID: batchID, DispatchID: newBatchClaim.dispatchID, Jobs: []BatchJob{{JobID: jobID}}}); err != nil { + t.Fatalf("recreate batch: %v", err) + } + newBatch, err := store.settleBatchOutcome(ctx, batchID, jobID, BatchJobSucceeded, nil, newBatchClaim) + if err != nil || !newBatch.receiptKnown || newBatch.receipt.owner != newBatchClaim || newBatch.receipt.workflowDispatchID != newBatchClaim.dispatchID || !newBatch.receipt.aggregateCompleted { + t.Fatalf("complete recreated batch = %+v err:%v", newBatch, err) + } +} + +func TestNewSQLStoreValidationAndDefaults(t *testing.T) { + if _, err := NewSQLStore(SQLStoreConfig{}); err == nil || !strings.Contains(err.Error(), "driver name is required") { + t.Fatalf("expected driver validation error, got %v", err) + } + if _, err := NewSQLStore(SQLStoreConfig{DriverName: "sqlite"}); err == nil || !strings.Contains(err.Error(), "dsn is required") { + t.Fatalf("expected dsn validation error, got %v", err) + } + + db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "defaults.db")) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + defer db.Close() + + store, err := NewSQLStore(SQLStoreConfig{DB: db}) + if err != nil { + t.Fatalf("new sql store with db: %v", err) + } + ss, ok := store.(*sqlStore) + if !ok { + t.Fatalf("expected *sqlStore, got %T", store) + } + if ss.driverName != "sqlite" { + t.Fatalf("expected default driver sqlite, got %q", ss.driverName) + } + if !ss.autoMigrate { + t.Fatal("expected autoMigrate default true") + } +} + +// TestNewSQLStoreWithManagedSchemaSkipsDDL keeps default migration behavior +// intact while giving externally provisioned deployments an explicit opt-out. +func TestNewSQLStoreWithManagedSchemaSkipsDDL(t *testing.T) { + ctx := context.Background() + db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "managed-schema.db")) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + + store, err := NewSQLStoreWithManagedSchema(SQLStoreConfig{DB: db, DriverName: "sqlite"}) + if err != nil { + t.Fatalf("new managed-schema store: %v", err) + } + managed := store.(*sqlStore) + if managed.autoMigrate { + t.Fatal("managed-schema constructor enabled migrations") + } + if _, err := managed.GetChain(ctx, "missing"); err == nil || errors.Is(err, ErrNotFound) { + t.Fatalf("unprovisioned managed schema error = %v", err) + } + var tableCount int + if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name LIKE 'bus_%'`).Scan(&tableCount); err != nil { + t.Fatalf("count managed tables: %v", err) + } + if tableCount != 0 { + t.Fatalf("managed-schema constructor created %d tables", tableCount) + } + + for _, statement := range managed.schemaStatements() { + if _, err := db.ExecContext(ctx, managed.rebind(statement)); err != nil { + t.Fatalf("provision managed schema: %v", err) + } + } + if err := managed.CreateChain(ctx, ChainRecord{ChainID: "managed-chain", Nodes: []ChainNode{{NodeID: "managed-node"}}}); err != nil { + t.Fatalf("use provisioned managed schema: %v", err) + } +} + +func TestSQLStoreFailChainAndCancelBatch(t *testing.T) { + s := newSQLiteStore(t) + ctx := context.Background() + + if err := s.CreateChain(ctx, ChainRecord{ + ChainID: "chain-fail", + DispatchID: "d-fail", + Queue: "default", + Nodes: []ChainNode{{NodeID: "n1", Job: StoredJob{Type: "monitor:poll"}}}, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create chain: %v", err) + } + if err := s.FailChain(ctx, "chain-fail", errors.New("boom")); err != nil { + t.Fatalf("fail chain: %v", err) + } + st, err := s.GetChain(ctx, "chain-fail") + if err != nil { + t.Fatalf("get chain: %v", err) + } + if !st.Failed || st.Failure != "boom" { + t.Fatalf("expected failed chain with boom, got %+v", st) + } + + if err := s.CreateBatch(ctx, BatchRecord{ + BatchID: "batch-cancel", + DispatchID: "d-cancel", + Name: "cancel-me", + Queue: "default", + AllowFailed: true, + Jobs: []BatchJob{{JobID: "j1", Job: StoredJob{Type: "monitor:poll"}}}, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create batch: %v", err) + } + if err := s.CancelBatch(ctx, "batch-cancel"); err != nil { + t.Fatalf("cancel batch: %v", err) + } + bs, err := s.GetBatch(ctx, "batch-cancel") + if err != nil { + t.Fatalf("get batch: %v", err) + } + if !bs.Cancelled || !bs.Completed { + t.Fatalf("expected cancelled completed batch, got %+v", bs) + } +} + +func TestSQLStoreBatchTerminalIdempotentAndNotFound(t *testing.T) { + s := newSQLiteStore(t) + ctx := context.Background() + + if _, _, err := s.MarkBatchJobSucceeded(ctx, "missing-batch", "missing-job"); !errors.Is(err, ErrNotFound) { + t.Fatalf("expected ErrNotFound for missing batch job, got %v", err) + } + + if err := s.CreateBatch(ctx, BatchRecord{ + BatchID: "batch-idem", + DispatchID: "d-idem", + Name: "idem", + Queue: "default", + AllowFailed: true, + Jobs: []BatchJob{{JobID: "j1", Job: StoredJob{Type: "monitor:poll"}}}, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create batch: %v", err) + } + + st1, done1, err := s.MarkBatchJobSucceeded(ctx, "batch-idem", "j1") + if err != nil { + t.Fatalf("first mark succeeded: %v", err) + } + if !done1 || st1.Processed != 1 || st1.Pending != 0 { + t.Fatalf("unexpected first terminal state: done=%v state=%+v", done1, st1) + } + + st2, done2, err := s.MarkBatchJobSucceeded(ctx, "batch-idem", "j1") + if err != nil { + t.Fatalf("second mark succeeded: %v", err) + } + if !done2 || st2.Processed != 1 || st2.Pending != 0 { + t.Fatalf("expected idempotent terminal state, got done=%v state=%+v", done2, st2) + } +} + +// TestSQLStoreChainClaimRollsBackWithParentFailure proves a failed parent +// mutation cannot leave the completed-node claim committed on its own. +func TestSQLStoreChainClaimRollsBackWithParentFailure(t *testing.T) { + store := newSQLiteStore(t).(*sqlStore) + ctx := context.Background() + const chainID = "chain-parent-update-rollback" + if err := store.CreateChain(ctx, ChainRecord{ + ChainID: chainID, + DispatchID: "dispatch-parent-update-rollback", + Nodes: []ChainNode{ + {NodeID: "node-first", Job: StoredJob{Type: "reports:first"}}, + {NodeID: "node-second", Job: StoredJob{Type: "reports:second"}}, + }, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create chain: %v", err) + } + const trigger = `CREATE TRIGGER reject_chain_parent_update +BEFORE UPDATE OF next_index ON bus_chains +BEGIN + SELECT RAISE(ABORT, 'forced chain parent update failure'); +END` + if _, err := store.db.ExecContext(ctx, trigger); err != nil { + t.Fatalf("create chain failure trigger: %v", err) + } + if _, _, err := store.AdvanceChain(ctx, chainID, "node-first"); err == nil || !strings.Contains(err.Error(), "forced chain parent update failure") { + t.Fatalf("advance with parent failure error = %v", err) + } + if _, err := store.db.ExecContext(ctx, `DROP TRIGGER reject_chain_parent_update`); err != nil { + t.Fatalf("drop chain failure trigger: %v", err) + } + next, done, err := store.AdvanceChain(ctx, chainID, "node-first") + if err != nil { + t.Fatalf("retry chain advance: %v", err) + } + if done || next == nil || next.NodeID != "node-second" { + t.Fatalf("retry chain advance = next:%+v done:%t, want second node", next, done) + } +} + +// TestSQLStoreChainCompletionRollsBackWithTerminalFlagFailure proves the node +// claim and index increment roll back when the final completion update fails. +func TestSQLStoreChainCompletionRollsBackWithTerminalFlagFailure(t *testing.T) { + store := newSQLiteStore(t).(*sqlStore) + ctx := context.Background() + const chainID = "chain-completion-update-rollback" + if err := store.CreateChain(ctx, ChainRecord{ + ChainID: chainID, + Nodes: []ChainNode{{NodeID: "node-final", Job: StoredJob{Type: "reports:final"}}}, + }); err != nil { + t.Fatalf("create chain: %v", err) + } + const trigger = `CREATE TRIGGER reject_chain_completion_update +BEFORE UPDATE OF completed ON bus_chains +WHEN NEW.completed=1 +BEGIN + SELECT RAISE(ABORT, 'forced chain completion update failure'); +END` + if _, err := store.db.ExecContext(ctx, trigger); err != nil { + t.Fatalf("create chain completion trigger: %v", err) + } + if _, _, err := store.AdvanceChain(ctx, chainID, "node-final"); err == nil || !strings.Contains(err.Error(), "forced chain completion update failure") { + t.Fatalf("advance with completion failure error = %v", err) + } + state, err := store.GetChain(ctx, chainID) + if err != nil { + t.Fatalf("get rolled-back chain: %v", err) + } + if state.NextIndex != 0 || state.Completed || state.Failed { + t.Fatalf("completion failure left partial chain state: %+v", state) + } + var claims int + if err := store.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM bus_chain_completed_nodes WHERE chain_id=?`, chainID).Scan(&claims); err != nil { + t.Fatalf("count rolled-back node claims: %v", err) + } + if claims != 0 { + t.Fatalf("rolled-back node claims = %d, want 0", claims) + } + if _, err := store.db.ExecContext(ctx, `DROP TRIGGER reject_chain_completion_update`); err != nil { + t.Fatalf("drop chain completion trigger: %v", err) + } + next, done, err := store.AdvanceChain(ctx, chainID, "node-final") + if err != nil || !done || next != nil { + t.Fatalf("retry chain completion = next:%+v done:%t err:%v", next, done, err) + } +} + +// TestSQLStoreChainFailureClaimRetriesAfterParentFailure proves the node remains +// claimable when the atomic failure compare-and-swap is rejected by storage. +func TestSQLStoreChainFailureClaimRetriesAfterParentFailure(t *testing.T) { + store := newSQLiteStore(t).(*sqlStore) + ctx := context.Background() + const chainID = "chain-failure-parent-rollback" + if err := store.CreateChain(ctx, ChainRecord{ChainID: chainID, Nodes: []ChainNode{{NodeID: "node-first"}, {NodeID: "node-second"}}}); err != nil { + t.Fatalf("create chain: %v", err) + } + const trigger = `CREATE TRIGGER reject_chain_failure_update +BEFORE UPDATE OF failed ON bus_chains +BEGIN + SELECT RAISE(ABORT, 'forced chain failure update failure'); +END` + if _, err := store.db.ExecContext(ctx, trigger); err != nil { + t.Fatalf("create chain failure trigger: %v", err) + } + if _, _, err := store.FailChainNode(ctx, chainID, "node-first", errors.New("application failed")); err == nil || !strings.Contains(err.Error(), "forced chain failure update failure") { + t.Fatalf("fail node with parent failure error = %v", err) + } + if _, err := store.db.ExecContext(ctx, `DROP TRIGGER reject_chain_failure_update`); err != nil { + t.Fatalf("drop chain failure trigger: %v", err) + } + state, owned, err := store.FailChainNode(ctx, chainID, "node-first", errors.New("application failed")) + if err != nil || !owned || !state.Failed || state.NextIndex != 0 { + t.Fatalf("retry chain failure = state:%+v owned:%t err:%v", state, owned, err) + } +} + +// TestSQLStoreChainFailureReceiptRollsBackParent proves a receipt insert fault +// cannot leave terminal parent state without the provenance needed for recovery. +func TestSQLStoreChainFailureReceiptRollsBackParent(t *testing.T) { + store := newSQLiteStore(t).(*sqlStore) + ctx := context.Background() + const ( + chainID = "chain-failure-receipt-rollback" + nodeID = "node-failure-receipt-rollback" + dispatchID = "dispatch-failure-receipt-rollback" + ) + if err := store.CreateChain(ctx, ChainRecord{ChainID: chainID, DispatchID: dispatchID, Nodes: []ChainNode{{NodeID: nodeID}}}); err != nil { + t.Fatalf("create chain: %v", err) + } + const trigger = `CREATE TRIGGER reject_chain_failure_receipt +BEFORE INSERT ON bus_workflow_transition_receipts +WHEN NEW.workflow_kind='chain' AND NEW.outcome='failed' +BEGIN + SELECT RAISE(ABORT, 'forced chain failure receipt insert failure'); +END` + if _, err := store.db.ExecContext(ctx, trigger); err != nil { + t.Fatalf("create failure receipt trigger: %v", err) + } + claim := transitionClaim{deliveryID: "generation-failure-receipt-rollback", attempt: 1, dispatchID: dispatchID, jobID: "job-failure-receipt-rollback", jobFingerprint: "fingerprint-failure-receipt-rollback"} + if _, err := store.failChainOutcome(ctx, chainID, nodeID, errors.New("application failed"), claim); err == nil || !strings.Contains(err.Error(), "forced chain failure receipt insert failure") { + t.Fatalf("failure receipt insert error = %v", err) + } + state, err := store.GetChain(ctx, chainID) + if err != nil || state.Failed || state.Completed || state.NextIndex != 0 || state.Failure != "" { + t.Fatalf("chain after receipt rollback = %+v err:%v", state, err) + } + if receipt, known, err := store.chainTransitionReceipt(ctx, chainID, nodeID); err != nil || known { + t.Fatalf("rolled-back failure receipt = known:%t receipt:%+v err:%v", known, receipt, err) + } + if _, err := store.db.ExecContext(ctx, `DROP TRIGGER reject_chain_failure_receipt`); err != nil { + t.Fatalf("drop failure receipt trigger: %v", err) + } + result, err := store.failChainOutcome(ctx, chainID, nodeID, errors.New("application failed"), claim) + if err != nil || !result.claimedNow || !result.owned || !result.receiptKnown || !result.state.Failed || result.receipt.owner != claim || result.receipt.outcome != BatchJobFailed { + t.Fatalf("retry failed chain receipt = %+v err:%v", result, err) + } +} + +// TestSQLStoreBatchAggregateIncarnationMismatchFailsClosed proves a stale +// aggregate receipt cannot be silently omitted from terminal ownership. +func TestSQLStoreBatchAggregateIncarnationMismatchFailsClosed(t *testing.T) { + store := newSQLiteStore(t).(*sqlStore) + ctx := context.Background() + const ( + batchID = "batch-aggregate-incarnation-mismatch" + jobID = "job-aggregate-incarnation-mismatch" + dispatchID = "dispatch-aggregate-incarnation-mismatch" + ) + if err := store.CreateBatch(ctx, BatchRecord{BatchID: batchID, DispatchID: dispatchID, Jobs: []BatchJob{{JobID: jobID}}}); err != nil { + t.Fatalf("create batch: %v", err) + } + claim := transitionClaim{deliveryID: "generation-aggregate-incarnation", attempt: 0, dispatchID: dispatchID, jobID: jobID, jobFingerprint: "fingerprint-aggregate-incarnation"} + if result, err := store.settleBatchOutcome(ctx, batchID, jobID, BatchJobSucceeded, nil, claim); err != nil || !result.receiptKnown || !result.receipt.aggregateCompleted { + t.Fatalf("settle terminal batch = %+v err:%v", result, err) + } + if _, err := store.db.ExecContext(ctx, `UPDATE bus_workflow_transition_receipts SET workflow_dispatch_id='dispatch-corrupt-aggregate' WHERE workflow_kind=? AND workflow_id=? AND member_id=''`, batchTransitionKind, batchID); err != nil { + t.Fatalf("corrupt aggregate receipt incarnation: %v", err) + } + if receipt, known, err := store.batchTransitionReceipt(ctx, batchID, jobID); err == nil || known || !strings.Contains(err.Error(), "aggregate transition receipt") { + t.Fatalf("mismatched aggregate receipt = known:%t receipt:%+v err:%v", known, receipt, err) + } +} + +// TestSQLStoreBatchAggregateOwnershipMismatchFailsClosed proves the terminal +// row cannot silently detach from the member transaction that created it. +func TestSQLStoreBatchAggregateOwnershipMismatchFailsClosed(t *testing.T) { + tests := []struct { + name string + diagnostic string + mutate string + }{ + {name: "different owner", diagnostic: "exactly one", mutate: `UPDATE bus_workflow_transition_receipts SET owner_attempt=owner_attempt+1 WHERE workflow_kind='batch' AND workflow_id=? AND member_id=''`}, + {name: "different outcome", diagnostic: "member outcome", mutate: `UPDATE bus_workflow_transition_receipts SET outcome='failed' WHERE workflow_kind='batch' AND workflow_id=? AND member_id=''`}, + {name: "missing completion", diagnostic: "does not own completion", mutate: `UPDATE bus_workflow_transition_receipts SET aggregate_completed=0 WHERE workflow_kind='batch' AND workflow_id=? AND member_id=''`}, + {name: "successful cancellation", diagnostic: "does not own failure", mutate: `UPDATE bus_workflow_transition_receipts SET aggregate_cancelled=1 WHERE workflow_kind='batch' AND workflow_id=? AND member_id=''`}, + {name: "missing member", diagnostic: "exactly one", mutate: `DELETE FROM bus_workflow_transition_receipts WHERE workflow_kind='batch' AND workflow_id=? AND member_id<>''`}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + store := newSQLiteStore(t).(*sqlStore) + ctx := context.Background() + const ( + batchID = "batch-aggregate-owner-mismatch" + dispatchID = "dispatch-aggregate-owner-mismatch" + jobID = "job-aggregate-owner-mismatch" + ) + claim := transitionClaim{deliveryID: "generation-aggregate-owner-mismatch", attempt: 2, dispatchID: dispatchID, jobID: jobID, jobFingerprint: "fingerprint-aggregate-owner-mismatch"} + if err := store.CreateBatch(ctx, BatchRecord{BatchID: batchID, DispatchID: dispatchID, Jobs: []BatchJob{{JobID: jobID}}}); err != nil { + t.Fatalf("create batch: %v", err) + } + if settled, err := store.settleBatchOutcome(ctx, batchID, jobID, BatchJobSucceeded, nil, claim); err != nil || !settled.receiptKnown || !settled.receipt.aggregateCompleted { + t.Fatalf("settle batch = %+v err:%v", settled, err) + } + if _, err := store.db.ExecContext(ctx, test.mutate, batchID); err != nil { + t.Fatalf("corrupt aggregate receipt: %v", err) + } + if receipt, known, err := store.batchTransitionReceipt(ctx, batchID, jobID); err == nil || known || !strings.Contains(err.Error(), test.diagnostic) { + t.Fatalf("corrupt aggregate receipt = known:%t receipt:%+v err:%v", known, receipt, err) + } + }) + } +} + +// TestSQLStoreBatchClaimRollsBackWithParentFailure proves a failed aggregate +// update cannot consume the member claim needed by a later retry. +func TestSQLStoreBatchClaimRollsBackWithParentFailure(t *testing.T) { + store := newSQLiteStore(t).(*sqlStore) + ctx := context.Background() + const batchID = "batch-parent-update-rollback" + if err := store.CreateBatch(ctx, BatchRecord{ + BatchID: batchID, + DispatchID: "dispatch-parent-update-rollback", + Jobs: []BatchJob{{JobID: "job-first", Job: StoredJob{Type: "reports:first"}}}, + CreatedAt: time.Now(), + }); err != nil { + t.Fatalf("create batch: %v", err) + } + const trigger = `CREATE TRIGGER reject_batch_parent_update +BEFORE UPDATE OF pending_jobs ON bus_batches +BEGIN + SELECT RAISE(ABORT, 'forced batch parent update failure'); +END` + if _, err := store.db.ExecContext(ctx, trigger); err != nil { + t.Fatalf("create batch failure trigger: %v", err) + } + if _, _, err := store.MarkBatchJobSucceeded(ctx, batchID, "job-first"); err == nil || !strings.Contains(err.Error(), "forced batch parent update failure") { + t.Fatalf("settle with parent failure error = %v", err) + } + if _, err := store.db.ExecContext(ctx, `DROP TRIGGER reject_batch_parent_update`); err != nil { + t.Fatalf("drop batch failure trigger: %v", err) + } + state, done, err := store.MarkBatchJobSucceeded(ctx, batchID, "job-first") + if err != nil { + t.Fatalf("retry batch settlement: %v", err) + } + if !done || !state.Completed || state.Pending != 0 || state.Processed != 1 { + t.Fatalf("retry batch state = %+v done:%t, want one terminal settlement", state, done) + } +} + +// TestSQLStoreReceiptFailureRollsBackWorkflowMutation proves transition state +// and provenance remain one atomic write, including terminal aggregate ownership. +func TestSQLStoreReceiptFailureRollsBackWorkflowMutation(t *testing.T) { + t.Run("chain member", func(t *testing.T) { + store := newSQLiteStore(t).(*sqlStore) + ctx := context.Background() + claim := transitionClaim{deliveryID: "chain-owner", attempt: 0, dispatchID: "chain-dispatch", jobID: "chain-job", jobFingerprint: "chain-fingerprint"} + if err := store.CreateChain(ctx, ChainRecord{ChainID: "chain-receipt-rollback", DispatchID: claim.dispatchID, Nodes: []ChainNode{{NodeID: "node-first"}, {NodeID: "node-final"}}}); err != nil { + t.Fatalf("create chain: %v", err) + } + const trigger = `CREATE TRIGGER reject_chain_receipt +BEFORE INSERT ON bus_workflow_transition_receipts +BEGIN + SELECT RAISE(ABORT, 'forced chain receipt failure'); +END` + if _, err := store.db.ExecContext(ctx, trigger); err != nil { + t.Fatalf("create chain receipt trigger: %v", err) + } + if _, err := store.advanceChainOutcome(ctx, "chain-receipt-rollback", "node-first", claim); err == nil || !strings.Contains(err.Error(), "forced chain receipt failure") { + t.Fatalf("advance with receipt failure error = %v", err) + } + state, err := store.GetChain(ctx, "chain-receipt-rollback") + if err != nil { + t.Fatalf("get rolled-back chain: %v", err) + } + if state.NextIndex != 0 || state.Completed || state.Failed { + t.Fatalf("receipt failure left partial chain state: %+v", state) + } + var completedNodes, receipts int + if err := store.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM bus_chain_completed_nodes WHERE chain_id=?`, state.ChainID).Scan(&completedNodes); err != nil { + t.Fatalf("count completed-node claims: %v", err) + } + if err := store.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM bus_workflow_transition_receipts WHERE workflow_kind=? AND workflow_id=?`, chainTransitionKind, state.ChainID).Scan(&receipts); err != nil { + t.Fatalf("count chain receipts: %v", err) + } + if completedNodes != 0 || receipts != 0 { + t.Fatalf("rolled-back chain rows = completed:%d receipts:%d", completedNodes, receipts) + } + }) + + t.Run("batch aggregate", func(t *testing.T) { + store := newSQLiteStore(t).(*sqlStore) + ctx := context.Background() + claim := transitionClaim{deliveryID: "batch-owner", attempt: 0, dispatchID: "batch-dispatch", jobID: "batch-job", jobFingerprint: "batch-fingerprint"} + if err := store.CreateBatch(ctx, BatchRecord{BatchID: "batch-receipt-rollback", DispatchID: claim.dispatchID, Jobs: []BatchJob{{JobID: claim.jobID}}}); err != nil { + t.Fatalf("create batch: %v", err) + } + const trigger = `CREATE TRIGGER reject_batch_aggregate_receipt +BEFORE INSERT ON bus_workflow_transition_receipts +WHEN NEW.workflow_kind='batch' AND NEW.member_id='' +BEGIN + SELECT RAISE(ABORT, 'forced batch aggregate receipt failure'); +END` + if _, err := store.db.ExecContext(ctx, trigger); err != nil { + t.Fatalf("create batch receipt trigger: %v", err) + } + if _, err := store.settleBatchOutcome(ctx, "batch-receipt-rollback", claim.jobID, BatchJobSucceeded, nil, claim); err == nil || !strings.Contains(err.Error(), "forced batch aggregate receipt failure") { + t.Fatalf("settle with aggregate receipt failure error = %v", err) + } + state, err := store.GetBatch(ctx, "batch-receipt-rollback") + if err != nil { + t.Fatalf("get rolled-back batch: %v", err) + } + if state.Pending != 1 || state.Processed != 0 || state.Completed || state.Cancelled { + t.Fatalf("receipt failure left partial batch state: %+v", state) + } + var done, receipts int + if err := store.db.QueryRowContext(ctx, `SELECT done FROM bus_batch_jobs WHERE batch_id=? AND job_id=?`, state.BatchID, claim.jobID).Scan(&done); err != nil { + t.Fatalf("read rolled-back batch member: %v", err) + } + if err := store.db.QueryRowContext(ctx, `SELECT COUNT(*) FROM bus_workflow_transition_receipts WHERE workflow_kind=? AND workflow_id=?`, batchTransitionKind, state.BatchID).Scan(&receipts); err != nil { + t.Fatalf("count batch receipts: %v", err) + } + if done != 0 || receipts != 0 { + t.Fatalf("rolled-back batch rows = done:%d receipts:%d", done, receipts) + } + }) +} + +// TestSQLStoreConflictingReceiptCannotAdoptTransition protects immutable owner +// identity when an inconsistent pre-existing receipt is encountered. +func TestSQLStoreConflictingReceiptCannotAdoptTransition(t *testing.T) { + store := newSQLiteStore(t).(*sqlStore) + ctx := context.Background() + const ( + chainID = "chain-conflicting-receipt" + nodeID = "node-conflicting-receipt" + ) + if err := store.CreateChain(ctx, ChainRecord{ChainID: chainID, DispatchID: "chain-dispatch", Nodes: []ChainNode{{NodeID: nodeID}, {NodeID: "node-final"}}}); err != nil { + t.Fatalf("create chain: %v", err) + } + state, err := store.GetChain(ctx, chainID) + if err != nil { + t.Fatalf("get chain: %v", err) + } + originalOwner := transitionClaim{deliveryID: "original-owner", attempt: 0, dispatchID: state.DispatchID, jobID: "chain-job", jobFingerprint: "chain-fingerprint"} + tx, err := store.db.BeginTx(ctx, nil) + if err != nil { + t.Fatalf("begin orphan receipt: %v", err) + } + persisted, known, err := store.insertTransitionReceipt(ctx, tx, transitionReceipt{ + workflowKind: chainTransitionKind, + workflowID: chainID, + workflowDispatchID: state.DispatchID, + workflowCreatedAt: state.CreatedAt, + memberID: nodeID, + outcome: BatchJobSucceeded, + owner: originalOwner, + createdAt: time.Now(), + }) + if err != nil || !known { + _ = tx.Rollback() + t.Fatalf("seed conflicting receipt = %+v known:%t err:%v", persisted, known, err) + } + if err := tx.Commit(); err != nil { + t.Fatalf("commit conflicting receipt: %v", err) + } + if persisted.createdAt.Nanosecond()%int(time.Millisecond) != 0 { + t.Fatalf("persisted receipt timestamp was not canonicalized: %v", persisted.createdAt) + } + + newOwner := transitionClaim{deliveryID: "new-owner", attempt: 1, dispatchID: state.DispatchID, jobID: originalOwner.jobID, jobFingerprint: originalOwner.jobFingerprint} + if _, err := store.advanceChainOutcome(ctx, chainID, nodeID, newOwner); err == nil || !strings.Contains(err.Error(), "conflicts with its persisted owner") { + t.Fatalf("advance over conflicting receipt error = %v", err) + } + state, err = store.GetChain(ctx, chainID) + if err != nil { + t.Fatalf("get chain after conflict: %v", err) + } + if state.NextIndex != 0 || state.Completed { + t.Fatalf("conflicting receipt advanced chain: %+v", state) + } + receipt, known, err := store.chainTransitionReceipt(ctx, chainID, nodeID) + if err != nil || !known || receipt.owner != originalOwner { + t.Fatalf("immutable receipt after conflict = %+v known:%t err:%v", receipt, known, err) + } +} + +func TestSQLStoreRebindForPostgres(t *testing.T) { + s := &sqlStore{driverName: "postgres"} + got := s.rebind("SELECT * FROM t WHERE a=? AND b=?") + if got != "SELECT * FROM t WHERE a=$1 AND b=$2" { + t.Fatalf("unexpected rebind result: %q", got) + } +} + +// TestSQLStoreSchemaStatementsUseDialectTypes pins the key and payload types +// required for each supported database to accept the shared legacy schema. +func TestSQLStoreSchemaStatementsUseDialectTypes(t *testing.T) { + tests := []struct { + name string + driverName string + want []string + reject []string + }{ + { + name: "sqlite", + driverName: "sqlite", + want: []string{ + "chain_id TEXT PRIMARY KEY", + "nodes_json BLOB NOT NULL", + "bus_workflow_transition_receipts", + "workflow_kind TEXT NOT NULL", + "owner_delivery_id TEXT NOT NULL", + "PRIMARY KEY (workflow_kind, workflow_id, member_id)", + }, + }, + { + name: "mysql", + driverName: "mysql", + want: []string{ + "chain_id VARBINARY(255) PRIMARY KEY", + "nodes_json LONGBLOB NOT NULL", + "callback_key VARBINARY(512) PRIMARY KEY", + "workflow_kind VARBINARY(16) NOT NULL", + "workflow_id VARBINARY(255) NOT NULL", + "member_id VARBINARY(255) NOT NULL", + "PRIMARY KEY (workflow_kind, workflow_id, member_id)", + }, + reject: []string{"chain_id TEXT PRIMARY KEY"}, + }, + { + name: "postgres", + driverName: "pgx", + want: []string{ + "chain_id TEXT PRIMARY KEY", + "nodes_json BYTEA NOT NULL", + "workflow_kind TEXT NOT NULL", + "PRIMARY KEY (workflow_kind, workflow_id, member_id)", + }, + reject: []string{"nodes_json BLOB NOT NULL"}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + schema := strings.Join((&sqlStore{driverName: test.driverName}).schemaStatements(), "\n") + for _, fragment := range test.want { + if !strings.Contains(schema, fragment) { + t.Fatalf("schema missing %q:\n%s", fragment, schema) + } + } + for _, fragment := range test.reject { + if strings.Contains(schema, fragment) { + t.Fatalf("schema unexpectedly contains %q:\n%s", fragment, schema) + } + } + }) + } +} + +// TestSQLStoreMySQLKeyValidation uses the connected column's character and +// byte capacity rather than imposing the generated schema's default globally. +func TestSQLStoreMySQLKeyValidation(t *testing.T) { + store := &sqlStore{driverName: "mysql"} + capacity := mysqlColumnCapacity{characters: 2, bytes: 4} + if err := store.validateMySQLKey("job id", "éé", capacity); err != nil { + t.Fatalf("exact-capacity identifier: %v", err) + } + if err := store.validateMySQLKey("job id", "aaa", mysqlColumnCapacity{characters: 2, bytes: 100}); err == nil || !strings.Contains(err.Error(), "2 characters") { + t.Fatalf("character-limit error = %v", err) + } + if err := store.validateMySQLKey("job id", "€€", mysqlColumnCapacity{characters: 100, bytes: 4}); err == nil || !strings.Contains(err.Error(), "4 bytes") { + t.Fatalf("byte-limit error = %v", err) + } + if err := (&sqlStore{driverName: "postgres"}).validateMySQLKey("job id", strings.Repeat("a", 100), mysqlColumnCapacity{}); err != nil { + t.Fatalf("PostgreSQL key inherited MySQL capacity: %v", err) + } + + store.mysqlKeyLimit = mysqlWorkflowKeyLimits{ + chainID: mysqlColumnCapacity{characters: 2, bytes: 2}, + chainNode: mysqlColumnCapacity{characters: 3, bytes: 3}, + batchID: mysqlColumnCapacity{characters: 4, bytes: 4}, + batchJob: mysqlColumnCapacity{characters: 5, bytes: 5}, + } + if err := store.validateTransitionReceiptKeys(transitionReceipt{workflowKind: batchTransitionKind, workflowID: "bbbb", memberID: "jjjjj"}); err != nil { + t.Fatalf("batch receipt inherited chain limits: %v", err) + } + if err := store.validateTransitionReceiptKeys(transitionReceipt{workflowKind: chainTransitionKind, workflowID: "ccc", memberID: "nnn"}); err == nil || !strings.Contains(err.Error(), "chain receipt id") { + t.Fatalf("chain receipt capacity error = %v", err) + } + if err := store.validateTransitionReceiptKeys(transitionReceipt{workflowKind: "unknown", workflowID: "id", memberID: "member"}); err == nil || !strings.Contains(err.Error(), "unsupported workflow transition receipt kind") { + t.Fatalf("unknown receipt-kind error = %v", err) + } +} + +// TestMySQLWorkflowKeyLimitsFromColumns pins capacity intersection for logical +// IDs shared by parent and child tables and rejects incomplete managed schemas. +func TestMySQLWorkflowKeyLimitsFromColumns(t *testing.T) { + columns := map[string]mysqlColumnCapacity{ + "bus_chains.chain_id": {dataType: "varbinary", characters: 512, bytes: 512}, + "bus_chain_completed_nodes.chain_id": {dataType: "varbinary", characters: 300, bytes: 300}, + "bus_chain_completed_nodes.node_id": {dataType: "varbinary", characters: 400, bytes: 400}, + "bus_batches.batch_id": {dataType: "varbinary", characters: 600, bytes: 600}, + "bus_batch_jobs.batch_id": {dataType: "varbinary", characters: 350, bytes: 350}, + "bus_batch_jobs.job_id": {dataType: "varbinary", characters: 450, bytes: 450}, + "bus_callback_invocations.callback_key": {dataType: "varbinary", characters: 1024, bytes: 1024}, + "bus_workflow_transition_receipts.workflow_id": {dataType: "varbinary", characters: 700, bytes: 700}, + "bus_workflow_transition_receipts.member_id": {dataType: "varbinary", characters: 700, bytes: 700}, + } + limits, err := mysqlWorkflowKeyLimitsFromColumns(columns) + if err != nil { + t.Fatalf("derive key limits: %v", err) + } + if limits.chainID.bytes != 300 || limits.chainNode.bytes != 400 || limits.batchID.bytes != 350 || limits.batchJob.bytes != 450 || limits.callback.bytes != 1024 { + t.Fatalf("derived key limits = %+v", limits) + } + columns["bus_workflow_transition_receipts.workflow_id"] = mysqlColumnCapacity{dataType: "varbinary", characters: 325, bytes: 325} + columns["bus_workflow_transition_receipts.member_id"] = mysqlColumnCapacity{dataType: "varbinary", characters: 425, bytes: 425} + limits, err = mysqlWorkflowKeyLimitsFromColumns(columns) + if err != nil { + t.Fatalf("derive narrowed receipt limits: %v", err) + } + if limits.chainID.bytes != 300 || limits.chainNode.bytes != 400 || limits.batchID.bytes != 325 || limits.batchJob.bytes != 425 { + t.Fatalf("receipt-intersected key limits = %+v", limits) + } + columns["bus_workflow_transition_receipts.workflow_id"] = mysqlColumnCapacity{dataType: "varbinary", characters: 700, bytes: 700} + columns["bus_workflow_transition_receipts.member_id"] = mysqlColumnCapacity{dataType: "varbinary", characters: 700, bytes: 700} + for _, dataType := range []string{"varchar", "text", "binary"} { + columns["bus_chain_completed_nodes.node_id"] = mysqlColumnCapacity{dataType: dataType, characters: 400, bytes: 1600} + if _, err := mysqlWorkflowKeyLimitsFromColumns(columns); err == nil || !strings.Contains(err.Error(), "bus_chain_completed_nodes.node_id must use VARBINARY") { + t.Fatalf("%s identity-column error = %v", dataType, err) + } + } + columns["bus_chain_completed_nodes.node_id"] = mysqlColumnCapacity{dataType: "varbinary", characters: 400, bytes: 400} + delete(columns, "bus_batch_jobs.job_id") + if _, err := mysqlWorkflowKeyLimitsFromColumns(columns); err == nil || !strings.Contains(err.Error(), "bus_batch_jobs.job_id") { + t.Fatalf("missing-column error = %v", err) + } +} + +// TestMySQLTransitionReceiptWidthsFromColumns pins effective legacy-table +// intersections before the two workflow models expand into one shared receipt. +func TestMySQLTransitionReceiptWidthsFromColumns(t *testing.T) { + columns := map[string]mysqlColumnCapacity{ + "bus_chains.chain_id": {dataType: "varbinary", characters: 512, bytes: 512}, + "bus_chain_completed_nodes.chain_id": {dataType: "varbinary", characters: 300, bytes: 300}, + "bus_chain_completed_nodes.node_id": {dataType: "varbinary", characters: 400, bytes: 400}, + "bus_batches.batch_id": {dataType: "varbinary", characters: 600, bytes: 600}, + "bus_batch_jobs.batch_id": {dataType: "varbinary", characters: 350, bytes: 350}, + "bus_batch_jobs.job_id": {dataType: "varbinary", characters: 450, bytes: 450}, + "bus_callback_invocations.callback_key": {dataType: "varbinary", characters: 1024, bytes: 1024}, + } + widths, err := mysqlTransitionReceiptWidthsFromColumns(columns) + if err != nil { + t.Fatalf("derive transition receipt widths: %v", err) + } + if widths.workflowID != 350 || widths.memberID != 450 { + t.Fatalf("derived receipt widths = %+v, want workflow:350 member:450", widths) + } + + columns["bus_chains.chain_id"] = mysqlColumnCapacity{dataType: "varbinary", characters: 700, bytes: 700} + columns["bus_chain_completed_nodes.chain_id"] = mysqlColumnCapacity{dataType: "varbinary", characters: 650, bytes: 650} + columns["bus_chain_completed_nodes.node_id"] = mysqlColumnCapacity{dataType: "varbinary", characters: 425, bytes: 425} + columns["bus_batches.batch_id"] = mysqlColumnCapacity{dataType: "varbinary", characters: 375, bytes: 375} + columns["bus_batch_jobs.batch_id"] = mysqlColumnCapacity{dataType: "varbinary", characters: 325, bytes: 325} + columns["bus_batch_jobs.job_id"] = mysqlColumnCapacity{dataType: "varbinary", characters: 725, bytes: 725} + widths, err = mysqlTransitionReceiptWidthsFromColumns(columns) + if err != nil { + t.Fatalf("derive asymmetric transition receipt widths: %v", err) + } + if widths.workflowID != 650 || widths.memberID != 725 { + t.Fatalf("asymmetric receipt widths = %+v, want workflow:650 member:725", widths) + } +} + +// TestMySQLTransitionReceiptWidthsRejectUnsafeBaseSchema proves automatic +// receipt creation fails before depending on missing or conflating key columns. +func TestMySQLTransitionReceiptWidthsRejectUnsafeBaseSchema(t *testing.T) { + columns := map[string]mysqlColumnCapacity{ + "bus_chains.chain_id": {dataType: "varbinary", characters: 512, bytes: 512}, + "bus_chain_completed_nodes.chain_id": {dataType: "varbinary", characters: 512, bytes: 512}, + "bus_chain_completed_nodes.node_id": {dataType: "varbinary", characters: 512, bytes: 512}, + "bus_batches.batch_id": {dataType: "varbinary", characters: 512, bytes: 512}, + "bus_batch_jobs.batch_id": {dataType: "varbinary", characters: 512, bytes: 512}, + "bus_batch_jobs.job_id": {dataType: "varbinary", characters: 512, bytes: 512}, + "bus_callback_invocations.callback_key": {dataType: "varbinary", characters: 1024, bytes: 1024}, + } + delete(columns, "bus_batch_jobs.job_id") + if _, err := mysqlTransitionReceiptWidthsFromColumns(columns); err == nil || !strings.Contains(err.Error(), "bus_batch_jobs.job_id") { + t.Fatalf("missing base-column error = %v", err) + } + columns["bus_batch_jobs.job_id"] = mysqlColumnCapacity{dataType: "varbinary", characters: 512, bytes: 512} + columns["bus_callback_invocations.callback_key"] = mysqlColumnCapacity{dataType: "varchar", characters: 1024, bytes: 4096} + if _, err := mysqlTransitionReceiptWidthsFromColumns(columns); err == nil || !strings.Contains(err.Error(), "bus_callback_invocations.callback_key must use VARBINARY") { + t.Fatalf("unsafe base-column error = %v", err) + } +} + +// TestSQLStoreMySQLTransitionReceiptSchemaUsesDerivedWidths proves generated +// DDL can preserve different workflow and member capacities without an ALTER. +func TestSQLStoreMySQLTransitionReceiptSchemaUsesDerivedWidths(t *testing.T) { + store := &sqlStore{driverName: "mysql"} + statement := store.transitionReceiptSchemaStatement("VARBINARY(16)", "VARBINARY(650)", "VARBINARY(725)") + for _, fragment := range []string{ + "CREATE TABLE IF NOT EXISTS bus_workflow_transition_receipts", + "workflow_kind VARBINARY(16) NOT NULL", + "workflow_id VARBINARY(650) NOT NULL", + "member_id VARBINARY(725) NOT NULL", + "PRIMARY KEY (workflow_kind, workflow_id, member_id)", + } { + if !strings.Contains(statement, fragment) { + t.Fatalf("derived receipt schema missing %q:\n%s", fragment, statement) + } + } + if strings.Contains(statement, "workflow_id VARBINARY(725)") || strings.Contains(statement, "member_id VARBINARY(650)") { + t.Fatalf("derived receipt schema conflated asymmetric widths:\n%s", statement) + } +} diff --git a/internal/workflow/test_runtime_adapter_test.go b/internal/workflow/test_runtime_adapter_test.go new file mode 100644 index 0000000..b840ccc --- /dev/null +++ b/internal/workflow/test_runtime_adapter_test.go @@ -0,0 +1,127 @@ +package workflow + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + + "github.com/goforj/queue/busruntime" +) + +type testInboundJob struct { + payload []byte +} + +func (j testInboundJob) Bind(dst any) error { + return json.Unmarshal(j.payload, dst) +} + +func (j testInboundJob) PayloadBytes() []byte { + return append([]byte(nil), j.payload...) +} + +type syncTestRuntime struct { + handlers map[string]busruntime.Handler + dispatchErr error +} + +// directTestRuntime adds canonical direct dispatch to the retained legacy test runtime. +type directTestRuntime struct { + *syncTestRuntime +} + +type syncTestAcceptedError struct { + cause error +} + +// Error preserves the inline handler failure returned by the synchronous test runtime. +func (e syncTestAcceptedError) Error() string { return e.cause.Error() } + +// Unwrap exposes the inline handler failure to errors.Is and errors.As. +func (e syncTestAcceptedError) Unwrap() error { return e.cause } + +// DispatchAccepted reports that the test runtime invoked a handler after accepting its delivery. +func (e syncTestAcceptedError) DispatchAccepted() bool { return true } + +func newSyncTestRuntime() *syncTestRuntime { + return &syncTestRuntime{handlers: make(map[string]busruntime.Handler)} +} + +// newDirectTestRuntime creates an inline runtime that exercises the optional +// direct-dispatch capability without changing legacy test fixtures. +func newDirectTestRuntime() *directTestRuntime { + return &directTestRuntime{syncTestRuntime: newSyncTestRuntime()} +} + +func (r *syncTestRuntime) BusRegister(jobType string, handler busruntime.Handler) { + if r.handlers == nil { + r.handlers = make(map[string]busruntime.Handler) + } + r.handlers[jobType] = handler +} + +func (r *syncTestRuntime) BusDispatch(ctx context.Context, jobType string, payload []byte, _ busruntime.JobOptions) error { + if r.dispatchErr != nil { + return r.dispatchErr + } + h, ok := r.handlers[jobType] + if !ok || h == nil { + return fmt.Errorf("handler not registered for %q", jobType) + } + if err := h(ctx, testInboundJob{payload: append([]byte(nil), payload...)}); err != nil { + return syncTestAcceptedError{cause: err} + } + return nil +} + +// BusDispatchDirect carries direct metadata beside the application payload in +// the same way a compatible physical runtime presents it to the engine. +func (r *directTestRuntime) BusDispatchDirect(ctx context.Context, jobType string, payload []byte, metadata busruntime.DeliveryMetadata, opts busruntime.JobOptions) error { + return r.BusDispatch(busruntime.WithDeliveryMetadata(ctx, metadata), jobType, payload, opts) +} + +func (r *syncTestRuntime) StartWorkers(context.Context) error { return nil } +func (r *syncTestRuntime) Shutdown(context.Context) error { return nil } + +func (r *syncTestRuntime) DispatchJSON(ctx context.Context, jobType string, payload any) error { + b, err := json.Marshal(payload) + if err != nil { + return err + } + return r.BusDispatch(ctx, jobType, b, busruntime.JobOptions{}) +} + +// TestRuntimeNilHandlerRegistrationIsNoop verifies direct runtimes never receive an executable target for a nil application handler. +func TestRuntimeNilHandlerRegistrationIsNoop(t *testing.T) { + transport := newDirectTestRuntime() + engine, err := New(transport) + if err != nil { + t.Fatalf("new runtime: %v", err) + } + + const jobType = "workflow:nil-registration" + engine.Register(jobType, nil) + if _, registered := transport.handlers[jobType]; registered { + t.Fatal("nil handler created a physical direct-delivery registration") + } + if _, err := engine.DispatchDirect(context.Background(), StoredJob{Type: jobType}); err == nil { + t.Fatal("nil registration accepted a direct delivery") + } else if !strings.Contains(err.Error(), "handler not registered") { + t.Fatalf("direct dispatch error = %v, want missing handler", err) + } + + handlerCalls := 0 + engine.Register(jobType, func(context.Context, Context) error { + handlerCalls++ + return nil + }) + engine.Register(jobType, nil) + if _, err := engine.DispatchDirect(context.Background(), StoredJob{Type: jobType}); err != nil { + t.Fatalf("dispatch after nil replacement: %v", err) + } + if handlerCalls != 1 { + t.Fatalf("handler calls = %d, want 1", handlerCalls) + } +} diff --git a/internal/workflow/transition_receipt_readback_test.go b/internal/workflow/transition_receipt_readback_test.go new file mode 100644 index 0000000..b4736be --- /dev/null +++ b/internal/workflow/transition_receipt_readback_test.go @@ -0,0 +1,218 @@ +package workflow + +import ( + "context" + "errors" + "testing" +) + +// TestTransitionReceiptAmbiguousCommitReadback proves a durable receipt can +// resolve an uncertain commit response only for the generation and outcome it +// names. +func TestTransitionReceiptAmbiguousCommitReadback(t *testing.T) { + t.Run("chain", func(t *testing.T) { + store := newSQLiteStore(t).(*sqlStore) + ctx := context.Background() + const ( + chainID = "chain-ambiguous-commit-readback" + nodeID = "node-ambiguous-commit-readback" + ) + claim := transitionClaim{ + deliveryID: "generation-chain-ambiguous-commit", + attempt: 3, + dispatchID: "dispatch-chain-ambiguous-commit", + jobID: "job-chain-ambiguous-commit", + jobFingerprint: "fingerprint-chain-ambiguous-commit", + } + if err := store.CreateChain(ctx, ChainRecord{ + ChainID: chainID, + DispatchID: claim.dispatchID, + Nodes: []ChainNode{{NodeID: nodeID}}, + }); err != nil { + t.Fatalf("create chain: %v", err) + } + advanced, err := store.advanceChainOutcome(ctx, chainID, nodeID, claim) + if err != nil || !advanced.claimedNow || !advanced.done || !advanced.receiptKnown { + t.Fatalf("commit chain transition = %+v, err:%v", advanced, err) + } + + commitErr := errors.New("connection lost after chain commit") + readback, err := store.readCommittedChainAdvance(ctx, chainID, nodeID, claim, commitErr) + if err != nil || !readback.claimedNow || !readback.successOwned || !readback.done || !readback.receiptKnown || readback.receipt.owner != claim { + t.Fatalf("chain commit readback = %+v, err:%v", readback, err) + } + other := claim + other.deliveryID = "generation-chain-other" + if _, err := store.readCommittedChainAdvance(ctx, chainID, nodeID, other, commitErr); !errors.Is(err, commitErr) { + t.Fatalf("different chain owner readback error = %v, want %v", err, commitErr) + } + }) + + t.Run("chain failure", func(t *testing.T) { + store := newSQLiteStore(t).(*sqlStore) + ctx := context.Background() + const ( + chainID = "chain-failure-ambiguous-commit-readback" + nodeID = "node-failure-ambiguous-commit-readback" + ) + claim := transitionClaim{ + deliveryID: "generation-chain-failure-ambiguous-commit", + attempt: 2, + dispatchID: "dispatch-chain-failure-ambiguous-commit", + jobID: "job-chain-failure-ambiguous-commit", + jobFingerprint: "fingerprint-chain-failure-ambiguous-commit", + } + if err := store.CreateChain(ctx, ChainRecord{ChainID: chainID, DispatchID: claim.dispatchID, Nodes: []ChainNode{{NodeID: nodeID}}}); err != nil { + t.Fatalf("create chain: %v", err) + } + failed, err := store.failChainOutcome(ctx, chainID, nodeID, errors.New("committed chain failure"), claim) + if err != nil || !failed.claimedNow || !failed.owned || !failed.receiptKnown || failed.receipt.outcome != BatchJobFailed { + t.Fatalf("commit chain failure = %+v err:%v", failed, err) + } + + commitErr := errors.New("connection lost after chain failure commit") + readback, err := store.readCommittedChainFailure(ctx, chainID, nodeID, claim, commitErr) + if err != nil || !readback.claimedNow || !readback.owned || !readback.receiptKnown || readback.receipt.owner != claim || readback.receipt.outcome != BatchJobFailed || !readback.state.Failed || readback.state.Completed { + t.Fatalf("chain failure readback = %+v err:%v", readback, err) + } + other := claim + other.deliveryID = "generation-chain-failure-other" + if _, err := store.readCommittedChainFailure(ctx, chainID, nodeID, other, commitErr); !errors.Is(err, commitErr) { + t.Fatalf("different chain failure owner readback error = %v, want %v", err, commitErr) + } + if _, err := store.db.ExecContext(ctx, `UPDATE bus_workflow_transition_receipts SET outcome='succeeded' WHERE workflow_kind=? AND workflow_id=? AND member_id=?`, chainTransitionKind, chainID, nodeID); err != nil { + t.Fatalf("change chain failure receipt outcome: %v", err) + } + if _, err := store.readCommittedChainFailure(ctx, chainID, nodeID, claim, commitErr); !errors.Is(err, commitErr) { + t.Fatalf("different chain failure outcome readback error = %v, want %v", err, commitErr) + } + if _, err := store.db.ExecContext(ctx, `UPDATE bus_workflow_transition_receipts SET outcome='failed', aggregate_completed=1 WHERE workflow_kind=? AND workflow_id=? AND member_id=?`, chainTransitionKind, chainID, nodeID); err != nil { + t.Fatalf("change chain failure receipt completion: %v", err) + } + if _, err := store.readCommittedChainFailure(ctx, chainID, nodeID, claim, commitErr); !errors.Is(err, commitErr) { + t.Fatalf("completed chain failure receipt readback error = %v, want %v", err, commitErr) + } + }) + + t.Run("batch", func(t *testing.T) { + store := newSQLiteStore(t).(*sqlStore) + ctx := context.Background() + const ( + batchID = "batch-ambiguous-commit-readback" + jobID = "job-ambiguous-commit-readback" + ) + claim := transitionClaim{ + deliveryID: "generation-batch-ambiguous-commit", + attempt: 4, + dispatchID: "dispatch-batch-ambiguous-commit", + jobID: jobID, + jobFingerprint: "fingerprint-batch-ambiguous-commit", + } + if err := store.CreateBatch(ctx, BatchRecord{ + BatchID: batchID, + DispatchID: claim.dispatchID, + AllowFailed: true, + Jobs: []BatchJob{{JobID: jobID}}, + }); err != nil { + t.Fatalf("create batch: %v", err) + } + settled, err := store.settleBatchOutcome(ctx, batchID, jobID, BatchJobFailed, errors.New("member failed"), claim) + if err != nil || !settled.claimedNow || !settled.owned || !settled.state.Completed || !settled.receiptKnown { + t.Fatalf("commit batch transition = %+v, err:%v", settled, err) + } + + commitErr := errors.New("connection lost after batch commit") + state, done, owned, claimed, receipt, known, err := store.readCommittedBatchSettlement(ctx, batchID, jobID, true, claim, commitErr) + if err != nil || !done || !owned || !claimed || !known || !state.Completed || !receipt.aggregateCompleted || receipt.owner != claim || receipt.outcome != BatchJobFailed { + t.Fatalf("batch commit readback = state:%+v done:%t owned:%t claimed:%t receipt:%+v known:%t err:%v", state, done, owned, claimed, receipt, known, err) + } + if _, _, _, _, _, _, err := store.readCommittedBatchSettlement(ctx, batchID, jobID, false, claim, commitErr); !errors.Is(err, commitErr) { + t.Fatalf("different batch outcome readback error = %v, want %v", err, commitErr) + } + }) + + t.Run("nonterminal batch member after later completion", func(t *testing.T) { + store := newSQLiteStore(t).(*sqlStore) + ctx := context.Background() + const ( + batchID = "batch-ambiguous-nonterminal-readback" + dispatchID = "dispatch-ambiguous-nonterminal-readback" + firstJob = "job-ambiguous-first" + finalJob = "job-ambiguous-final" + ) + firstClaim := transitionClaim{deliveryID: "generation-ambiguous-first", attempt: 1, dispatchID: dispatchID, jobID: firstJob, jobFingerprint: "fingerprint-ambiguous-first"} + finalClaim := transitionClaim{deliveryID: "generation-ambiguous-final", attempt: 2, dispatchID: dispatchID, jobID: finalJob, jobFingerprint: "fingerprint-ambiguous-final"} + if err := store.CreateBatch(ctx, BatchRecord{ + BatchID: batchID, + DispatchID: dispatchID, + AllowFailed: true, + Jobs: []BatchJob{{JobID: firstJob}, {JobID: finalJob}}, + }); err != nil { + t.Fatalf("create batch: %v", err) + } + first, err := store.settleBatchOutcome(ctx, batchID, firstJob, BatchJobSucceeded, nil, firstClaim) + if err != nil || !first.claimedNow || first.state.Completed || !first.receiptKnown || first.receipt.aggregateCompleted { + t.Fatalf("settle first member = %+v, err:%v", first, err) + } + final, err := store.settleBatchOutcome(ctx, batchID, finalJob, BatchJobSucceeded, nil, finalClaim) + if err != nil || !final.claimedNow || !final.state.Completed || !final.receiptKnown || !final.receipt.aggregateCompleted { + t.Fatalf("settle final member = %+v, err:%v", final, err) + } + + commitErr := errors.New("connection lost after first member commit") + state, done, owned, claimed, receipt, known, err := store.readCommittedBatchSettlement(ctx, batchID, firstJob, false, firstClaim, commitErr) + if err != nil || !done || !owned || !claimed || !known || !state.Completed || receipt.aggregateCompleted || receipt.owner != firstClaim { + t.Fatalf("first member readback = state:%+v done:%t owned:%t claimed:%t receipt:%+v known:%t err:%v", state, done, owned, claimed, receipt, known, err) + } + settled := batchSettlementResult{state: state, owned: owned, claimedNow: claimed, receipt: receipt, receiptKnown: known} + if batchSettlementOwnsTerminal(settled, BatchJobSucceeded) { + t.Fatal("nonterminal member was credited with completion committed by a later member") + } + if !batchSettlementOwnsTerminal(final, BatchJobSucceeded) { + t.Fatal("exact terminal member receipt did not retain completion ownership") + } + mismatched := final + mismatched.receipt.aggregateCancelled = true + if batchSettlementOwnsTerminal(mismatched, BatchJobSucceeded) { + t.Fatal("receipt with contradictory cancellation was accepted as terminal owner") + } + }) +} + +// TestTransitionReceiptUnknownVersionsFailClosed ensures a mixed-version +// worker distinguishes unreadable provenance from an absent receipt. +func TestTransitionReceiptUnknownVersionsFailClosed(t *testing.T) { + for _, test := range []struct { + name string + column string + }{ + {name: "receipt version", column: "receipt_version"}, + {name: "event schema", column: "event_schema_version"}, + } { + t.Run(test.name, func(t *testing.T) { + store := newSQLiteStore(t).(*sqlStore) + ctx := context.Background() + claim := transitionClaim{ + deliveryID: "generation-unknown-version", + attempt: 0, + dispatchID: "dispatch-unknown-version", + jobID: "job-unknown-version", + jobFingerprint: "fingerprint-unknown-version", + } + if err := store.CreateChain(ctx, ChainRecord{ChainID: "chain-unknown-version", DispatchID: claim.dispatchID, Nodes: []ChainNode{{NodeID: "node-unknown-version"}}}); err != nil { + t.Fatalf("create chain: %v", err) + } + if result, err := store.advanceChainOutcome(ctx, "chain-unknown-version", "node-unknown-version", claim); err != nil || !result.receiptKnown { + t.Fatalf("advance chain = %+v, err:%v", result, err) + } + query := "UPDATE bus_workflow_transition_receipts SET " + test.column + "=? WHERE workflow_kind=? AND workflow_id=? AND member_id=?" + if _, err := store.db.ExecContext(ctx, query, 99, chainTransitionKind, "chain-unknown-version", "node-unknown-version"); err != nil { + t.Fatalf("install unknown version: %v", err) + } + receipt, known, err := store.chainTransitionReceipt(ctx, "chain-unknown-version", "node-unknown-version") + if !errors.Is(err, errUnsupportedTransitionReceipt) || known { + t.Fatalf("unknown-version receipt = known:%t receipt:%+v err:%v", known, receipt, err) + } + }) + } +} diff --git a/internal/workflow/transition_receipt_version_test.go b/internal/workflow/transition_receipt_version_test.go new file mode 100644 index 0000000..21a7eaa --- /dev/null +++ b/internal/workflow/transition_receipt_version_test.go @@ -0,0 +1,95 @@ +package workflow + +import ( + "context" + "errors" + "testing" + + "github.com/goforj/queue/busruntime" +) + +// TestUnknownTransitionReceiptVersionsBlockRecoveredApplicationExecution +// proves a mixed-version worker neither acknowledges nor replays application +// code when durable provenance exists but cannot be interpreted. +func TestUnknownTransitionReceiptVersionsBlockRecoveredApplicationExecution(t *testing.T) { + t.Run("chain receipt version", func(t *testing.T) { + const ( + chainID = "chain-unknown-receipt-runtime" + nodeID = "node-unknown-receipt-runtime" + dispatchID = "dispatch-unknown-receipt-runtime" + jobID = "job-unknown-receipt-runtime" + jobType = "workflow:chain:unknown-receipt-runtime" + owner = "generation-chain-unknown-receipt" + ) + store := NewMemoryStore().(*memoryStore) + env := envelope{SchemaVersion: schemaVersion, DispatchID: dispatchID, Kind: "chain_node", ChainID: chainID, NodeID: nodeID, JobID: jobID, Job: StoredJob{Type: jobType}} + if err := store.CreateChain(context.Background(), ChainRecord{ChainID: chainID, DispatchID: dispatchID, Nodes: []ChainNode{{NodeID: nodeID, Job: env.Job}}}); err != nil { + t.Fatalf("create chain: %v", err) + } + if result, err := store.advanceChainOutcome(context.Background(), chainID, nodeID, workflowTransitionClaim(env, 2, owner)); err != nil || !result.receiptKnown { + t.Fatalf("advance chain = %+v, err:%v", result, err) + } + key := transitionReceiptKey{workflowKind: chainTransitionKind, workflowID: chainID, memberID: nodeID} + store.mu.Lock() + receipt := store.transitionReceipts[key] + receipt.version++ + store.transitionReceipts[key] = receipt + store.mu.Unlock() + + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, store) + var handlerCalls int + runtime.Register(jobType, func(context.Context, Context) error { + handlerCalls++ + return nil + }) + deliveryContext, settlement := busruntime.WithDeliverySettlement(exhaustedWorkflowContext()) + deliveryContext = workflowRecoveryContext(deliveryContext, "generation-chain-unknown-replay", owner) + err := queueRuntime.DispatchJSON(deliveryContext, internalJobChainNode, env) + if !busruntime.IsUncommitted(err) || !errors.Is(err, errUnsupportedTransitionReceipt) { + t.Fatalf("unknown chain receipt error = %v", err) + } + if handlerCalls != 0 || settlement.ApplicationStateCommitted() || len(recorder.events) != 0 { + t.Fatalf("chain handler/committed/events = %d/%t/%d, want 0/false/0", handlerCalls, settlement.ApplicationStateCommitted(), len(recorder.events)) + } + }) + + t.Run("batch event schema", func(t *testing.T) { + const ( + batchID = "batch-unknown-receipt-runtime" + dispatchID = "dispatch-batch-unknown-receipt-runtime" + jobID = "job-batch-unknown-receipt-runtime" + jobType = "workflow:batch:unknown-receipt-runtime" + owner = "generation-batch-unknown-receipt" + ) + store := NewMemoryStore().(*memoryStore) + env := envelope{SchemaVersion: schemaVersion, DispatchID: dispatchID, Kind: "batch_job", BatchID: batchID, JobID: jobID, Job: StoredJob{Type: jobType}} + if err := store.CreateBatch(context.Background(), BatchRecord{BatchID: batchID, DispatchID: dispatchID, Jobs: []BatchJob{{JobID: jobID, Job: env.Job}}}); err != nil { + t.Fatalf("create batch: %v", err) + } + if result, err := store.settleBatchOutcome(context.Background(), batchID, jobID, BatchJobSucceeded, nil, workflowTransitionClaim(env, 2, owner)); err != nil || !result.receiptKnown { + t.Fatalf("settle batch = %+v, err:%v", result, err) + } + key := transitionReceiptKey{workflowKind: batchTransitionKind, workflowID: batchID, memberID: jobID} + store.mu.Lock() + receipt := store.transitionReceipts[key] + receipt.eventSchemaVersion++ + store.transitionReceipts[key] = receipt + store.mu.Unlock() + + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, store) + var handlerCalls int + runtime.Register(jobType, func(context.Context, Context) error { + handlerCalls++ + return nil + }) + deliveryContext, settlement := busruntime.WithDeliverySettlement(exhaustedWorkflowContext()) + deliveryContext = workflowRecoveryContext(deliveryContext, "generation-batch-unknown-replay", owner) + err := queueRuntime.DispatchJSON(deliveryContext, internalJobBatchJob, env) + if !busruntime.IsUncommitted(err) || !errors.Is(err, errUnsupportedTransitionReceipt) { + t.Fatalf("unknown batch receipt error = %v", err) + } + if handlerCalls != 0 || settlement.ApplicationStateCommitted() || len(recorder.events) != 0 { + t.Fatalf("batch handler/committed/events = %d/%t/%d, want 0/false/0", handlerCalls, settlement.ApplicationStateCommitted(), len(recorder.events)) + } + }) +} diff --git a/internal/workflow/types.go b/internal/workflow/types.go new file mode 100644 index 0000000..39540a3 --- /dev/null +++ b/internal/workflow/types.go @@ -0,0 +1,120 @@ +package workflow + +import ( + "context" + "encoding/json" + "time" +) + +// Handler processes one logical workflow delivery. +type Handler func(ctx context.Context, j Context) error + +// Job describes one logical application job and its workflow delivery policy. +type Job struct { + Type string + Payload any + Options JobOptions +} + +// NewJob creates a typed workflow job payload with optional fluent policy. +func NewJob(jobType string, payload any) Job { + return Job{Type: jobType, Payload: payload} +} + +// OnQueue sets the target queue for this job. +func (j Job) OnQueue(name string) Job { + j.Options.Queue = name + return j +} + +// Delay defers job execution. +func (j Job) Delay(delay time.Duration) Job { + j.Options.Delay = delay + return j +} + +// Timeout sets execution timeout for this job. +func (j Job) Timeout(timeout time.Duration) Job { + j.Options.Timeout = timeout + return j +} + +// Retry sets max retry attempts for this job. +func (j Job) Retry(max int) Job { + j.Options.Retry = max + return j +} + +// Backoff sets retry backoff for this job. +func (j Job) Backoff(backoff time.Duration) Job { + j.Options.Backoff = backoff + return j +} + +// UniqueFor sets dedupe TTL for this job. +func (j Job) UniqueFor(ttl time.Duration) Job { + j.Options.UniqueFor = ttl + return j +} + +// JobOptions carries queue delivery policy through the versioned workflow envelope. +type JobOptions struct { + Queue string + Delay time.Duration + Timeout time.Duration + Retry int + Backoff time.Duration + UniqueFor time.Duration +} + +// DispatchResult identifies an accepted logical dispatch. +type DispatchResult struct { + DispatchID string +} + +// Context carries workflow correlation and isolated payload data into handlers and middleware. +type Context struct { + SchemaVersion int + DispatchID string + JobID string + ChainID string + BatchID string + Attempt int + JobType string + payload []byte +} + +// NewContext reconstructs an engine message from correlation metadata and raw payload bytes. +// The payload is copied because compatibility adapters may reuse their input buffers. +func NewContext(schemaVersion int, dispatchID, jobID, chainID, batchID string, attempt int, jobType string, payload []byte) Context { + var isolatedPayload []byte + if payload != nil { + isolatedPayload = make([]byte, len(payload)) + copy(isolatedPayload, payload) + } + return Context{ + SchemaVersion: schemaVersion, + DispatchID: dispatchID, + JobID: jobID, + ChainID: chainID, + BatchID: batchID, + Attempt: attempt, + JobType: jobType, + payload: isolatedPayload, + } +} + +// PayloadBytes returns a copy of raw job payload bytes. +func (c Context) PayloadBytes() []byte { + if c.payload == nil { + return nil + } + payload := make([]byte, len(c.payload)) + copy(payload, c.payload) + return payload +} + +// Bind unmarshals the job payload into dst. +func (c Context) Bind(dst any) error { + return json.Unmarshal(c.payload, dst) +} diff --git a/bus/types_context_test.go b/internal/workflow/types_context_test.go similarity index 98% rename from bus/types_context_test.go rename to internal/workflow/types_context_test.go index 2b3ac15..9f5dc20 100644 --- a/bus/types_context_test.go +++ b/internal/workflow/types_context_test.go @@ -1,4 +1,4 @@ -package bus +package workflow import "testing" @@ -40,4 +40,3 @@ func TestContextBind(t *testing.T) { } }) } - diff --git a/internal/workflow/workflow_mutation_retry_test.go b/internal/workflow/workflow_mutation_retry_test.go new file mode 100644 index 0000000..c5b5ee4 --- /dev/null +++ b/internal/workflow/workflow_mutation_retry_test.go @@ -0,0 +1,3263 @@ +package workflow + +import ( + "context" + "encoding/json" + "errors" + "strings" + "testing" + + "github.com/goforj/queue/busruntime" +) + +type workflowMutationFaultStore struct { + Store + advanceChainErr error + failChainErr error + markBatchStartedErr error + markBatchSucceededErr error + markBatchFailedErr error + cancelBatchErr error + getChainErr error + getChainErrOnCall int + getChainCalls int + getChainState *ChainState + getBatchErr error + getBatchState *BatchState + markCallbackErr error + advanceDoneWithoutState bool + failChainWithoutState bool +} + +type nonterminalWorkflowOutcomeStore struct { + Store +} + +// compatibilityOutcomeStore exposes only the public outcome capability so +// runtime tests cannot accidentally inherit a built-in private claim method. +type compatibilityOutcomeStore struct { + Store +} + +// FailChainNode claims failure without committing terminal state to exercise runtime confirmation. +func (s nonterminalWorkflowOutcomeStore) FailChainNode(ctx context.Context, chainID, _ string, _ error) (ChainState, bool, error) { + state, err := s.Store.GetChain(ctx, chainID) + return state, true, err +} + +// SettleBatchJob is unused by the chain-focused fault but completes the atomic capability contract. +func (s nonterminalWorkflowOutcomeStore) SettleBatchJob(context.Context, string, string, BatchJobOutcome, error) (BatchState, bool, error) { + return BatchState{}, false, errors.New("unexpected batch settlement") +} + +// FailChainNode delegates the public outcome capability without exposing any +// private transition claim metadata. +func (s compatibilityOutcomeStore) FailChainNode(ctx context.Context, chainID, nodeID string, cause error) (ChainState, bool, error) { + store, ok := s.Store.(outcomeStore) + if !ok { + return ChainState{}, false, errors.New("wrapped store does not support outcome arbitration") + } + return store.FailChainNode(ctx, chainID, nodeID, cause) +} + +// SettleBatchJob delegates the public outcome capability without exposing any +// private transition claim metadata. +func (s compatibilityOutcomeStore) SettleBatchJob(ctx context.Context, batchID, jobID string, outcome BatchJobOutcome, cause error) (BatchState, bool, error) { + store, ok := s.Store.(outcomeStore) + if !ok { + return BatchState{}, false, errors.New("wrapped store does not support outcome arbitration") + } + return store.SettleBatchJob(ctx, batchID, jobID, outcome, cause) +} + +// AdvanceChain injects a chain progression persistence failure when configured. +func (s *workflowMutationFaultStore) AdvanceChain(ctx context.Context, chainID, completedNode string) (*ChainNode, bool, error) { + if s.advanceChainErr != nil { + return nil, false, s.advanceChainErr + } + if s.advanceDoneWithoutState { + return nil, true, nil + } + return s.Store.AdvanceChain(ctx, chainID, completedNode) +} + +// FailChain injects a terminal chain persistence failure when configured. +func (s *workflowMutationFaultStore) FailChain(ctx context.Context, chainID string, cause error) error { + if s.failChainErr != nil { + return s.failChainErr + } + if s.failChainWithoutState { + return nil + } + return s.Store.FailChain(ctx, chainID, cause) +} + +// MarkBatchJobStarted injects a batch-start persistence failure when configured. +func (s *workflowMutationFaultStore) MarkBatchJobStarted(ctx context.Context, batchID, jobID string) error { + if s.markBatchStartedErr != nil { + return s.markBatchStartedErr + } + return s.Store.MarkBatchJobStarted(ctx, batchID, jobID) +} + +// MarkBatchJobSucceeded injects a successful batch outcome persistence failure when configured. +func (s *workflowMutationFaultStore) MarkBatchJobSucceeded(ctx context.Context, batchID, jobID string) (BatchState, bool, error) { + if s.markBatchSucceededErr != nil { + return BatchState{}, false, s.markBatchSucceededErr + } + return s.Store.MarkBatchJobSucceeded(ctx, batchID, jobID) +} + +// MarkBatchJobFailed injects a failed batch outcome persistence failure when configured. +func (s *workflowMutationFaultStore) MarkBatchJobFailed(ctx context.Context, batchID, jobID string, cause error) (BatchState, bool, error) { + if s.markBatchFailedErr != nil { + return BatchState{}, false, s.markBatchFailedErr + } + return s.Store.MarkBatchJobFailed(ctx, batchID, jobID, cause) +} + +// CancelBatch injects an initial batch cancellation persistence failure when configured. +func (s *workflowMutationFaultStore) CancelBatch(ctx context.Context, batchID string) error { + if s.cancelBatchErr != nil { + return s.cancelBatchErr + } + return s.Store.CancelBatch(ctx, batchID) +} + +// GetChain injects a callback chain-state read failure when configured. +func (s *workflowMutationFaultStore) GetChain(ctx context.Context, chainID string) (ChainState, error) { + s.getChainCalls++ + if s.getChainErr != nil { + return ChainState{}, s.getChainErr + } + if s.getChainErrOnCall > 0 && s.getChainCalls == s.getChainErrOnCall { + return ChainState{}, errors.New("injected chain read failure") + } + if s.getChainState != nil { + return *s.getChainState, nil + } + return s.Store.GetChain(ctx, chainID) +} + +// GetBatch injects a callback batch-state read failure when configured. +func (s *workflowMutationFaultStore) GetBatch(ctx context.Context, batchID string) (BatchState, error) { + if s.getBatchErr != nil { + return BatchState{}, s.getBatchErr + } + if s.getBatchState != nil { + return *s.getBatchState, nil + } + return s.Store.GetBatch(ctx, batchID) +} + +// MarkCallbackInvoked injects a callback idempotency persistence failure when configured. +func (s *workflowMutationFaultStore) MarkCallbackInvoked(ctx context.Context, key string) (bool, error) { + if s.markCallbackErr != nil { + return false, s.markCallbackErr + } + return s.Store.MarkCallbackInvoked(ctx, key) +} + +type workflowMutationEventRecorder struct { + events []Event +} + +// Observe records workflow facts synchronously for mutation-boundary assertions. +func (r *workflowMutationEventRecorder) Observe(_ context.Context, event Event) { + r.events = append(r.events, event) +} + +// newWorkflowMutationRuntime constructs the smallest runtime that can invoke internal workflow deliveries directly. +func newWorkflowMutationRuntime(t *testing.T, store Store) (*runtime, *syncTestRuntime, *workflowMutationEventRecorder) { + t.Helper() + queueRuntime := newSyncTestRuntime() + recorder := &workflowMutationEventRecorder{} + workflow, err := NewWithStore(queueRuntime, store, WithObserver(recorder)) + if err != nil { + t.Fatalf("new workflow runtime: %v", err) + } + workflowRuntime, ok := workflow.(*runtime) + if !ok { + t.Fatalf("runtime type = %T", workflow) + } + return workflowRuntime, queueRuntime, recorder +} + +// exhaustedWorkflowContext fixes the physical attempt at its application retry boundary. +func exhaustedWorkflowContext() context.Context { + return busruntime.WithDeliveryAttempt(context.Background(), busruntime.DeliveryAttempt{Number: 2, MaxRetry: 2}) +} + +// workflowGenerationContext attaches one opaque settlement generation to a +// direct workflow-delivery test context. +func workflowGenerationContext(ctx context.Context, generationID string) context.Context { + return busruntime.WithDeliveryProvenance(ctx, busruntime.DeliveryProvenance{GenerationID: generationID}) +} + +// workflowRecoveryContext identifies both the current claim and the earlier +// unsettled generation whose receipt may be reconstructed. +func workflowRecoveryContext(ctx context.Context, generationID, recoveredGenerationID string) context.Context { + return busruntime.WithDeliveryProvenance(ctx, busruntime.DeliveryProvenance{ + GenerationID: generationID, + RecoveredGenerationID: recoveredGenerationID, + Recovered: true, + }) +} + +// workflowTransitionClaim creates the durable receipt identity expected from +// one direct workflow-delivery fixture. +func workflowTransitionClaim(env envelope, attempt int, generationID string) transitionClaim { + return transitionClaim{ + deliveryID: generationID, + attempt: attempt, + dispatchID: env.DispatchID, + jobID: env.JobID, + jobFingerprint: storedJobReceiptFingerprint(env.Job), + } +} + +// assertUncommittedMutation verifies the store cause survives the same-attempt redelivery marker. +func assertUncommittedMutation(t *testing.T, err, storeErr error) { + t.Helper() + if !busruntime.IsUncommitted(err) || !errors.Is(err, storeErr) { + t.Fatalf("mutation error = %v, want uncommitted store cause %v", err, storeErr) + } + if decision := busruntime.ClassifyAttempt(busruntime.DeliveryAttempt{Number: 2, MaxRetry: 2}, err); decision != busruntime.AttemptRedeliver { + t.Fatalf("exhausted mutation decision = %v, want redeliver", decision) + } +} + +// assertNoCommittedEvents rejects terminal facts that require a successful workflow mutation. +func assertNoCommittedEvents(t *testing.T, events []Event, forbidden ...EventKind) { + t.Helper() + for _, event := range events { + for _, kind := range forbidden { + if event.Kind == kind { + t.Fatalf("unexpected committed event %q in %+v", kind, events) + } + } + } +} + +// TestInitialDispatchRejectionRequiresTerminalStoreMutation verifies enqueue failure cannot fabricate chain or batch terminal facts. +func TestInitialDispatchRejectionRequiresTerminalStoreMutation(t *testing.T) { + enqueueErr := errors.New("queue rejected initial workflow job") + storeErr := errors.New("workflow terminal state unavailable") + tests := []struct { + name string + configure func(*workflowMutationFaultStore) + dispatch func(Engine) (string, error) + forbidden []EventKind + }{ + { + name: "chain", + configure: func(store *workflowMutationFaultStore) { + store.failChainErr = storeErr + }, + dispatch: func(workflow Engine) (string, error) { + return workflow.Chain(NewJob("initial:chain", nil)).Dispatch(context.Background()) + }, + forbidden: []EventKind{EventChainFailed, EventCallbackStarted, EventCallbackSucceeded, EventCallbackFailed}, + }, + { + name: "batch", + configure: func(store *workflowMutationFaultStore) { + store.cancelBatchErr = storeErr + }, + dispatch: func(workflow Engine) (string, error) { + return workflow.Batch(NewJob("initial:batch", nil)).Dispatch(context.Background()) + }, + forbidden: []EventKind{EventBatchFailed, EventBatchCancelled, EventCallbackStarted, EventCallbackSucceeded, EventCallbackFailed}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + baseStore := NewMemoryStore() + store := &workflowMutationFaultStore{Store: baseStore} + test.configure(store) + queueRuntime := newSyncTestRuntime() + queueRuntime.dispatchErr = enqueueErr + recorder := &workflowMutationEventRecorder{} + workflow, err := NewWithStore(queueRuntime, store, WithObserver(recorder)) + if err != nil { + t.Fatalf("new workflow: %v", err) + } + _, dispatchErr := test.dispatch(workflow) + if !busruntime.IsUncommitted(dispatchErr) || !errors.Is(dispatchErr, enqueueErr) || !errors.Is(dispatchErr, storeErr) { + t.Fatalf("dispatch error = %v, want uncommitted enqueue and store causes", dispatchErr) + } + assertNoCommittedEvents(t, recorder.events, test.forbidden...) + }) + } +} + +// TestInitialDispatchRejectionUsesObservedCallbackLifecycle verifies inline compatibility callbacks match queue-delivered observability. +func TestInitialDispatchRejectionUsesObservedCallbackLifecycle(t *testing.T) { + enqueueErr := errors.New("queue rejected initial workflow job") + tests := []struct { + name string + dispatch func(Engine, *int) (string, error) + failed EventKind + cancelled bool + }{ + { + name: "chain", + dispatch: func(workflow Engine, calls *int) (string, error) { + return workflow.Chain(NewJob("initial:chain:callbacks", nil)). + Catch(func(context.Context, ChainState, error) error { *calls++; return nil }). + Finally(func(context.Context, ChainState) error { *calls++; return nil }). + Dispatch(context.Background()) + }, + failed: EventChainFailed, + }, + { + name: "batch", + dispatch: func(workflow Engine, calls *int) (string, error) { + return workflow.Batch(NewJob("initial:batch:callbacks", nil)). + Catch(func(context.Context, BatchState, error) error { *calls++; return nil }). + Finally(func(context.Context, BatchState) error { *calls++; return nil }). + Dispatch(context.Background()) + }, + failed: EventBatchFailed, + cancelled: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + queueRuntime := newSyncTestRuntime() + queueRuntime.dispatchErr = enqueueErr + recorder := &workflowMutationEventRecorder{} + workflow, err := NewWithStore(queueRuntime, NewMemoryStore(), WithObserver(recorder)) + if err != nil { + t.Fatalf("new workflow: %v", err) + } + var calls int + _, dispatchErr := test.dispatch(workflow, &calls) + if !errors.Is(dispatchErr, enqueueErr) { + t.Fatalf("dispatch error = %v, want %v", dispatchErr, enqueueErr) + } + if calls != 2 { + t.Fatalf("callback calls = %d, want 2", calls) + } + var failed, cancelled, started, succeeded, callbackFailed int + for _, event := range recorder.events { + switch event.Kind { + case test.failed: + failed++ + case EventBatchCancelled: + cancelled++ + case EventCallbackStarted: + started++ + case EventCallbackSucceeded: + succeeded++ + case EventCallbackFailed: + callbackFailed++ + } + } + if failed != 1 || started != 2 || succeeded != 2 || callbackFailed != 0 { + t.Fatalf("failed/started/succeeded/callback-failed = %d/%d/%d/%d, want 1/2/2/0", failed, started, succeeded, callbackFailed) + } + if test.cancelled != (cancelled == 1) { + t.Fatalf("cancelled events = %d, expected=%t", cancelled, test.cancelled) + } + }) + } +} + +// TestChainNextDispatchRejectionRemainsUncommitted verifies an advanced chain cannot settle until its next node is accepted. +func TestChainNextDispatchRejectionRemainsUncommitted(t *testing.T) { + store := NewMemoryStore() + const chainID = "chain_next_dispatch_rejected" + first := StoredJob{Type: "chain:first"} + second := StoredJob{Type: "chain:second"} + if err := store.CreateChain(context.Background(), ChainRecord{ + ChainID: chainID, + DispatchID: "dispatch_next_rejected", + Nodes: []ChainNode{ + {NodeID: "node_first", Job: first}, + {NodeID: "node_second", Job: second}, + }, + }); err != nil { + t.Fatalf("create chain: %v", err) + } + queueRuntime := newSyncTestRuntime() + workflow, err := NewWithStore(queueRuntime, store) + if err != nil { + t.Fatalf("new workflow: %v", err) + } + runtime := workflow.(*runtime) + var firstCalls, secondCalls int + runtime.Register(first.Type, func(context.Context, Context) error { + firstCalls++ + return nil + }) + runtime.Register(second.Type, func(context.Context, Context) error { + secondCalls++ + return nil + }) + payload, err := json.Marshal(envelope{ + SchemaVersion: schemaVersion, + DispatchID: "dispatch_next_rejected", + Kind: "chain_node", + ChainID: chainID, + NodeID: "node_first", + JobID: "job_first", + Job: first, + }) + if err != nil { + t.Fatalf("marshal first node: %v", err) + } + rejection := errors.New("next node enqueue rejected") + queueRuntime.dispatchErr = rejection + firstErr := runtime.handleInternalChainNode(context.Background(), testInboundJob{payload: payload}) + if !busruntime.IsUncommitted(firstErr) || !errors.Is(firstErr, rejection) { + t.Fatalf("first delivery error = %v, want uncommitted rejection", firstErr) + } + state, err := store.GetChain(context.Background(), chainID) + if err != nil { + t.Fatalf("get advanced chain: %v", err) + } + if state.NextIndex != 1 || state.Completed || state.Failed { + t.Fatalf("state after rejected continuation = %+v", state) + } + + queueRuntime.dispatchErr = nil + if err := runtime.handleInternalChainNode(context.Background(), testInboundJob{payload: payload}); err != nil { + t.Fatalf("redeliver first node: %v", err) + } + state, err = store.GetChain(context.Background(), chainID) + if err != nil { + t.Fatalf("get completed chain: %v", err) + } + if !state.Completed || state.Failed || state.NextIndex != 2 { + t.Fatalf("completed chain state = %+v", state) + } + if firstCalls != 2 || secondCalls != 1 { + t.Fatalf("handler calls = first:%d second:%d, want 2/1", firstCalls, secondCalls) + } +} + +// TestAllowFailuresBatchStopsOnUncommittedMutation verifies infrastructure failure cannot be mistaken for an allowed application failure. +func TestAllowFailuresBatchStopsOnUncommittedMutation(t *testing.T) { + storeErr := errors.New("batch store unavailable") + baseStore := NewMemoryStore() + faultStore := &workflowMutationFaultStore{Store: baseStore, markBatchStartedErr: storeErr} + runtime, _, _ := newWorkflowMutationRuntime(t, faultStore) + var handlerCalls int + runtime.Register("workflow:batch:first", func(context.Context, Context) error { + handlerCalls++ + return nil + }) + runtime.Register("workflow:batch:second", func(context.Context, Context) error { + handlerCalls++ + return nil + }) + + batchID, err := runtime.Batch( + NewJob("workflow:batch:first", nil), + NewJob("workflow:batch:second", nil), + ).AllowFailures().Dispatch(context.Background()) + assertUncommittedMutation(t, err, storeErr) + if handlerCalls != 0 { + t.Fatalf("handler calls = %d, want 0", handlerCalls) + } + state, stateErr := baseStore.GetBatch(context.Background(), batchID) + if stateErr != nil { + t.Fatalf("get batch: %v", stateErr) + } + if state.Processed != 0 || state.Pending != 2 || state.Completed || state.Cancelled { + t.Fatalf("batch advanced after uncommitted mutation: %+v", state) + } +} + +// TestChainDispatchDoesNotTerminalizeAcceptedMutationFailure verifies post-acceptance store errors remain redeliverable. +func TestChainDispatchDoesNotTerminalizeAcceptedMutationFailure(t *testing.T) { + storeErr := errors.New("chain store unavailable") + baseStore := NewMemoryStore() + faultStore := &workflowMutationFaultStore{Store: baseStore, advanceChainErr: storeErr} + runtime, _, recorder := newWorkflowMutationRuntime(t, faultStore) + var handlerCalls int + runtime.Register("workflow:chain:first", func(context.Context, Context) error { + handlerCalls++ + return nil + }) + + chainID, err := runtime.Chain(NewJob("workflow:chain:first", nil)).Dispatch(context.Background()) + assertUncommittedMutation(t, err, storeErr) + if handlerCalls != 1 { + t.Fatalf("handler calls = %d, want 1", handlerCalls) + } + state, stateErr := baseStore.GetChain(context.Background(), chainID) + if stateErr != nil { + t.Fatalf("get chain: %v", stateErr) + } + if state.Failed || state.Completed || state.NextIndex != 0 { + t.Fatalf("chain terminalized after uncommitted mutation: %+v", state) + } + assertNoCommittedEvents(t, recorder.events, EventJobSucceeded, EventChainAdvanced, EventChainCompleted, EventChainFailed) +} + +// TestSuccessfulDuplicateCannotCompleteFailedChain proves a competing failure +// remains authoritative when the same physical node also returns success. +func TestSuccessfulDuplicateCannotCompleteFailedChain(t *testing.T) { + const ( + chainID = "chain-concurrent-terminal-outcome" + nodeID = "node-concurrent-terminal-outcome" + jobType = "workflow:chain:concurrent-terminal-outcome" + ) + store := NewMemoryStore() + if err := store.CreateChain(context.Background(), ChainRecord{ + ChainID: chainID, + Nodes: []ChainNode{{ + NodeID: nodeID, + Job: StoredJob{Type: jobType}, + }}, + }); err != nil { + t.Fatalf("create chain: %v", err) + } + if err := store.FailChain(context.Background(), chainID, errors.New("competing delivery failed")); err != nil { + t.Fatalf("fail chain: %v", err) + } + runtime, _, recorder := newWorkflowMutationRuntime(t, store) + var handlerCalls, callbackCalls int + runtime.Register(jobType, func(context.Context, Context) error { + handlerCalls++ + return nil + }) + runtime.chainCallbacks[chainID] = chainCallbacks{ + finally: func(context.Context, ChainState) error { + callbackCalls++ + return nil + }, + } + payload, err := json.Marshal(envelope{ + SchemaVersion: schemaVersion, + DispatchID: "dispatch-concurrent-terminal-outcome", + Kind: "chain_node", + ChainID: chainID, + NodeID: nodeID, + JobID: "job-concurrent-terminal-outcome", + Job: StoredJob{Type: jobType}, + }) + if err != nil { + t.Fatalf("marshal duplicate node: %v", err) + } + if err := runtime.handleInternalChainNode(context.Background(), testInboundJob{payload: payload}); err != nil { + t.Fatalf("handle successful duplicate: %v", err) + } + if handlerCalls != 1 || callbackCalls != 0 { + t.Fatalf("handler/callback calls = %d/%d, want 1/0", handlerCalls, callbackCalls) + } + state, err := store.GetChain(context.Background(), chainID) + if err != nil { + t.Fatalf("get chain: %v", err) + } + if !state.Failed || state.Completed { + t.Fatalf("chain state = %+v, want failed only", state) + } + assertNoCommittedEvents(t, recorder.events, EventJobSucceeded, EventChainAdvanced, EventChainCompleted) +} + +// TestCompletedChainPublishesOnlyFinalNodeReplay preserves post-commit +// recovery without letting an earlier node impersonate terminal completion. +func TestCompletedChainPublishesOnlyFinalNodeReplay(t *testing.T) { + const ( + chainID = "chain-completed-node-replay" + firstNodeID = "node-completed-first" + finalNodeID = "node-completed-final" + firstJobType = "workflow:chain:completed-first" + finalJobType = "workflow:chain:completed-final" + ) + store := NewMemoryStore() + if err := store.CreateChain(context.Background(), ChainRecord{ + ChainID: chainID, + Nodes: []ChainNode{ + {NodeID: firstNodeID, Job: StoredJob{Type: firstJobType}}, + {NodeID: finalNodeID, Job: StoredJob{Type: finalJobType}}, + }, + }); err != nil { + t.Fatalf("create chain: %v", err) + } + if _, _, err := store.AdvanceChain(context.Background(), chainID, firstNodeID); err != nil { + t.Fatalf("advance first node: %v", err) + } + if _, done, err := store.AdvanceChain(context.Background(), chainID, finalNodeID); err != nil || !done { + t.Fatalf("complete final node = done:%t err:%v", done, err) + } + + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, store) + var firstCalls, finalCalls, finallyCalls int + runtime.Register(firstJobType, func(context.Context, Context) error { + firstCalls++ + return nil + }) + runtime.Register(finalJobType, func(context.Context, Context) error { + finalCalls++ + return nil + }) + runtime.chainCallbacks[chainID] = chainCallbacks{ + finally: func(context.Context, ChainState) error { + finallyCalls++ + return nil + }, + } + if err := queueRuntime.DispatchJSON(exhaustedWorkflowContext(), internalJobChainNode, envelope{ + SchemaVersion: schemaVersion, + DispatchID: "dispatch-completed-node-replay", + Kind: "chain_node", + ChainID: chainID, + NodeID: firstNodeID, + JobID: "job-completed-first-replay", + Job: StoredJob{Type: firstJobType}, + }); err != nil { + t.Fatalf("replay stale first node: %v", err) + } + if firstCalls != 1 || finalCalls != 0 || finallyCalls != 0 { + t.Fatalf("calls after stale replay = first:%d final:%d finally:%d, want 1/0/0", firstCalls, finalCalls, finallyCalls) + } + assertNoCommittedEvents(t, recorder.events, EventJobSucceeded, EventChainAdvanced, EventChainCompleted, EventCallbackStarted, EventCallbackSucceeded, EventCallbackFailed) + + if err := queueRuntime.DispatchJSON(exhaustedWorkflowContext(), internalJobChainNode, envelope{ + SchemaVersion: schemaVersion, + DispatchID: "dispatch-completed-node-replay", + Kind: "chain_node", + ChainID: chainID, + NodeID: finalNodeID, + JobID: "job-completed-final-replay", + Job: StoredJob{Type: finalJobType}, + }); err != nil { + t.Fatalf("replay final node: %v", err) + } + if firstCalls != 1 || finalCalls != 1 || finallyCalls != 1 { + t.Fatalf("calls after final replay = first:%d final:%d finally:%d, want 1/1/1", firstCalls, finalCalls, finallyCalls) + } + var succeeded, completed, callbackSucceeded int + for _, event := range recorder.events { + switch event.Kind { + case EventJobSucceeded: + succeeded++ + case EventChainCompleted: + completed++ + case EventCallbackSucceeded: + callbackSucceeded++ + } + } + if succeeded != 0 || completed != 0 || callbackSucceeded != 1 { + t.Fatalf("final replay events = job:%d chain:%d callback:%d, want 0/0/1", succeeded, completed, callbackSucceeded) + } +} + +// TestTerminalChainUnknownNodeCannotPublishFacts prevents a malformed +// delivery from borrowing either terminal outcome or its pending callback. +func TestTerminalChainUnknownNodeCannotPublishFacts(t *testing.T) { + for _, terminal := range []string{"completed", "failed"} { + t.Run(terminal, func(t *testing.T) { + chainID := "chain-runtime-unknown-" + terminal + store := NewMemoryStore() + if err := store.CreateChain(context.Background(), ChainRecord{ + ChainID: chainID, + Nodes: []ChainNode{ + {NodeID: "node-0", Job: StoredJob{Type: "workflow:chain:known-0"}}, + {NodeID: "node-1", Job: StoredJob{Type: "workflow:chain:known-1"}}, + }, + }); err != nil { + t.Fatalf("create chain: %v", err) + } + if _, _, err := store.AdvanceChain(context.Background(), chainID, "node-0"); err != nil { + t.Fatalf("advance first node: %v", err) + } + if terminal == "completed" { + if _, done, err := store.AdvanceChain(context.Background(), chainID, "node-1"); err != nil || !done { + t.Fatalf("complete chain = done:%t err:%v", done, err) + } + } else { + outcomes := requireOutcomeStore(t, store) + if _, owned, err := outcomes.FailChainNode(context.Background(), chainID, "node-1", errors.New("known failure")); err != nil || !owned { + t.Fatalf("fail chain = owned:%t err:%v", owned, err) + } + } + + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, store) + var handlerCalls, callbackCalls int + runtime.Register("workflow:chain:unknown", func(context.Context, Context) error { + handlerCalls++ + return nil + }) + runtime.chainCallbacks[chainID] = chainCallbacks{ + finally: func(context.Context, ChainState) error { + callbackCalls++ + return nil + }, + } + err := queueRuntime.DispatchJSON(exhaustedWorkflowContext(), internalJobChainNode, envelope{ + SchemaVersion: schemaVersion, + DispatchID: "dispatch-runtime-unknown", + Kind: "chain_node", + ChainID: chainID, + NodeID: "node-missing", + JobID: "job-runtime-unknown", + Job: StoredJob{Type: "workflow:chain:unknown"}, + }) + if !busruntime.IsUncommitted(err) || !strings.Contains(err.Error(), "does not contain node") { + t.Fatalf("unknown-node error = %v, want uncommitted membership rejection", err) + } + if handlerCalls != 1 || callbackCalls != 0 { + t.Fatalf("handler/callback calls = %d/%d, want 1/0", handlerCalls, callbackCalls) + } + assertNoCommittedEvents(t, recorder.events, EventJobSucceeded, EventChainAdvanced, EventChainCompleted, EventCallbackStarted, EventCallbackSucceeded, EventCallbackFailed) + }) + } +} + +// TestLegacyDualTerminalChainPreservesCompletion pins completion precedence +// for rows written before FailChain began protecting completed state. +func TestLegacyDualTerminalChainPreservesCompletion(t *testing.T) { + const ( + chainID = "chain-legacy-dual-terminal" + nodeID = "node-legacy-dual-terminal" + jobType = "workflow:chain:legacy-dual-terminal" + ) + baseStore := NewMemoryStore() + if err := baseStore.CreateChain(context.Background(), ChainRecord{ChainID: chainID, Nodes: []ChainNode{{NodeID: nodeID, Job: StoredJob{Type: jobType}}}}); err != nil { + t.Fatalf("create chain: %v", err) + } + legacyState := ChainState{ + ChainID: chainID, + Nodes: []ChainNode{{NodeID: nodeID, Job: StoredJob{Type: jobType}}}, + NextIndex: 1, + Completed: true, + Failed: true, + Failure: "late legacy failure", + } + faultStore := &workflowMutationFaultStore{Store: baseStore, advanceDoneWithoutState: true, getChainState: &legacyState} + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, faultStore) + var handlerCalls, catchCalls, finallyCalls int + runtime.Register(jobType, func(context.Context, Context) error { handlerCalls++; return nil }) + runtime.chainCallbacks[chainID] = chainCallbacks{ + catch: func(context.Context, ChainState, error) error { catchCalls++; return nil }, + finally: func(context.Context, ChainState) error { finallyCalls++; return nil }, + } + if err := queueRuntime.DispatchJSON(exhaustedWorkflowContext(), internalJobChainNode, envelope{ + SchemaVersion: schemaVersion, + DispatchID: "dispatch-legacy-dual-terminal", + Kind: "chain_node", + ChainID: chainID, + NodeID: nodeID, + JobID: "job-legacy-dual-terminal", + Job: StoredJob{Type: jobType}, + }); err != nil { + t.Fatalf("handle legacy dual-terminal chain: %v", err) + } + if handlerCalls != 1 || catchCalls != 0 || finallyCalls != 1 { + t.Fatalf("handler/catch/finally calls = %d/%d/%d, want 1/0/1", handlerCalls, catchCalls, finallyCalls) + } + var succeeded, completed int + for _, event := range recorder.events { + switch event.Kind { + case EventJobSucceeded: + succeeded++ + case EventChainCompleted: + completed++ + } + } + if succeeded != 1 || completed != 1 { + t.Fatalf("success/completion events = %d/%d, want 1/1", succeeded, completed) + } + assertNoCommittedEvents(t, recorder.events, EventJobFailed, EventChainFailed) +} + +// TestFailedDuplicateCannotReplaceSuccessfulChainNode proves completion or +// advancement remains authoritative when the same physical node later fails. +func TestFailedDuplicateCannotReplaceSuccessfulChainNode(t *testing.T) { + for _, test := range []struct { + name string + nodes []ChainNode + wantCompleted bool + wantNextIndex int + }{ + { + name: "completed chain", + nodes: []ChainNode{{NodeID: "node-completed", Job: StoredJob{Type: "workflow:chain:late-failure"}}}, + wantCompleted: true, + wantNextIndex: 1, + }, + { + name: "advanced chain", + nodes: []ChainNode{ + {NodeID: "node-advanced", Job: StoredJob{Type: "workflow:chain:late-failure"}}, + {NodeID: "node-pending", Job: StoredJob{Type: "workflow:chain:pending"}}, + }, + wantNextIndex: 1, + }, + } { + t.Run(test.name, func(t *testing.T) { + const chainID = "chain-late-failure" + applicationErr := errors.New("late duplicate failed") + store := NewMemoryStore() + if err := store.CreateChain(context.Background(), ChainRecord{ChainID: chainID, Nodes: test.nodes}); err != nil { + t.Fatalf("create chain: %v", err) + } + if _, _, err := store.AdvanceChain(context.Background(), chainID, test.nodes[0].NodeID); err != nil { + t.Fatalf("commit successful node: %v", err) + } + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, store) + var handlerCalls, catchCalls, finallyCalls int + runtime.Register(test.nodes[0].Job.Type, func(context.Context, Context) error { + handlerCalls++ + return applicationErr + }) + runtime.chainCallbacks[chainID] = chainCallbacks{ + catch: func(context.Context, ChainState, error) error { + catchCalls++ + return nil + }, + finally: func(context.Context, ChainState) error { + finallyCalls++ + return nil + }, + } + err := queueRuntime.DispatchJSON(exhaustedWorkflowContext(), internalJobChainNode, envelope{ + SchemaVersion: schemaVersion, + DispatchID: "dispatch-late-failure", + Kind: "chain_node", + ChainID: chainID, + NodeID: test.nodes[0].NodeID, + JobID: "job-late-failure", + Job: test.nodes[0].Job, + }) + if err != nil { + t.Fatalf("late failed duplicate: %v", err) + } + if handlerCalls != 1 || catchCalls != 0 || finallyCalls != 0 { + t.Fatalf("handler/catch/finally calls = %d/%d/%d, want 1/0/0", handlerCalls, catchCalls, finallyCalls) + } + state, err := store.GetChain(context.Background(), chainID) + if err != nil { + t.Fatalf("get chain: %v", err) + } + if state.NextIndex != test.wantNextIndex || state.Completed != test.wantCompleted || state.Failed { + t.Fatalf("chain state = %+v, want next=%d completed=%t failed=false", state, test.wantNextIndex, test.wantCompleted) + } + assertNoCommittedEvents(t, recorder.events, EventJobFailed, EventChainFailed, EventCallbackStarted, EventCallbackSucceeded, EventCallbackFailed) + }) + } +} + +// TestBatchDuplicateCannotPublishContradictoryOutcome proves a losing physical +// result cannot emit facts, progress, or callbacks against the stored winner. +func TestBatchDuplicateCannotPublishContradictoryOutcome(t *testing.T) { + for _, first := range []BatchJobOutcome{BatchJobSucceeded, BatchJobFailed} { + t.Run(string(first), func(t *testing.T) { + const ( + batchID = "batch-contradictory-outcome" + jobID = "job-contradictory-outcome" + jobType = "workflow:batch:contradictory-outcome" + ) + applicationErr := errors.New("contradictory physical failure") + store := NewMemoryStore() + if err := store.CreateBatch(context.Background(), BatchRecord{ + BatchID: batchID, + AllowFailed: true, + Jobs: []BatchJob{{JobID: jobID, Job: StoredJob{Type: jobType}}}, + }); err != nil { + t.Fatalf("create batch: %v", err) + } + outcomes := requireOutcomeStore(t, store) + before, owned, err := outcomes.SettleBatchJob(context.Background(), batchID, jobID, first, applicationErr) + if err != nil || !owned { + t.Fatalf("commit first outcome = owned:%t err:%v", owned, err) + } + + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, store) + var handlerCalls, progressCalls, thenCalls, catchCalls, finallyCalls int + runtime.Register(jobType, func(context.Context, Context) error { + handlerCalls++ + if first == BatchJobSucceeded { + return applicationErr + } + return nil + }) + runtime.batchCallbacks[batchID] = batchCallbacks{ + progress: func(context.Context, BatchState) error { progressCalls++; return nil }, + then: func(context.Context, BatchState) error { thenCalls++; return nil }, + catch: func(context.Context, BatchState, error) error { catchCalls++; return nil }, + finally: func(context.Context, BatchState) error { finallyCalls++; return nil }, + } + err = queueRuntime.DispatchJSON(exhaustedWorkflowContext(), internalJobBatchJob, envelope{ + SchemaVersion: schemaVersion, + DispatchID: "dispatch-contradictory-outcome", + Kind: "batch_job", + BatchID: batchID, + JobID: jobID, + Job: StoredJob{Type: jobType}, + }) + if err != nil { + t.Fatalf("contradictory duplicate: %v", err) + } + if handlerCalls != 1 || progressCalls != 0 || thenCalls != 0 || catchCalls != 0 || finallyCalls != 0 { + t.Fatalf("handler/progress/then/catch/finally calls = %d/%d/%d/%d/%d, want 1/0/0/0/0", handlerCalls, progressCalls, thenCalls, catchCalls, finallyCalls) + } + after, err := store.GetBatch(context.Background(), batchID) + if err != nil { + t.Fatalf("get batch: %v", err) + } + if after.Pending != before.Pending || after.Processed != before.Processed || after.Failed != before.Failed || after.Cancelled != before.Cancelled || after.Completed != before.Completed { + t.Fatalf("batch state changed: before=%+v after=%+v", before, after) + } + assertNoCommittedEvents(t, recorder.events, + EventJobSucceeded, + EventJobFailed, + EventBatchProgressed, + EventBatchCompleted, + EventBatchFailed, + EventBatchCancelled, + EventCallbackStarted, + EventCallbackSucceeded, + EventCallbackFailed, + ) + }) + } +} + +// TestBatchSameOutcomeDuplicateSeparatesFactsFromCallbackRecovery proves the +// exact private claim gates logical facts while ordinary replays may still +// finish idempotently claimed compatibility callbacks. +func TestBatchSameOutcomeDuplicateSeparatesFactsFromCallbackRecovery(t *testing.T) { + for _, test := range []struct { + name string + outcome BatchJobOutcome + recovered bool + wantSucceeded int + wantProgressed int + wantCompleted int + wantThen int + wantCatch int + wantFinally int + wantHandler int + wantPermanent bool + }{ + {name: "ordinary success", outcome: BatchJobSucceeded, wantThen: 1, wantFinally: 1, wantHandler: 1}, + {name: "recovered success", outcome: BatchJobSucceeded, recovered: true, wantSucceeded: 1, wantProgressed: 1, wantCompleted: 1}, + {name: "ordinary failure", outcome: BatchJobFailed, wantCatch: 1, wantFinally: 1, wantHandler: 1}, + {name: "recovered failure", outcome: BatchJobFailed, recovered: true, wantPermanent: true}, + } { + t.Run(test.name, func(t *testing.T) { + const ( + batchID = "batch-same-outcome-duplicate" + jobID = "job-same-outcome-duplicate" + jobType = "workflow:batch:same-outcome-duplicate" + ) + store := NewMemoryStore() + if err := store.CreateBatch(context.Background(), BatchRecord{ + BatchID: batchID, + DispatchID: "dispatch-same-outcome-duplicate", + Jobs: []BatchJob{{JobID: jobID, Job: StoredJob{Type: jobType}}}, + }); err != nil { + t.Fatalf("create batch: %v", err) + } + delivery := envelope{ + SchemaVersion: schemaVersion, + DispatchID: "dispatch-same-outcome-duplicate", + Kind: "batch_job", + BatchID: batchID, + JobID: jobID, + Job: StoredJob{Type: jobType}, + } + settlementStore := requireBatchSettlementStore(t, store) + applicationErr := errors.New("same outcome failure") + seeded, err := settlementStore.settleBatchOutcome(context.Background(), batchID, jobID, test.outcome, applicationErr, workflowTransitionClaim(delivery, 2, "generation-batch-seed")) + if err != nil || !seeded.owned || !seeded.receiptKnown { + t.Fatalf("seed batch outcome = %+v err:%v", seeded, err) + } + + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, store) + var handlerCalls, progressCalls, thenCalls, catchCalls, finallyCalls int + runtime.Register(jobType, func(context.Context, Context) error { + handlerCalls++ + if test.outcome == BatchJobFailed { + return applicationErr + } + return nil + }) + runtime.batchCallbacks[batchID] = batchCallbacks{ + progress: func(context.Context, BatchState) error { progressCalls++; return nil }, + then: func(context.Context, BatchState) error { thenCalls++; return nil }, + catch: func(context.Context, BatchState, error) error { catchCalls++; return nil }, + finally: func(context.Context, BatchState) error { finallyCalls++; return nil }, + } + deliveryContext := exhaustedWorkflowContext() + var settlement *busruntime.DeliverySettlement + if test.recovered { + deliveryContext, settlement = busruntime.WithDeliverySettlement(deliveryContext) + deliveryContext = workflowRecoveryContext(deliveryContext, "generation-batch-replay", "generation-batch-seed") + } + err = queueRuntime.DispatchJSON(deliveryContext, internalJobBatchJob, delivery) + if test.wantPermanent { + if !busruntime.IsPermanent(err) || busruntime.IsUncommitted(err) || errors.Is(err, applicationErr) { + t.Fatalf("same-outcome duplicate error = %v, want generic permanent settlement", err) + } + } else if err != nil { + t.Fatalf("same-outcome duplicate: %v", err) + } + if test.recovered { + assertNoCommittedEvents(t, recorder.events, EventJobSucceeded, EventBatchProgressed, EventBatchCompleted) + settlement.Commit() + } + var succeeded, failed, progressed, completed, batchFailed, cancelled int + for _, event := range recorder.events { + switch event.Kind { + case EventJobSucceeded: + succeeded++ + case EventJobFailed: + failed++ + case EventBatchProgressed: + progressed++ + case EventBatchCompleted: + completed++ + case EventBatchFailed: + batchFailed++ + case EventBatchCancelled: + cancelled++ + } + } + if handlerCalls != test.wantHandler || succeeded != test.wantSucceeded || progressed != test.wantProgressed || completed != test.wantCompleted || failed != 0 || batchFailed != 0 || cancelled != 0 { + t.Fatalf("handler/job/progress/completion/failure counts = %d/%d/%d/%d/%d/%d/%d, want %d/%d/%d/%d/0/0/0", handlerCalls, succeeded, progressed, completed, failed, batchFailed, cancelled, test.wantHandler, test.wantSucceeded, test.wantProgressed, test.wantCompleted) + } + if progressCalls != 0 || thenCalls != test.wantThen || catchCalls != test.wantCatch || finallyCalls != test.wantFinally { + t.Fatalf("progress/then/catch/finally calls = %d/%d/%d/%d, want 0/%d/%d/%d", progressCalls, thenCalls, catchCalls, finallyCalls, test.wantThen, test.wantCatch, test.wantFinally) + } + }) + } +} + +// TestBatchRecoverySettlesNonFactOwnersWithoutFacts proves a valid member +// receipt settles physical nonowners without granting them fact ownership. +func TestBatchRecoverySettlesNonFactOwnersWithoutFacts(t *testing.T) { + const owner = "generation-batch-non-fact-owner" + recoveryCases := []struct { + name string + attempt int + recoveredGenerationID string + }{ + {name: "different physical attempt", attempt: 3, recoveredGenerationID: owner}, + {name: "negative current attempt", attempt: -1, recoveredGenerationID: owner}, + {name: "different recovered generation", attempt: 2, recoveredGenerationID: "generation-batch-different-recovered"}, + {name: "legacy recovery without generation", attempt: 2}, + } + for _, receiptOutcome := range []BatchJobOutcome{BatchJobSucceeded, BatchJobFailed} { + for _, recoveryCase := range recoveryCases { + t.Run(string(receiptOutcome)+"/"+recoveryCase.name, func(t *testing.T) { + const ( + batchID = "batch-non-fact-owner" + dispatchID = "dispatch-batch-non-fact-owner" + jobID = "job-batch-non-fact-owner" + jobType = "workflow:batch:non-fact-owner" + ) + store := NewMemoryStore() + env := envelope{SchemaVersion: schemaVersion, DispatchID: dispatchID, Kind: "batch_job", BatchID: batchID, JobID: jobID, Job: StoredJob{Type: jobType, Payload: []byte(`{"id":5}`)}} + if err := store.CreateBatch(context.Background(), BatchRecord{BatchID: batchID, DispatchID: dispatchID, AllowFailed: true, Jobs: []BatchJob{{JobID: jobID, Job: env.Job}}}); err != nil { + t.Fatalf("create batch: %v", err) + } + applicationCause := errors.New("original batch member failure") + settled, err := requireBatchSettlementStore(t, store).settleBatchOutcome(context.Background(), batchID, jobID, receiptOutcome, applicationCause, workflowTransitionClaim(env, 2, owner)) + if err != nil || !settled.claimedNow || !settled.receiptKnown || !settled.state.Completed { + t.Fatalf("commit batch member = %+v err:%v", settled, err) + } + + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, store) + var handlerCalls, callbackCalls int + runtime.Register(jobType, func(context.Context, Context) error { + handlerCalls++ + return applicationCause + }) + runtime.batchCallbacks[batchID] = batchCallbacks{ + progress: func(context.Context, BatchState) error { + callbackCalls++ + return nil + }, + then: func(context.Context, BatchState) error { + callbackCalls++ + return nil + }, + catch: func(context.Context, BatchState, error) error { + callbackCalls++ + return nil + }, + finally: func(context.Context, BatchState) error { + callbackCalls++ + return nil + }, + } + attemptContext := busruntime.WithDeliveryAttempt(context.Background(), busruntime.DeliveryAttempt{Number: recoveryCase.attempt, MaxRetry: 3}) + recoveryContext, deliverySettlement := busruntime.WithDeliverySettlement(attemptContext) + recoveryContext = workflowRecoveryContext(recoveryContext, "generation-batch-non-fact-owner-current", recoveryCase.recoveredGenerationID) + recoveryErr := queueRuntime.DispatchJSON(recoveryContext, internalJobBatchJob, env) + if receiptOutcome == BatchJobFailed { + if recoveryErr == nil || !busruntime.IsPermanent(recoveryErr) || busruntime.IsUncommitted(recoveryErr) || errors.Is(recoveryErr, applicationCause) { + t.Fatalf("failed-member nonowner recovery = %v, want generic permanent", recoveryErr) + } + } else if recoveryErr != nil { + t.Fatalf("successful-member nonowner recovery: %v", recoveryErr) + } + deliverySettlement.Commit() + if handlerCalls != 0 || callbackCalls != 0 || deliverySettlement.ApplicationStateCommitted() || len(recorder.events) != 0 { + t.Fatalf("handler/callback/committed/events = %d/%d/%t/%d, want 0/0/false/0", handlerCalls, callbackCalls, deliverySettlement.ApplicationStateCommitted(), len(recorder.events)) + } + }) + } + } +} + +// TestBatchNonterminalReceiptCannotReplayLaterCompletion proves an ordinary +// duplicate of an earlier member cannot run terminal callbacks merely because +// another member completed the aggregate before its receipt was re-read. +func TestBatchNonterminalReceiptCannotReplayLaterCompletion(t *testing.T) { + const ( + batchID = "batch-nonterminal-receipt-replay" + dispatchID = "dispatch-nonterminal-receipt-replay" + firstJobID = "job-nonterminal-receipt-replay" + finalJobID = "job-terminal-receipt-owner" + firstJobType = "workflow:batch:nonterminal-receipt-replay" + ) + store := NewMemoryStore() + if err := store.CreateBatch(context.Background(), BatchRecord{ + BatchID: batchID, + DispatchID: dispatchID, + AllowFailed: true, + Jobs: []BatchJob{ + {JobID: firstJobID, Job: StoredJob{Type: firstJobType}}, + {JobID: finalJobID, Job: StoredJob{Type: "workflow:batch:terminal-receipt-owner"}}, + }, + }); err != nil { + t.Fatalf("create batch: %v", err) + } + settlements := requireBatchSettlementStore(t, store) + first := envelope{DispatchID: dispatchID, BatchID: batchID, JobID: firstJobID, Job: StoredJob{Type: firstJobType}} + final := envelope{DispatchID: dispatchID, BatchID: batchID, JobID: finalJobID, Job: StoredJob{Type: "workflow:batch:terminal-receipt-owner"}} + firstResult, err := settlements.settleBatchOutcome(context.Background(), batchID, firstJobID, BatchJobSucceeded, nil, workflowTransitionClaim(first, 0, "generation-nonterminal-receipt")) + if err != nil || !firstResult.claimedNow || firstResult.state.Completed || firstResult.receipt.aggregateCompleted { + t.Fatalf("settle nonterminal member = %+v, err:%v", firstResult, err) + } + finalResult, err := settlements.settleBatchOutcome(context.Background(), batchID, finalJobID, BatchJobSucceeded, nil, workflowTransitionClaim(final, 0, "generation-terminal-receipt")) + if err != nil || !finalResult.claimedNow || !finalResult.state.Completed || !finalResult.receipt.aggregateCompleted { + t.Fatalf("settle terminal member = %+v, err:%v", finalResult, err) + } + + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, store) + var handlerCalls, thenCalls, finallyCalls int + runtime.Register(firstJobType, func(context.Context, Context) error { + handlerCalls++ + return nil + }) + runtime.batchCallbacks[batchID] = batchCallbacks{ + then: func(context.Context, BatchState) error { thenCalls++; return nil }, + finally: func(context.Context, BatchState) error { finallyCalls++; return nil }, + } + if err := queueRuntime.DispatchJSON(exhaustedWorkflowContext(), internalJobBatchJob, first); err != nil { + t.Fatalf("replay nonterminal member: %v", err) + } + if handlerCalls != 1 || thenCalls != 0 || finallyCalls != 0 { + t.Fatalf("handler/then/finally calls = %d/%d/%d, want 1/0/0", handlerCalls, thenCalls, finallyCalls) + } + assertNoCommittedEvents(t, recorder.events, EventBatchCompleted, EventBatchFailed, EventBatchCancelled, EventCallbackStarted, EventCallbackSucceeded, EventCallbackFailed) +} + +// TestBatchSettlementPublicOutcomeFallbackPreservesCompatibility proves a +// custom additive store retains its established category ownership even though +// the public interface cannot expose an exact per-call claim result. +func TestBatchSettlementPublicOutcomeFallbackPreservesCompatibility(t *testing.T) { + const ( + batchID = "batch-public-outcome-fallback" + jobID = "job-public-outcome-fallback" + ) + baseStore := NewMemoryStore() + if err := baseStore.CreateBatch(context.Background(), BatchRecord{ + BatchID: batchID, + Jobs: []BatchJob{{JobID: jobID}}, + }); err != nil { + t.Fatalf("create batch: %v", err) + } + store := compatibilityOutcomeStore{Store: baseStore} + runtime, _, _ := newWorkflowMutationRuntime(t, store) + first, err := runtime.settleBatchJob(context.Background(), batchID, jobID, BatchJobSucceeded, nil, transitionClaim{}) + if err != nil || !first.owned || !first.claimedNow || first.state.Processed != 1 { + t.Fatalf("first settlement = %+v err:%v, want owned compatibility claim", first, err) + } + replayed, err := runtime.settleBatchJob(context.Background(), batchID, jobID, BatchJobSucceeded, nil, transitionClaim{}) + if err != nil || !replayed.owned || !replayed.claimedNow || replayed.state.Processed != 1 { + t.Fatalf("replayed settlement = %+v err:%v, want projected compatibility claim", replayed, err) + } + contradictory, err := runtime.settleBatchJob(context.Background(), batchID, jobID, BatchJobFailed, errors.New("contradictory"), transitionClaim{}) + if err != nil || contradictory.owned || !contradictory.claimedNow || contradictory.state.Processed != 1 { + t.Fatalf("contradictory settlement = %+v err:%v, want losing projected compatibility claim", contradictory, err) + } +} + +// TestUnknownBatchMemberStopsBeforeExecution prevents malformed workflow +// correlation from running a handler or mutating the real aggregate. +func TestUnknownBatchMemberStopsBeforeExecution(t *testing.T) { + const ( + batchID = "batch-unknown-member" + jobType = "workflow:batch:unknown-member" + ) + store := NewMemoryStore() + if err := store.CreateBatch(context.Background(), BatchRecord{ + BatchID: batchID, + Jobs: []BatchJob{{JobID: "job-known"}}, + }); err != nil { + t.Fatalf("create batch: %v", err) + } + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, store) + var handlerCalls, progressCalls, finallyCalls int + runtime.Register(jobType, func(context.Context, Context) error { + handlerCalls++ + return nil + }) + runtime.batchCallbacks[batchID] = batchCallbacks{ + progress: func(context.Context, BatchState) error { progressCalls++; return nil }, + finally: func(context.Context, BatchState) error { finallyCalls++; return nil }, + } + err := queueRuntime.DispatchJSON(exhaustedWorkflowContext(), internalJobBatchJob, envelope{ + SchemaVersion: schemaVersion, + DispatchID: "dispatch-unknown-member", + Kind: "batch_job", + BatchID: batchID, + JobID: "job-missing", + Job: StoredJob{Type: jobType}, + }) + if !busruntime.IsUncommitted(err) || !errors.Is(err, ErrNotFound) { + t.Fatalf("unknown member error = %v, want uncommitted ErrNotFound", err) + } + if handlerCalls != 0 || progressCalls != 0 || finallyCalls != 0 { + t.Fatalf("handler/progress/finally calls = %d/%d/%d, want 0/0/0", handlerCalls, progressCalls, finallyCalls) + } + state, stateErr := store.GetBatch(context.Background(), batchID) + if stateErr != nil { + t.Fatalf("get batch: %v", stateErr) + } + if state.Pending != 1 || state.Processed != 0 || state.Failed != 0 || state.Completed || state.Cancelled { + t.Fatalf("unknown member changed batch: %+v", state) + } + assertNoCommittedEvents(t, recorder.events, + EventJobStarted, + EventJobSucceeded, + EventJobFailed, + EventBatchProgressed, + EventBatchCompleted, + EventBatchFailed, + EventBatchCancelled, + EventCallbackStarted, + EventCallbackSucceeded, + EventCallbackFailed, + ) +} + +// TestChainCompletionReadFailureRedeliversWithoutFacts proves a committed +// terminal mutation is replayed until its state can be confirmed for events. +func TestChainCompletionReadFailureRedeliversWithoutFacts(t *testing.T) { + storeErr := errors.New("chain completion read unavailable") + const ( + chainID = "chain-completion-read-failure" + nodeID = "node-completion-read-failure" + jobType = "workflow:chain:completion-read-failure" + ) + baseStore := NewMemoryStore() + if err := baseStore.CreateChain(context.Background(), ChainRecord{ + ChainID: chainID, + Nodes: []ChainNode{{NodeID: nodeID, Job: StoredJob{Type: jobType}}}, + }); err != nil { + t.Fatalf("create chain: %v", err) + } + faultStore := &workflowMutationFaultStore{Store: baseStore, getChainErr: storeErr} + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, faultStore) + var handlerCalls, finallyCalls int + runtime.Register(jobType, func(context.Context, Context) error { + handlerCalls++ + return nil + }) + runtime.chainCallbacks[chainID] = chainCallbacks{ + finally: func(context.Context, ChainState) error { + finallyCalls++ + return nil + }, + } + delivery := envelope{ + SchemaVersion: schemaVersion, + DispatchID: "dispatch-completion-read-failure", + Kind: "chain_node", + ChainID: chainID, + NodeID: nodeID, + JobID: "job-completion-read-failure", + Job: StoredJob{Type: jobType}, + } + err := queueRuntime.DispatchJSON(exhaustedWorkflowContext(), internalJobChainNode, delivery) + assertUncommittedMutation(t, err, storeErr) + if handlerCalls != 1 || finallyCalls != 0 { + t.Fatalf("handler/finally calls before recovery = %d/%d, want 1/0", handlerCalls, finallyCalls) + } + state, err := baseStore.GetChain(context.Background(), chainID) + if err != nil { + t.Fatalf("get committed chain: %v", err) + } + if !state.Completed || state.Failed { + t.Fatalf("committed chain state = %+v, want completed only", state) + } + assertNoCommittedEvents(t, recorder.events, EventJobSucceeded, EventChainCompleted, EventCallbackStarted, EventCallbackSucceeded) + + faultStore.getChainErr = nil + if err := queueRuntime.DispatchJSON(exhaustedWorkflowContext(), internalJobChainNode, delivery); err != nil { + t.Fatalf("redeliver after store recovery: %v", err) + } + if handlerCalls != 2 || finallyCalls != 1 { + t.Fatalf("handler/finally calls after recovery = %d/%d, want 2/1", handlerCalls, finallyCalls) + } + var succeeded, completed, callbackSucceeded int + for _, event := range recorder.events { + switch event.Kind { + case EventJobSucceeded: + succeeded++ + case EventChainCompleted: + completed++ + case EventCallbackSucceeded: + callbackSucceeded++ + } + } + if succeeded != 1 || completed != 1 || callbackSucceeded != 1 { + t.Fatalf("job/chain/callback success events = %d/%d/%d, want 1/1/1", succeeded, completed, callbackSucceeded) + } +} + +// TestChainCommittedSuccessSurvivesContradictorySettlementReplay proves a +// failed physical settlement recovers receipt-owned terminal facts without +// re-executing application code that could produce a contradictory result. +func TestChainCommittedSuccessSurvivesContradictorySettlementReplay(t *testing.T) { + const ( + chainID = "chain-committed-success-settlement-replay" + nodeID = "node-committed-success-settlement-replay" + jobType = "workflow:chain:committed-success-settlement-replay" + ) + store := NewMemoryStore() + if err := store.CreateChain(context.Background(), ChainRecord{ + ChainID: chainID, + DispatchID: "dispatch-committed-success-settlement-replay", + Nodes: []ChainNode{{NodeID: nodeID, Job: StoredJob{Type: jobType}}}, + }); err != nil { + t.Fatalf("create chain: %v", err) + } + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, store) + var handlerCalls int + runtime.Register(jobType, func(context.Context, Context) error { + handlerCalls++ + if handlerCalls == 1 { + return nil + } + return busruntime.Permanent(errors.New("contradictory replay failure")) + }) + delivery := envelope{ + SchemaVersion: schemaVersion, + DispatchID: "dispatch-committed-success-settlement-replay", + Kind: "chain_node", + ChainID: chainID, + NodeID: nodeID, + JobID: "job-committed-success-settlement-replay", + Job: StoredJob{Type: jobType}, + } + + firstContext, _ := busruntime.WithDeliverySettlement(exhaustedWorkflowContext()) + firstContext = workflowGenerationContext(firstContext, "generation-chain-committed-success") + if err := queueRuntime.DispatchJSON(firstContext, internalJobChainNode, delivery); err != nil { + t.Fatalf("first delivery: %v", err) + } + assertNoCommittedEvents(t, recorder.events, EventJobSucceeded, EventChainCompleted) + state, err := store.GetChain(context.Background(), chainID) + if err != nil { + t.Fatalf("get committed chain: %v", err) + } + if !state.Completed || state.Failed { + t.Fatalf("committed chain state = %+v, want completed only", state) + } + + replayContext, replaySettlement := busruntime.WithDeliverySettlement(exhaustedWorkflowContext()) + replayContext = workflowRecoveryContext(replayContext, "generation-chain-replay", "generation-chain-committed-success") + if err := queueRuntime.DispatchJSON(replayContext, internalJobChainNode, delivery); err != nil { + t.Fatalf("contradictory redelivery: %v", err) + } + assertNoCommittedEvents(t, recorder.events, EventJobSucceeded, EventChainCompleted) + replaySettlement.Commit() + + var succeeded, completed, failed int + for _, event := range recorder.events { + switch event.Kind { + case EventJobSucceeded: + succeeded++ + case EventChainCompleted: + completed++ + case EventJobFailed, EventChainFailed: + failed++ + } + } + if handlerCalls != 1 || succeeded != 1 || completed != 1 || failed != 0 { + t.Fatalf("handler/job/chain/failure counts = %d/%d/%d/%d, want 1/1/1/0", handlerCalls, succeeded, completed, failed) + } +} + +// TestChainPostTransitionFailureMarksCurrentGenerationForRecovery proves a +// recovered delivery that becomes the transition owner does not retain an +// older generation when successor enqueue requires same-attempt redelivery. +func TestChainPostTransitionFailureMarksCurrentGenerationForRecovery(t *testing.T) { + const ( + chainID = "chain-post-transition-generation" + dispatchID = "dispatch-post-transition-generation" + firstNodeID = "node-post-transition-generation" + finalNodeID = "node-post-transition-generation-final" + firstJobType = "workflow:chain:post-transition-generation" + ) + store := NewMemoryStore() + if err := store.CreateChain(context.Background(), ChainRecord{ + ChainID: chainID, + DispatchID: dispatchID, + Nodes: []ChainNode{ + {NodeID: firstNodeID, Job: StoredJob{Type: firstJobType}}, + {NodeID: finalNodeID, Job: StoredJob{Type: "workflow:chain:post-transition-generation:final"}}, + }, + }); err != nil { + t.Fatalf("create chain: %v", err) + } + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, store) + var handlerCalls int + runtime.Register(firstJobType, func(context.Context, Context) error { + handlerCalls++ + return nil + }) + delivery := envelope{ + SchemaVersion: schemaVersion, + DispatchID: dispatchID, + Kind: "chain_node", + ChainID: chainID, + NodeID: firstNodeID, + JobID: "job-post-transition-generation", + Job: StoredJob{Type: firstJobType}, + } + payload, err := json.Marshal(delivery) + if err != nil { + t.Fatalf("encode delivery: %v", err) + } + successorErr := errors.New("successor queue unavailable") + queueRuntime.dispatchErr = successorErr + deliveryContext, settlement := busruntime.WithDeliverySettlement(exhaustedWorkflowContext()) + deliveryContext = workflowRecoveryContext(deliveryContext, "generation-post-transition-current", "generation-post-transition-older") + handler := queueRuntime.handlers[internalJobChainNode] + if handler == nil { + t.Fatal("chain delivery handler is not registered") + } + err = handler(deliveryContext, testInboundJob{payload: payload}) + if !busruntime.IsUncommitted(err) || !errors.Is(err, successorErr) { + t.Fatalf("post-transition error = %v, want uncommitted successor error", err) + } + if handlerCalls != 1 || !settlement.ApplicationStateCommitted() { + t.Fatalf("handler calls/application-state signal = %d/%t, want 1/true", handlerCalls, settlement.ApplicationStateCommitted()) + } + receipt, known, err := requireTransitionReceiptStore(t, store).chainTransitionReceipt(context.Background(), chainID, firstNodeID) + if err != nil || !known || receipt.owner.deliveryID != "generation-post-transition-current" { + t.Fatalf("post-transition receipt = known:%t receipt:%+v err:%v", known, receipt, err) + } + assertNoCommittedEvents(t, recorder.events, EventJobSucceeded, EventChainAdvanced) +} + +// TestChainRecoveryWithoutExactReceiptOwnershipPreservesOnlyLiveContinuation +// proves compatibility recovery restores liveness without replaying application effects. +func TestChainRecoveryWithoutExactReceiptOwnershipPreservesOnlyLiveContinuation(t *testing.T) { + const ( + chainID = "chain-recovery-without-exact-receipt" + dispatchID = "dispatch-recovery-without-exact-receipt" + firstNodeID = "node-recovery-without-exact-receipt-first" + secondNodeID = "node-recovery-without-exact-receipt-second" + finalNodeID = "node-recovery-without-exact-receipt-final" + firstJobType = "workflow:chain:recovery-without-exact-receipt:first" + secondJobType = "workflow:chain:recovery-without-exact-receipt:second" + finalJobType = "workflow:chain:recovery-without-exact-receipt:final" + ) + tests := []struct { + name string + decorateStore bool + receiptOwner string + recoveredOwner string + advances int + fail bool + wantSuccessor bool + rejectSuccessor bool + }{ + {name: "memory missing receipt live successor", recoveredOwner: "generation-unrecorded", advances: 1, wantSuccessor: true}, + {name: "memory missing receipt rejected successor", recoveredOwner: "generation-unrecorded", advances: 1, wantSuccessor: true, rejectSuccessor: true}, + {name: "memory missing receipt progressed successor", recoveredOwner: "generation-unrecorded", advances: 2}, + {name: "memory missing receipt completed chain", recoveredOwner: "generation-unrecorded", advances: 3}, + {name: "memory missing receipt failed chain", recoveredOwner: "generation-unrecorded", advances: 1, fail: true}, + {name: "decorated store live successor", decorateStore: true, recoveredOwner: "generation-unrecorded", advances: 1, wantSuccessor: true}, + {name: "decorated store progressed successor", decorateStore: true, recoveredOwner: "generation-unrecorded", advances: 2}, + {name: "decorated store completed chain", decorateStore: true, recoveredOwner: "generation-unrecorded", advances: 3}, + {name: "decorated store failed chain", decorateStore: true, recoveredOwner: "generation-unrecorded", advances: 1, fail: true}, + {name: "supported receipt different recovered generation", receiptOwner: "generation-receipt-owner", recoveredOwner: "generation-different", advances: 1, wantSuccessor: true}, + {name: "supported receipt legacy recovery without generation", receiptOwner: "generation-receipt-owner", advances: 1, wantSuccessor: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + baseStore := NewMemoryStore() + nodes := []ChainNode{ + {NodeID: firstNodeID, Job: StoredJob{Type: firstJobType}}, + {NodeID: secondNodeID, Job: StoredJob{Type: secondJobType}}, + {NodeID: finalNodeID, Job: StoredJob{Type: finalJobType}}, + } + if err := baseStore.CreateChain(context.Background(), ChainRecord{ + ChainID: chainID, + DispatchID: dispatchID, + Nodes: nodes, + }); err != nil { + t.Fatalf("create chain: %v", err) + } + delivery := envelope{ + SchemaVersion: schemaVersion, + DispatchID: dispatchID, + Kind: "chain_node", + ChainID: chainID, + NodeID: firstNodeID, + JobID: "job-recovery-without-exact-receipt", + Job: nodes[0].Job, + } + if test.receiptOwner != "" { + advanced, err := requireChainAdvanceStore(t, baseStore).advanceChainOutcome( + context.Background(), + chainID, + firstNodeID, + workflowTransitionClaim(delivery, 2, test.receiptOwner), + ) + if err != nil || !advanced.claimedNow || advanced.done || advanced.next == nil || advanced.next.NodeID != secondNodeID { + t.Fatalf("advance receipt owner = %+v, err:%v", advanced, err) + } + } else { + next, done, err := baseStore.AdvanceChain(context.Background(), chainID, firstNodeID) + if err != nil || done || next == nil || next.NodeID != secondNodeID { + t.Fatalf("advance receiptless predecessor = next:%+v done:%t err:%v", next, done, err) + } + } + for index := 1; index < test.advances; index++ { + next, done, err := baseStore.AdvanceChain(context.Background(), chainID, nodes[index].NodeID) + if err != nil { + t.Fatalf("advance node %d: %v", index, err) + } + if index == len(nodes)-1 { + if !done || next != nil { + t.Fatalf("complete chain = next:%+v done:%t, want nil/true", next, done) + } + continue + } + if done || next == nil || next.NodeID != nodes[index+1].NodeID { + t.Fatalf("advance node %d = next:%+v done:%t", index, next, done) + } + } + if test.fail { + if err := baseStore.FailChain(context.Background(), chainID, errors.New("committed chain failure")); err != nil { + t.Fatalf("fail chain: %v", err) + } + } + + receipt, receiptKnown, err := requireTransitionReceiptStore(t, baseStore).chainTransitionReceipt(context.Background(), chainID, firstNodeID) + if err != nil { + t.Fatalf("read predecessor receipt: %v", err) + } + if test.receiptOwner == "" && receiptKnown { + t.Fatalf("receiptless predecessor unexpectedly persisted receipt %+v", receipt) + } + if test.receiptOwner != "" && (!receiptKnown || receipt.owner.deliveryID != test.receiptOwner) { + t.Fatalf("predecessor receipt = known:%t receipt:%+v", receiptKnown, receipt) + } + + var runtimeStore Store = baseStore + if test.decorateStore { + runtimeStore = &workflowMutationFaultStore{Store: baseStore} + if _, capable := runtimeStore.(transitionReceiptStore); capable { + t.Fatal("decorated compatibility store unexpectedly exposes transition receipts") + } + } + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, runtimeStore) + var applicationCalls, callbackCalls int + for _, jobType := range []string{firstJobType, secondJobType, finalJobType} { + runtime.Register(jobType, func(context.Context, Context) error { + applicationCalls++ + return nil + }) + } + runtime.chainCallbacks[chainID] = chainCallbacks{ + catch: func(context.Context, ChainState, error) error { + callbackCalls++ + return nil + }, + finally: func(context.Context, ChainState) error { + callbackCalls++ + return nil + }, + } + payload, err := json.Marshal(delivery) + if err != nil { + t.Fatalf("encode predecessor delivery: %v", err) + } + handler := queueRuntime.handlers[internalJobChainNode] + if handler == nil { + t.Fatal("chain delivery handler is not registered") + } + var successors []envelope + queueRuntime.handlers[internalJobChainNode] = func(_ context.Context, job busruntime.InboundJob) error { + var successor envelope + if err := job.Bind(&successor); err != nil { + return err + } + successors = append(successors, successor) + return nil + } + var successorErr error + if test.rejectSuccessor { + successorErr = errors.New("receiptless successor enqueue rejected") + queueRuntime.dispatchErr = successorErr + } + recoveryContext := workflowRecoveryContext(exhaustedWorkflowContext(), "generation-recovery-current", test.recoveredOwner) + recoveryErr := handler(recoveryContext, testInboundJob{payload: payload}) + if test.rejectSuccessor { + assertUncommittedMutation(t, recoveryErr, successorErr) + } else if recoveryErr != nil { + t.Fatalf("recover predecessor: %v", recoveryErr) + } + + wantSuccessors := 0 + if test.wantSuccessor && !test.rejectSuccessor { + wantSuccessors = 1 + } + if len(successors) != wantSuccessors { + t.Fatalf("successor dispatches = %d, want %d", len(successors), wantSuccessors) + } + if test.wantSuccessor && !test.rejectSuccessor { + successor := successors[0] + if successor.ChainID != chainID || successor.DispatchID != dispatchID || successor.NodeID != secondNodeID || successor.Job.Type != secondJobType || successor.JobID == "" { + t.Fatalf("successor envelope = %+v, want immediate live successor", successor) + } + } + if applicationCalls != 0 || callbackCalls != 0 || len(recorder.events) != 0 { + t.Fatalf("application/callback/event counts = %d/%d/%d, want 0/0/0", applicationCalls, callbackCalls, len(recorder.events)) + } + state, err := baseStore.GetChain(context.Background(), chainID) + if err != nil { + t.Fatalf("get recovered chain: %v", err) + } + if state.NextIndex != test.advances || state.Completed != (test.advances == len(nodes)) || state.Failed != test.fail { + t.Fatalf("chain state after recovery = %+v", state) + } + }) + } +} + +// TestChainSuccessRecoveryAllowsDifferentPhysicalDeliveryIdentity proves +// duplicate chain rows restore only liveness when another job or attempt owns facts. +func TestChainSuccessRecoveryAllowsDifferentPhysicalDeliveryIdentity(t *testing.T) { + for _, test := range []struct { + name string + currentAttempt int + currentJobID string + }{ + {name: "different physical job", currentAttempt: 2, currentJobID: "job-chain-success-duplicate"}, + {name: "different physical attempt", currentAttempt: 3, currentJobID: "job-chain-success-owner"}, + {name: "negative current attempt", currentAttempt: -1, currentJobID: "job-chain-success-owner"}, + } { + t.Run(test.name, func(t *testing.T) { + const ( + chainID = "chain-success-physical-nonowner" + dispatchID = "dispatch-chain-success-physical-nonowner" + firstNodeID = "node-chain-success-physical-owner" + secondNodeID = "node-chain-success-physical-next" + firstJobType = "workflow:chain:success-physical-owner" + nextJobType = "workflow:chain:success-physical-next" + owner = "generation-chain-success-physical-owner" + ) + store := NewMemoryStore() + nodes := []ChainNode{ + {NodeID: firstNodeID, Job: StoredJob{Type: firstJobType, Payload: []byte(`{"id":4}`)}}, + {NodeID: secondNodeID, Job: StoredJob{Type: nextJobType}}, + } + if err := store.CreateChain(context.Background(), ChainRecord{ChainID: chainID, DispatchID: dispatchID, Nodes: nodes}); err != nil { + t.Fatalf("create chain: %v", err) + } + ownerEnv := envelope{SchemaVersion: schemaVersion, DispatchID: dispatchID, Kind: "chain_node", ChainID: chainID, NodeID: firstNodeID, JobID: "job-chain-success-owner", Job: nodes[0].Job} + advanced, err := requireChainAdvanceStore(t, store).advanceChainOutcome(context.Background(), chainID, firstNodeID, workflowTransitionClaim(ownerEnv, 2, owner)) + if err != nil || !advanced.claimedNow || advanced.done || advanced.next == nil || advanced.next.NodeID != secondNodeID { + t.Fatalf("commit predecessor success = %+v err:%v", advanced, err) + } + + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, store) + var handlerCalls, callbackCalls int + for _, jobType := range []string{firstJobType, nextJobType} { + runtime.Register(jobType, func(context.Context, Context) error { + handlerCalls++ + return nil + }) + } + runtime.chainCallbacks[chainID] = chainCallbacks{finally: func(context.Context, ChainState) error { + callbackCalls++ + return nil + }} + payloadEnv := ownerEnv + payloadEnv.JobID = test.currentJobID + payload, err := json.Marshal(payloadEnv) + if err != nil { + t.Fatalf("encode duplicate predecessor: %v", err) + } + handler := queueRuntime.handlers[internalJobChainNode] + if handler == nil { + t.Fatal("chain delivery handler is not registered") + } + var successors []envelope + queueRuntime.handlers[internalJobChainNode] = func(_ context.Context, job busruntime.InboundJob) error { + var successor envelope + if err := job.Bind(&successor); err != nil { + return err + } + successors = append(successors, successor) + return nil + } + attemptContext := busruntime.WithDeliveryAttempt(context.Background(), busruntime.DeliveryAttempt{Number: test.currentAttempt, MaxRetry: 3}) + recoveryContext, settlement := busruntime.WithDeliverySettlement(attemptContext) + recoveryContext = workflowRecoveryContext(recoveryContext, "generation-chain-success-current", owner) + if err := handler(recoveryContext, testInboundJob{payload: payload}); err != nil { + t.Fatalf("recover physical nonowner predecessor: %v", err) + } + settlement.Commit() + if len(successors) != 1 || successors[0].NodeID != secondNodeID || successors[0].Job.Type != nextJobType || successors[0].JobID == "" { + t.Fatalf("successor dispatches = %+v, want one immediate successor", successors) + } + if handlerCalls != 0 || callbackCalls != 0 || settlement.ApplicationStateCommitted() || len(recorder.events) != 0 { + t.Fatalf("handler/callback/committed/events = %d/%d/%t/%d, want 0/0/false/0", handlerCalls, callbackCalls, settlement.ApplicationStateCommitted(), len(recorder.events)) + } + }) + } +} + +// TestChainSuccessRecoveryRejectsInvalidReceiptShapeBeforeLiveness proves a +// malformed supported receipt cannot emit facts or restore a continuation. +func TestChainSuccessRecoveryRejectsInvalidReceiptShapeBeforeLiveness(t *testing.T) { + const ( + chainID = "chain-invalid-success-receipt-shape" + dispatchID = "dispatch-invalid-success-receipt-shape" + firstNodeID = "node-invalid-success-receipt-shape-first" + secondNodeID = "node-invalid-success-receipt-shape-second" + firstJobType = "workflow:chain:invalid-success-receipt-shape:first" + secondJobType = "workflow:chain:invalid-success-receipt-shape:second" + owner = "generation-invalid-success-receipt-shape" + ) + tests := []struct { + name string + final bool + recoveredOwner string + diagnostic string + mutate func(*transitionReceipt) + }{ + { + name: "nonfinal completion exact owner", + recoveredOwner: owner, + diagnostic: "completion", + mutate: func(receipt *transitionReceipt) { + receipt.aggregateCompleted = true + }, + }, + { + name: "nonfinal cancellation different owner", + recoveredOwner: "generation-different-owner", + diagnostic: "cancellation", + mutate: func(receipt *transitionReceipt) { + receipt.aggregateCancelled = true + }, + }, + { + name: "final missing completion legacy recovery", + final: true, + diagnostic: "completion", + mutate: func(receipt *transitionReceipt) { + receipt.aggregateCompleted = false + }, + }, + { + name: "final cancellation different owner", + final: true, + recoveredOwner: "generation-different-owner", + diagnostic: "cancellation", + mutate: func(receipt *transitionReceipt) { + receipt.aggregateCancelled = true + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + store := NewMemoryStore().(*memoryStore) + nodes := []ChainNode{{NodeID: firstNodeID, Job: StoredJob{Type: firstJobType}}} + if !test.final { + nodes = append(nodes, ChainNode{NodeID: secondNodeID, Job: StoredJob{Type: secondJobType}}) + } + if err := store.CreateChain(context.Background(), ChainRecord{ChainID: chainID, DispatchID: dispatchID, Nodes: nodes}); err != nil { + t.Fatalf("create chain: %v", err) + } + delivery := envelope{ + SchemaVersion: schemaVersion, + DispatchID: dispatchID, + Kind: "chain_node", + ChainID: chainID, + NodeID: firstNodeID, + JobID: "job-invalid-success-receipt-shape", + Job: nodes[0].Job, + } + advanced, err := store.advanceChainOutcome(context.Background(), chainID, firstNodeID, workflowTransitionClaim(delivery, 2, owner)) + if err != nil || !advanced.claimedNow || !advanced.receiptKnown || advanced.done != test.final { + t.Fatalf("advance chain = %+v err:%v", advanced, err) + } + key := transitionReceiptKey{workflowKind: chainTransitionKind, workflowID: chainID, memberID: firstNodeID} + store.mu.Lock() + receipt := store.transitionReceipts[key] + test.mutate(&receipt) + store.transitionReceipts[key] = receipt + store.mu.Unlock() + + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, store) + var applicationCalls, callbackCalls, successorDispatches int + for _, jobType := range []string{firstJobType, secondJobType} { + runtime.Register(jobType, func(context.Context, Context) error { + applicationCalls++ + return nil + }) + } + runtime.chainCallbacks[chainID] = chainCallbacks{ + catch: func(context.Context, ChainState, error) error { + callbackCalls++ + return nil + }, + finally: func(context.Context, ChainState) error { + callbackCalls++ + return nil + }, + } + payload, err := json.Marshal(delivery) + if err != nil { + t.Fatalf("encode delivery: %v", err) + } + handler := queueRuntime.handlers[internalJobChainNode] + if handler == nil { + t.Fatal("chain delivery handler is not registered") + } + queueRuntime.handlers[internalJobChainNode] = func(context.Context, busruntime.InboundJob) error { + successorDispatches++ + return nil + } + recoveryContext := workflowRecoveryContext(exhaustedWorkflowContext(), "generation-invalid-shape-current", test.recoveredOwner) + recoveryErr := handler(recoveryContext, testInboundJob{payload: payload}) + if !busruntime.IsUncommitted(recoveryErr) || !strings.Contains(recoveryErr.Error(), test.diagnostic) { + t.Fatalf("invalid receipt recovery error = %v, want uncommitted %q diagnostic", recoveryErr, test.diagnostic) + } + if applicationCalls != 0 || callbackCalls != 0 || successorDispatches != 0 || len(recorder.events) != 0 { + t.Fatalf("application/callback/successor/event counts = %d/%d/%d/%d, want 0/0/0/0", applicationCalls, callbackCalls, successorDispatches, len(recorder.events)) + } + }) + } +} + +// TestChainRecoveredPredecessorDoesNotRedispatchAfterTerminalSuccessor proves +// a stale parent can release its own winner facts without repeating terminal work. +func TestChainRecoveredPredecessorDoesNotRedispatchAfterTerminalSuccessor(t *testing.T) { + const ( + chainID = "chain-recovered-predecessor" + firstNodeID = "node-recovered-predecessor" + finalNodeID = "node-recovered-final" + firstJobType = "workflow:chain:recovered-predecessor" + finalJobType = "workflow:chain:recovered-final" + ) + store := NewMemoryStore() + if err := store.CreateChain(context.Background(), ChainRecord{ + ChainID: chainID, + DispatchID: "dispatch-recovered-predecessor", + Nodes: []ChainNode{ + {NodeID: firstNodeID, Job: StoredJob{Type: firstJobType}}, + {NodeID: finalNodeID, Job: StoredJob{Type: finalJobType}}, + }, + }); err != nil { + t.Fatalf("create chain: %v", err) + } + delivery := envelope{ + SchemaVersion: schemaVersion, + DispatchID: "dispatch-recovered-predecessor", + Kind: "chain_node", + ChainID: chainID, + NodeID: firstNodeID, + JobID: "job-recovered-predecessor", + Job: StoredJob{Type: firstJobType}, + } + if advanced, err := requireChainAdvanceStore(t, store).advanceChainOutcome(context.Background(), chainID, firstNodeID, workflowTransitionClaim(delivery, 2, "generation-chain-predecessor")); err != nil || !advanced.claimedNow { + t.Fatalf("advance predecessor = %+v, err:%v", advanced, err) + } + if _, done, err := store.AdvanceChain(context.Background(), chainID, finalNodeID); err != nil || !done { + t.Fatalf("complete successor = done:%t err:%v", done, err) + } + + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, store) + var firstCalls, finalCalls, finallyCalls int + runtime.Register(firstJobType, func(context.Context, Context) error { + firstCalls++ + return busruntime.Permanent(errors.New("contradictory predecessor failure")) + }) + runtime.Register(finalJobType, func(context.Context, Context) error { + finalCalls++ + return nil + }) + runtime.chainCallbacks[chainID] = chainCallbacks{ + finally: func(context.Context, ChainState) error { + finallyCalls++ + return nil + }, + } + deliveryContext, settlement := busruntime.WithDeliverySettlement(exhaustedWorkflowContext()) + deliveryContext = workflowRecoveryContext(deliveryContext, "generation-chain-predecessor-replay", "generation-chain-predecessor") + if err := queueRuntime.DispatchJSON(deliveryContext, internalJobChainNode, delivery); err != nil { + t.Fatalf("recover predecessor: %v", err) + } + if firstCalls != 0 || finalCalls != 0 || finallyCalls != 0 { + t.Fatalf("handler/callback calls = first:%d final:%d finally:%d, want 0/0/0", firstCalls, finalCalls, finallyCalls) + } + assertNoCommittedEvents(t, recorder.events, EventJobSucceeded, EventChainAdvanced, EventChainCompleted) + settlement.Commit() + + var succeeded, advanced, completed, failed int + for _, event := range recorder.events { + switch event.Kind { + case EventJobSucceeded: + succeeded++ + case EventChainAdvanced: + advanced++ + case EventChainCompleted: + completed++ + case EventJobFailed, EventChainFailed: + failed++ + } + } + if succeeded != 1 || advanced != 1 || completed != 0 || failed != 0 { + t.Fatalf("job/advance/completion/failure counts = %d/%d/%d/%d, want 1/1/0/0", succeeded, advanced, completed, failed) + } +} + +// TestChainSuccessorRejectionRecoversWithoutPredecessorReplay proves exact +// receipt recovery restores a continuation stranded by definite rejection. +func TestChainSuccessorRejectionRecoversWithoutPredecessorReplay(t *testing.T) { + const ( + chainID = "chain-recovered-successful-predecessor" + dispatchID = "dispatch-recovered-successful-predecessor" + firstNodeID = "node-recovered-successful-predecessor" + finalNodeID = "node-recovered-successful-final" + firstJobType = "workflow:chain:recovered-successful-predecessor" + finalJobType = "workflow:chain:recovered-successful-final" + owner = "generation-chain-successful-predecessor" + ) + store := NewMemoryStore() + if err := store.CreateChain(context.Background(), ChainRecord{ + ChainID: chainID, + DispatchID: dispatchID, + Nodes: []ChainNode{ + {NodeID: firstNodeID, Job: StoredJob{Type: firstJobType}}, + {NodeID: finalNodeID, Job: StoredJob{Type: finalJobType}}, + }, + }); err != nil { + t.Fatalf("create chain: %v", err) + } + delivery := envelope{ + SchemaVersion: schemaVersion, + DispatchID: dispatchID, + Kind: "chain_node", + ChainID: chainID, + NodeID: firstNodeID, + JobID: "job-recovered-successful-predecessor", + Job: StoredJob{Type: firstJobType}, + } + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, store) + var firstCalls, finalCalls int + runtime.Register(firstJobType, func(context.Context, Context) error { + firstCalls++ + return nil + }) + runtime.Register(finalJobType, func(context.Context, Context) error { + finalCalls++ + return nil + }) + payload, err := json.Marshal(delivery) + if err != nil { + t.Fatalf("encode predecessor delivery: %v", err) + } + handler := queueRuntime.handlers[internalJobChainNode] + if handler == nil { + t.Fatal("chain delivery handler is not registered") + } + + successorErr := errors.New("successor enqueue rejected") + queueRuntime.dispatchErr = successorErr + initialContext, initialSettlement := busruntime.WithDeliverySettlement(exhaustedWorkflowContext()) + initialContext = workflowGenerationContext(initialContext, owner) + initialErr := handler(initialContext, testInboundJob{payload: payload}) + if !busruntime.IsUncommitted(initialErr) || !errors.Is(initialErr, successorErr) { + t.Fatalf("initial predecessor error = %v, want uncommitted successor rejection", initialErr) + } + if firstCalls != 1 || finalCalls != 0 || !initialSettlement.ApplicationStateCommitted() { + t.Fatalf("initial calls/application-state signal = %d/%d/%t, want 1/0/true", firstCalls, finalCalls, initialSettlement.ApplicationStateCommitted()) + } + receiptStore := requireTransitionReceiptStore(t, store) + receipt, known, err := receiptStore.chainTransitionReceipt(context.Background(), chainID, firstNodeID) + if err != nil || !known || receipt.owner.deliveryID != owner { + t.Fatalf("initial predecessor receipt = known:%t receipt:%+v err:%v", known, receipt, err) + } + assertNoCommittedEvents(t, recorder.events, EventJobSucceeded, EventChainAdvanced, EventChainCompleted) + + for index, generationID := range []string{"generation-chain-recovery-one", "generation-chain-recovery-two"} { + recoveryContext, recoverySettlement := busruntime.WithDeliverySettlement(exhaustedWorkflowContext()) + recoveryContext = workflowRecoveryContext(recoveryContext, generationID, owner) + recoveryErr := handler(recoveryContext, testInboundJob{payload: payload}) + if !busruntime.IsUncommitted(recoveryErr) || !errors.Is(recoveryErr, successorErr) { + t.Fatalf("recovery %d error = %v, want uncommitted successor rejection", index+1, recoveryErr) + } + if recoverySettlement.ApplicationStateCommitted() { + t.Fatalf("recovery %d marked current generation as transition owner", index+1) + } + receipt, known, err = receiptStore.chainTransitionReceipt(context.Background(), chainID, firstNodeID) + if err != nil || !known || receipt.owner.deliveryID != owner { + t.Fatalf("recovery %d predecessor receipt = known:%t receipt:%+v err:%v", index+1, known, receipt, err) + } + } + if firstCalls != 1 || finalCalls != 0 { + t.Fatalf("handler calls after repeated rejection = first:%d final:%d, want 1/0", firstCalls, finalCalls) + } + assertNoCommittedEvents(t, recorder.events, EventJobSucceeded, EventChainAdvanced, EventChainCompleted) + + queueRuntime.dispatchErr = nil + recoveryContext, settlement := busruntime.WithDeliverySettlement(exhaustedWorkflowContext()) + recoveryContext = workflowRecoveryContext(recoveryContext, "generation-chain-recovery-success", owner) + if err := handler(recoveryContext, testInboundJob{payload: payload}); err != nil { + t.Fatalf("recover successful predecessor: %v", err) + } + if firstCalls != 1 || finalCalls != 1 { + t.Fatalf("handler calls after recovery = first:%d final:%d, want 1/1", firstCalls, finalCalls) + } + state, err := store.GetChain(context.Background(), chainID) + if err != nil { + t.Fatalf("get recovered chain: %v", err) + } + if !state.Completed || state.Failed || state.NextIndex != 2 { + t.Fatalf("recovered chain state = %+v, want completed", state) + } + assertNoCommittedEvents(t, recorder.events, EventJobSucceeded, EventChainAdvanced, EventChainCompleted) + settlement.Commit() + + var succeeded, advanced, completed int + for _, event := range recorder.events { + switch event.Kind { + case EventJobSucceeded: + succeeded++ + case EventChainAdvanced: + advanced++ + case EventChainCompleted: + completed++ + } + } + if succeeded != 2 || advanced != 1 || completed != 1 { + t.Fatalf("job/advance/completion counts = %d/%d/%d, want 2/1/1", succeeded, advanced, completed) + } +} + +// TestChainRecoveredPredecessorDoesNotRedispatchAfterSuccessorProgress proves +// recovery cannot duplicate a continuation after the immediate successor won. +func TestChainRecoveredPredecessorDoesNotRedispatchAfterSuccessorProgress(t *testing.T) { + const ( + chainID = "chain-recovered-progressed-successor" + dispatchID = "dispatch-recovered-progressed-successor" + firstNodeID = "node-recovered-progressed-first" + secondNodeID = "node-recovered-progressed-second" + finalNodeID = "node-recovered-progressed-final" + firstJobType = "workflow:chain:recovered-progressed-first" + finalJobType = "workflow:chain:recovered-progressed-final" + owner = "generation-chain-progressed-predecessor" + ) + store := NewMemoryStore() + if err := store.CreateChain(context.Background(), ChainRecord{ + ChainID: chainID, + DispatchID: dispatchID, + Nodes: []ChainNode{ + {NodeID: firstNodeID, Job: StoredJob{Type: firstJobType}}, + {NodeID: secondNodeID, Job: StoredJob{Type: "workflow:chain:recovered-progressed-second"}}, + {NodeID: finalNodeID, Job: StoredJob{Type: finalJobType}}, + }, + }); err != nil { + t.Fatalf("create chain: %v", err) + } + delivery := envelope{ + SchemaVersion: schemaVersion, + DispatchID: dispatchID, + Kind: "chain_node", + ChainID: chainID, + NodeID: firstNodeID, + JobID: "job-recovered-progressed-first", + Job: StoredJob{Type: firstJobType}, + } + if advanced, err := requireChainAdvanceStore(t, store).advanceChainOutcome(context.Background(), chainID, firstNodeID, workflowTransitionClaim(delivery, 2, owner)); err != nil || !advanced.claimedNow { + t.Fatalf("advance predecessor = %+v, err:%v", advanced, err) + } + if next, done, err := store.AdvanceChain(context.Background(), chainID, secondNodeID); err != nil || done || next == nil || next.NodeID != finalNodeID { + t.Fatalf("advance successor = next:%+v done:%t err:%v", next, done, err) + } + + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, store) + var firstCalls, finalCalls int + runtime.Register(firstJobType, func(context.Context, Context) error { + firstCalls++ + return nil + }) + runtime.Register(finalJobType, func(context.Context, Context) error { + finalCalls++ + return nil + }) + recoveryContext, settlement := busruntime.WithDeliverySettlement(exhaustedWorkflowContext()) + recoveryContext = workflowRecoveryContext(recoveryContext, "generation-chain-progressed-recovery", owner) + if err := queueRuntime.DispatchJSON(recoveryContext, internalJobChainNode, delivery); err != nil { + t.Fatalf("recover progressed predecessor: %v", err) + } + if firstCalls != 0 || finalCalls != 0 { + t.Fatalf("handler calls = first:%d final:%d, want 0/0", firstCalls, finalCalls) + } + state, err := store.GetChain(context.Background(), chainID) + if err != nil { + t.Fatalf("get progressed chain: %v", err) + } + if state.NextIndex != 2 || state.Completed || state.Failed { + t.Fatalf("progressed chain state = %+v, want active final node", state) + } + assertNoCommittedEvents(t, recorder.events, EventJobSucceeded, EventChainAdvanced, EventChainCompleted) + settlement.Commit() + var succeeded, advanced, completed int + for _, event := range recorder.events { + switch event.Kind { + case EventJobSucceeded: + succeeded++ + case EventChainAdvanced: + advanced++ + case EventChainCompleted: + completed++ + } + } + if succeeded != 1 || advanced != 1 || completed != 0 { + t.Fatalf("recovered fact counts = job:%d advanced:%d completed:%d, want 1/1/0", succeeded, advanced, completed) + } +} + +// TestRecoverCommittedChainSuccessRejectsInconsistentState covers every +// validation boundary before recovery can publish a persisted winner fact. +func TestRecoverCommittedChainSuccessRejectsInconsistentState(t *testing.T) { + baseNode := ChainNode{NodeID: "node-recovery-validation", Job: StoredJob{Type: "workflow:chain:recovery-validation"}} + tests := []struct { + name string + state ChainState + withoutProof bool + wantRecovered bool + wantErr bool + }{ + { + name: "unknown node", + state: ChainState{ChainID: "chain-recovery-validation", Nodes: []ChainNode{{NodeID: "other-node"}}, NextIndex: 1, Completed: true}, + wantErr: true, + }, + { + name: "negative next index", + state: ChainState{ChainID: "chain-recovery-validation", Nodes: []ChainNode{baseNode}, NextIndex: -1}, + wantErr: true, + }, + { + name: "oversized next index", + state: ChainState{ChainID: "chain-recovery-validation", Nodes: []ChainNode{baseNode}, NextIndex: 2}, + wantErr: true, + }, + { + name: "all nodes advanced without completion", + state: ChainState{ + ChainID: "chain-recovery-validation", + Nodes: []ChainNode{baseNode, {NodeID: "node-recovery-final"}}, + NextIndex: 2, + }, + wantErr: true, + }, + { + name: "completed before all nodes advanced", + state: ChainState{ + ChainID: "chain-recovery-validation", + Nodes: []ChainNode{baseNode, {NodeID: "node-recovery-final"}}, + NextIndex: 1, + Completed: true, + }, + wantErr: true, + }, + { + name: "completed before node advance", + state: ChainState{ChainID: "chain-recovery-validation", Nodes: []ChainNode{baseNode}, Completed: true}, + wantErr: true, + }, + { + name: "final node advanced without completion", + state: ChainState{ChainID: "chain-recovery-validation", Nodes: []ChainNode{baseNode}, NextIndex: 1}, + wantErr: true, + }, + { + name: "current node remains unsettled", + state: ChainState{ChainID: "chain-recovery-validation", Nodes: []ChainNode{baseNode}}, + }, + { + name: "chain identity mismatch", + state: ChainState{ChainID: "different-chain", Nodes: []ChainNode{baseNode}, NextIndex: 1, Completed: true}, + wantErr: true, + }, + { + name: "dispatch identity mismatch", + state: ChainState{ + ChainID: "chain-recovery-validation", + DispatchID: "different-dispatch", + Nodes: []ChainNode{baseNode}, + NextIndex: 1, + Completed: true, + }, + wantErr: true, + }, + { + name: "persisted job type mismatch", + state: ChainState{ChainID: "chain-recovery-validation", Nodes: []ChainNode{{NodeID: baseNode.NodeID, Job: StoredJob{Type: "different-job"}}}, NextIndex: 1, Completed: true}, + wantErr: true, + }, + { + name: "persisted job payload mismatch", + state: ChainState{ + ChainID: "chain-recovery-validation", + Nodes: []ChainNode{{NodeID: baseNode.NodeID, Job: StoredJob{Type: baseNode.Job.Type, Payload: []byte(`{"different":true}`)}}}, + NextIndex: 1, + Completed: true, + }, + wantErr: true, + }, + { + name: "persisted job options mismatch", + state: ChainState{ + ChainID: "chain-recovery-validation", + Nodes: []ChainNode{{NodeID: baseNode.NodeID, Job: StoredJob{Type: baseNode.Job.Type, Options: JobOptions{Queue: "different"}}}}, + NextIndex: 1, + Completed: true, + }, + wantErr: true, + }, + { + name: "missing recovery proof", + state: ChainState{ChainID: "chain-recovery-validation", Nodes: []ChainNode{baseNode}, NextIndex: 1, Completed: true}, + withoutProof: true, + }, + { + name: "receipt capability absent", + state: ChainState{ChainID: "chain-recovery-validation", Nodes: []ChainNode{baseNode}, NextIndex: 1, Completed: true}, + wantRecovered: true, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + faultStore := &workflowMutationFaultStore{Store: NewMemoryStore(), getChainState: &test.state} + runtime, _, recorder := newWorkflowMutationRuntime(t, faultStore) + env := envelope{ + DispatchID: "dispatch-recovery-validation", + ChainID: "chain-recovery-validation", + NodeID: baseNode.NodeID, + JobID: "job-recovery-validation", + Job: baseNode.Job, + } + recoveryContext := context.Background() + if !test.withoutProof { + recoveryContext = workflowRecoveryContext(recoveryContext, "generation-chain-validation-current", "generation-chain-validation-recovered") + } + recovered, err := runtime.recoverCommittedChainSuccess(recoveryContext, env) + wantRecovered := test.wantRecovered || test.wantErr + if (err != nil) != test.wantErr || recovered != wantRecovered { + t.Fatalf("recovery = recovered:%t err:%v, want recovered:%t err:%t", recovered, err, wantRecovered, test.wantErr) + } + if err != nil && !busruntime.IsUncommitted(err) { + t.Fatalf("recovery validation error = %v, want uncommitted", err) + } + assertNoCommittedEvents(t, recorder.events, EventJobSucceeded, EventChainAdvanced, EventChainCompleted) + }) + } +} + +// TestBatchCommittedSuccessSurvivesContradictorySettlementReplay proves a +// terminal member's receipt remains observable after settlement failure +// without re-executing application code that could contradict it. +func TestBatchCommittedSuccessSurvivesContradictorySettlementReplay(t *testing.T) { + const ( + batchID = "batch-committed-success-settlement-replay" + jobID = "job-committed-success-settlement-replay" + jobType = "workflow:batch:committed-success-settlement-replay" + ) + store := NewMemoryStore() + if err := store.CreateBatch(context.Background(), BatchRecord{ + BatchID: batchID, + DispatchID: "dispatch-committed-success-settlement-replay", + Jobs: []BatchJob{{JobID: jobID, Job: StoredJob{Type: jobType}}}, + }); err != nil { + t.Fatalf("create batch: %v", err) + } + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, store) + var handlerCalls int + runtime.Register(jobType, func(context.Context, Context) error { + handlerCalls++ + if handlerCalls == 1 { + return nil + } + return busruntime.Permanent(errors.New("contradictory replay failure")) + }) + delivery := envelope{ + SchemaVersion: schemaVersion, + DispatchID: "dispatch-committed-success-settlement-replay", + Kind: "batch_job", + BatchID: batchID, + JobID: jobID, + Job: StoredJob{Type: jobType}, + } + + firstContext, _ := busruntime.WithDeliverySettlement(exhaustedWorkflowContext()) + firstContext = workflowGenerationContext(firstContext, "generation-batch-committed-success") + if err := queueRuntime.DispatchJSON(firstContext, internalJobBatchJob, delivery); err != nil { + t.Fatalf("first delivery: %v", err) + } + assertNoCommittedEvents(t, recorder.events, EventJobSucceeded, EventBatchProgressed, EventBatchCompleted) + state, err := store.GetBatch(context.Background(), batchID) + if err != nil { + t.Fatalf("get committed batch: %v", err) + } + if !state.Completed || state.Cancelled || state.Processed != 1 || state.Failed != 0 { + t.Fatalf("committed batch state = %+v, want successful completion", state) + } + + replayContext, replaySettlement := busruntime.WithDeliverySettlement(exhaustedWorkflowContext()) + replayContext = workflowRecoveryContext(replayContext, "generation-batch-replay", "generation-batch-committed-success") + if err := queueRuntime.DispatchJSON(replayContext, internalJobBatchJob, delivery); err != nil { + t.Fatalf("contradictory redelivery: %v", err) + } + assertNoCommittedEvents(t, recorder.events, EventJobSucceeded, EventBatchProgressed, EventBatchCompleted) + replaySettlement.Commit() + + var succeeded, progressed, completed, failed int + for _, event := range recorder.events { + switch event.Kind { + case EventJobSucceeded: + succeeded++ + case EventBatchProgressed: + progressed++ + case EventBatchCompleted: + completed++ + case EventJobFailed, EventBatchFailed, EventBatchCancelled: + failed++ + } + } + if handlerCalls != 1 || succeeded != 1 || progressed != 1 || completed != 1 || failed != 0 { + t.Fatalf("handler/job/progress/completion/failure counts = %d/%d/%d/%d/%d, want 1/1/1/1/0", handlerCalls, succeeded, progressed, completed, failed) + } +} + +// TestBatchCompletionReceiptIdentifiesCompletingMember proves member facts +// and aggregate completion are recovered only from their exact receipt owners. +func TestBatchCompletionReceiptIdentifiesCompletingMember(t *testing.T) { + const ( + batchID = "batch-recovery-terminal-owner" + dispatchID = "dispatch-batch-recovery-terminal-owner" + staleJobID = "job-batch-recovery-stale" + terminalJobID = "job-batch-recovery-terminal" + staleJobType = "workflow:batch:recovery-stale" + terminalType = "workflow:batch:recovery-terminal" + ) + store := NewMemoryStore() + if err := store.CreateBatch(context.Background(), BatchRecord{ + BatchID: batchID, + DispatchID: dispatchID, + Jobs: []BatchJob{ + {JobID: staleJobID, Job: StoredJob{Type: staleJobType}}, + {JobID: terminalJobID, Job: StoredJob{Type: terminalType}}, + }, + }); err != nil { + t.Fatalf("create batch: %v", err) + } + stale := envelope{SchemaVersion: schemaVersion, DispatchID: dispatchID, Kind: "batch_job", BatchID: batchID, JobID: staleJobID, Job: StoredJob{Type: staleJobType}} + terminal := envelope{SchemaVersion: schemaVersion, DispatchID: dispatchID, Kind: "batch_job", BatchID: batchID, JobID: terminalJobID, Job: StoredJob{Type: terminalType}} + settlements := requireBatchSettlementStore(t, store) + staleResult, err := settlements.settleBatchOutcome(context.Background(), batchID, staleJobID, BatchJobSucceeded, nil, workflowTransitionClaim(stale, 2, "generation-batch-stale")) + if err != nil || !staleResult.claimedNow || staleResult.state.Completed || staleResult.receipt.aggregateCompleted { + t.Fatalf("settle stale member = %+v, err:%v", staleResult, err) + } + terminalResult, err := settlements.settleBatchOutcome(context.Background(), batchID, terminalJobID, BatchJobSucceeded, nil, workflowTransitionClaim(terminal, 2, "generation-batch-terminal")) + if err != nil || !terminalResult.claimedNow || !terminalResult.state.Completed || !terminalResult.receipt.aggregateCompleted { + t.Fatalf("settle terminal member = %+v, err:%v", terminalResult, err) + } + + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, store) + var handlerCalls int + runtime.Register(staleJobType, func(context.Context, Context) error { + handlerCalls++ + return busruntime.Permanent(errors.New("stale handler must not run")) + }) + runtime.Register(terminalType, func(context.Context, Context) error { + handlerCalls++ + return busruntime.Permanent(errors.New("terminal handler must not run")) + }) + + staleContext, staleSettlement := busruntime.WithDeliverySettlement(exhaustedWorkflowContext()) + staleContext = workflowRecoveryContext(staleContext, "generation-batch-stale-replay", "generation-batch-stale") + if err := queueRuntime.DispatchJSON(staleContext, internalJobBatchJob, stale); err != nil { + t.Fatalf("recover stale member: %v", err) + } + staleSettlement.Commit() + assertNoCommittedEvents(t, recorder.events, EventBatchCompleted) + + terminalContext, terminalSettlement := busruntime.WithDeliverySettlement(exhaustedWorkflowContext()) + terminalContext = workflowRecoveryContext(terminalContext, "generation-batch-terminal-replay", "generation-batch-terminal") + if err := queueRuntime.DispatchJSON(terminalContext, internalJobBatchJob, terminal); err != nil { + t.Fatalf("recover terminal member: %v", err) + } + terminalSettlement.Commit() + + var staleSucceeded, terminalSucceeded, progressed, completed int + completedJobID := "" + for _, event := range recorder.events { + switch event.Kind { + case EventJobSucceeded: + if event.JobID == staleJobID { + staleSucceeded++ + } + if event.JobID == terminalJobID { + terminalSucceeded++ + } + case EventBatchProgressed: + progressed++ + case EventBatchCompleted: + completed++ + completedJobID = event.JobID + } + } + if handlerCalls != 0 || staleSucceeded != 1 || terminalSucceeded != 1 || progressed != 2 || completed != 1 || completedJobID != terminalJobID { + t.Fatalf("handlers/stale/terminal/progress/completion/completer = %d/%d/%d/%d/%d/%q, want 0/1/1/2/1/%q", handlerCalls, staleSucceeded, terminalSucceeded, progressed, completed, completedJobID, terminalJobID) + } +} + +// TestBatchCompletionReceiptSurvivesFailedCompletingMember proves an +// allow-failures aggregate can recover completion even though the member error +// itself is intentionally absent from the durable receipt. +func TestBatchCompletionReceiptSurvivesFailedCompletingMember(t *testing.T) { + const ( + batchID = "batch-recovery-failed-completer" + dispatchID = "dispatch-batch-recovery-failed-completer" + firstJobID = "job-batch-recovery-first" + finalJobID = "job-batch-recovery-failed-completer" + finalType = "workflow:batch:recovery-failed-completer" + ) + store := NewMemoryStore() + if err := store.CreateBatch(context.Background(), BatchRecord{ + BatchID: batchID, + DispatchID: dispatchID, + AllowFailed: true, + Jobs: []BatchJob{ + {JobID: firstJobID, Job: StoredJob{Type: "workflow:batch:recovery-first"}}, + {JobID: finalJobID, Job: StoredJob{Type: finalType}}, + }, + }); err != nil { + t.Fatalf("create batch: %v", err) + } + first := envelope{DispatchID: dispatchID, BatchID: batchID, JobID: firstJobID, Job: StoredJob{Type: "workflow:batch:recovery-first"}} + final := envelope{SchemaVersion: schemaVersion, DispatchID: dispatchID, Kind: "batch_job", BatchID: batchID, JobID: finalJobID, Job: StoredJob{Type: finalType}} + settlements := requireBatchSettlementStore(t, store) + if result, err := settlements.settleBatchOutcome(context.Background(), batchID, firstJobID, BatchJobSucceeded, nil, workflowTransitionClaim(first, 2, "generation-batch-first")); err != nil || !result.claimedNow || result.state.Completed { + t.Fatalf("settle first member = %+v, err:%v", result, err) + } + originalCause := errors.New("durable cause remains delivery-local") + result, err := settlements.settleBatchOutcome(context.Background(), batchID, finalJobID, BatchJobFailed, originalCause, workflowTransitionClaim(final, 2, "generation-batch-failed-completer")) + if err != nil || !result.claimedNow || !result.state.Completed || result.state.Cancelled || !result.receipt.aggregateCompleted { + t.Fatalf("settle failed completer = %+v, err:%v", result, err) + } + + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, store) + var handlerCalls int + runtime.Register(finalType, func(context.Context, Context) error { + handlerCalls++ + return errors.New("handler must not run") + }) + recoveryContext, settlement := busruntime.WithDeliverySettlement(exhaustedWorkflowContext()) + recoveryContext = workflowRecoveryContext(recoveryContext, "generation-batch-failed-completer-replay", "generation-batch-failed-completer") + recoveryErr := queueRuntime.DispatchJSON(recoveryContext, internalJobBatchJob, final) + if !busruntime.IsPermanent(recoveryErr) || busruntime.IsUncommitted(recoveryErr) || errors.Is(recoveryErr, originalCause) { + t.Fatalf("recover failed completer error = %v, want generic permanent settlement without original cause", recoveryErr) + } + assertNoCommittedEvents(t, recorder.events, EventBatchCompleted) + settlement.Commit() + nestedContext, nestedSettlement := busruntime.WithDeliverySettlement(exhaustedWorkflowContext()) + nestedContext = workflowRecoveryContext(nestedContext, "generation-batch-failed-completer-replay-2", "generation-batch-failed-completer-replay") + nestedErr := queueRuntime.DispatchJSON(nestedContext, internalJobBatchJob, final) + if !busruntime.IsPermanent(nestedErr) || busruntime.IsUncommitted(nestedErr) || errors.Is(nestedErr, originalCause) { + t.Fatalf("recover failed completer after another unsettled generation = %v, want generic permanent settlement", nestedErr) + } + nestedSettlement.Commit() + + var completed, memberFacts int + for _, event := range recorder.events { + switch event.Kind { + case EventBatchCompleted: + completed++ + if event.JobID != finalJobID { + t.Fatalf("completion job id = %q, want %q", event.JobID, finalJobID) + } + case EventJobSucceeded, EventJobFailed, EventBatchProgressed: + memberFacts++ + } + } + if handlerCalls != 0 || completed != 1 || memberFacts != 0 { + t.Fatalf("handler/completion/member facts = %d/%d/%d, want 0/1/0", handlerCalls, completed, memberFacts) + } +} + +// TestBatchRecoveryRejectsInvalidAggregateReceiptShape proves corrupt terminal +// ownership cannot acknowledge a recovered member or publish partial facts. +func TestBatchRecoveryRejectsInvalidAggregateReceiptShape(t *testing.T) { + tests := []struct { + name string + diagnostic string + mutate func(*memoryStore, transitionReceiptKey) + }{ + {name: "completion for nonterminal state", diagnostic: "nonterminal state", mutate: func(store *memoryStore, _ transitionReceiptKey) { + state := &store.batch["batch-invalid-aggregate-receipt"].state + state.Total = 2 + state.Pending = 1 + state.Processed = 1 + state.Completed = false + }}, + {name: "cancellation without completion", diagnostic: "cancellation is not completed", mutate: func(store *memoryStore, key transitionReceiptKey) { + receipt := store.transitionReceipts[key] + receipt.aggregateCompleted = false + receipt.aggregateCancelled = true + store.transitionReceipts[key] = receipt + }}, + {name: "cancellation owns success", diagnostic: "does not own failure", mutate: func(store *memoryStore, key transitionReceiptKey) { + receipt := store.transitionReceipts[key] + receipt.aggregateCancelled = true + store.transitionReceipts[key] = receipt + store.batch["batch-invalid-aggregate-receipt"].state.Cancelled = true + }}, + {name: "cancellation disagrees with state", diagnostic: "does not match aggregate state", mutate: func(store *memoryStore, key transitionReceiptKey) { + receipt := store.transitionReceipts[key] + receipt.outcome = BatchJobFailed + receipt.aggregateCancelled = true + store.transitionReceipts[key] = receipt + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + const ( + batchID = "batch-invalid-aggregate-receipt" + dispatchID = "dispatch-invalid-aggregate-receipt" + jobID = "job-invalid-aggregate-receipt" + jobType = "workflow:batch:invalid-aggregate-receipt" + owner = "generation-invalid-aggregate-receipt" + ) + store := NewMemoryStore().(*memoryStore) + env := envelope{SchemaVersion: schemaVersion, DispatchID: dispatchID, Kind: "batch_job", BatchID: batchID, JobID: jobID, Job: StoredJob{Type: jobType}} + if err := store.CreateBatch(context.Background(), BatchRecord{BatchID: batchID, DispatchID: dispatchID, Jobs: []BatchJob{{JobID: jobID, Job: env.Job}}}); err != nil { + t.Fatalf("create batch: %v", err) + } + if settled, err := store.settleBatchOutcome(context.Background(), batchID, jobID, BatchJobSucceeded, nil, workflowTransitionClaim(env, 2, owner)); err != nil || !settled.receiptKnown || !settled.receipt.aggregateCompleted { + t.Fatalf("settle batch = %+v err:%v", settled, err) + } + key := transitionReceiptKey{workflowKind: batchTransitionKind, workflowID: batchID, memberID: jobID} + store.mu.Lock() + test.mutate(store, key) + store.mu.Unlock() + + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, store) + var handlerCalls, callbackCalls int + runtime.Register(jobType, func(context.Context, Context) error { handlerCalls++; return nil }) + runtime.batchCallbacks[batchID] = batchCallbacks{finally: func(context.Context, BatchState) error { callbackCalls++; return nil }} + recoveryContext, settlement := busruntime.WithDeliverySettlement(exhaustedWorkflowContext()) + recoveryContext = workflowRecoveryContext(recoveryContext, "generation-invalid-aggregate-current", owner) + recoveryErr := queueRuntime.DispatchJSON(recoveryContext, internalJobBatchJob, env) + if !busruntime.IsUncommitted(recoveryErr) || !strings.Contains(recoveryErr.Error(), test.diagnostic) { + t.Fatalf("invalid aggregate receipt recovery = %v, want uncommitted %q", recoveryErr, test.diagnostic) + } + if handlerCalls != 0 || callbackCalls != 0 || settlement.ApplicationStateCommitted() || len(recorder.events) != 0 { + t.Fatalf("handler/callback/committed/events = %d/%d/%t/%d, want 0/0/false/0", handlerCalls, callbackCalls, settlement.ApplicationStateCommitted(), len(recorder.events)) + } + }) + } +} + +// TestRecoverCommittedBatchTransitionRejectsInconsistentState covers every +// aggregate validation branch before recovery can publish member facts. +func TestRecoverCommittedBatchTransitionRejectsInconsistentState(t *testing.T) { + env := envelope{ + DispatchID: "dispatch-batch-recovery-validation", + BatchID: "batch-recovery-validation", + JobID: "job-batch-recovery-validation", + Job: StoredJob{Type: "workflow:batch:recovery-validation"}, + } + valid := BatchState{ + BatchID: env.BatchID, + DispatchID: env.DispatchID, + Total: 2, + Pending: 1, + Processed: 1, + } + tests := []struct { + name string + state BatchState + withoutProof bool + valid bool + }{ + {name: "batch identity mismatch", state: func() BatchState { state := valid; state.BatchID = "different-batch"; return state }()}, + {name: "dispatch identity mismatch", state: func() BatchState { state := valid; state.DispatchID = "different-dispatch"; return state }()}, + {name: "nonpositive total", state: func() BatchState { state := valid; state.Total = 0; return state }()}, + {name: "negative pending", state: func() BatchState { state := valid; state.Pending = -1; return state }()}, + {name: "negative processed", state: func() BatchState { state := valid; state.Processed = -1; return state }()}, + {name: "negative failed", state: func() BatchState { state := valid; state.Failed = -1; return state }()}, + {name: "counter sum mismatch", state: func() BatchState { state := valid; state.Total = 3; return state }()}, + {name: "failures exceed processed", state: func() BatchState { state := valid; state.Failed = 2; return state }()}, + {name: "exhausted without completion", state: BatchState{BatchID: env.BatchID, DispatchID: env.DispatchID, Total: 1, Processed: 1}}, + {name: "completed with pending member", state: func() BatchState { state := valid; state.Completed = true; return state }()}, + {name: "missing recovery proof", state: valid, withoutProof: true, valid: true}, + {name: "valid aggregate without receipt", state: valid, valid: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + faultStore := &workflowMutationFaultStore{Store: NewMemoryStore(), getBatchState: &test.state} + runtime, _, recorder := newWorkflowMutationRuntime(t, faultStore) + recoveryContext := context.Background() + if !test.withoutProof { + recoveryContext = workflowRecoveryContext(recoveryContext, "generation-batch-validation-current", "generation-batch-validation-recovered") + } + handled, err := runtime.recoverCommittedBatchTransition(recoveryContext, env) + wantErr := !test.withoutProof && !test.valid + if (err != nil) != wantErr || handled != wantErr { + t.Fatalf("recovery = handled:%t err:%v, want handled:%t err:%t", handled, err, wantErr, wantErr) + } + if err != nil && !busruntime.IsUncommitted(err) { + t.Fatalf("recovery validation error = %v, want uncommitted", err) + } + assertNoCommittedEvents(t, recorder.events, EventJobSucceeded, EventBatchProgressed, EventBatchCompleted) + }) + } +} + +// TestChainFailureReadFailureRedeliversWithoutFacts proves a committed failure +// is replayed until compatibility stores can expose its authoritative state. +func TestChainFailureReadFailureRedeliversWithoutFacts(t *testing.T) { + committedCause := errors.New("first application failure") + replayedCause := errors.New("different replayed failure") + committedErr := busruntime.Permanent(committedCause) + replayedErr := busruntime.Permanent(replayedCause) + const ( + chainID = "chain-failure-read-failure" + nodeID = "node-failure-read-failure" + jobType = "workflow:chain:failure-read-failure" + ) + baseStore := NewMemoryStore() + if err := baseStore.CreateChain(context.Background(), ChainRecord{ + ChainID: chainID, + Nodes: []ChainNode{{NodeID: nodeID, Job: StoredJob{Type: jobType}}}, + }); err != nil { + t.Fatalf("create chain: %v", err) + } + faultStore := &workflowMutationFaultStore{Store: baseStore, getChainErrOnCall: 2} + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, faultStore) + var handlerCalls, catchCalls, finallyCalls int + runtime.Register(jobType, func(context.Context, Context) error { + handlerCalls++ + if handlerCalls == 1 { + return committedErr + } + return replayedErr + }) + var observedCatchErr error + runtime.chainCallbacks[chainID] = chainCallbacks{ + catch: func(_ context.Context, _ ChainState, err error) error { + catchCalls++ + observedCatchErr = err + return nil + }, + finally: func(context.Context, ChainState) error { + finallyCalls++ + return nil + }, + } + delivery := envelope{ + SchemaVersion: schemaVersion, + DispatchID: "dispatch-failure-read-failure", + Kind: "chain_node", + ChainID: chainID, + NodeID: nodeID, + JobID: "job-failure-read-failure", + Job: StoredJob{Type: jobType}, + } + deliveryContext := busruntime.WithDeliveryAttempt(context.Background(), busruntime.DeliveryAttempt{Number: 0, MaxRetry: 2}) + err := queueRuntime.DispatchJSON(deliveryContext, internalJobChainNode, delivery) + if !busruntime.IsUncommitted(err) || !strings.Contains(err.Error(), "injected chain read failure") { + t.Fatalf("failure confirmation error = %v, want uncommitted injected read failure", err) + } + if handlerCalls != 1 || catchCalls != 0 || finallyCalls != 0 { + t.Fatalf("handler/catch/finally calls before recovery = %d/%d/%d, want 1/0/0", handlerCalls, catchCalls, finallyCalls) + } + state, err := baseStore.GetChain(context.Background(), chainID) + if err != nil { + t.Fatalf("get committed chain: %v", err) + } + if state.Completed || !state.Failed || state.Failure != committedErr.Error() { + t.Fatalf("committed chain state = %+v, want failed only", state) + } + assertNoCommittedEvents(t, recorder.events, EventJobFailed, EventChainFailed, EventCallbackStarted, EventCallbackSucceeded, EventCallbackFailed) + + if err := queueRuntime.DispatchJSON(deliveryContext, internalJobChainNode, delivery); !errors.Is(err, replayedCause) || !busruntime.IsPermanent(err) { + t.Fatalf("redeliver after store recovery: %v", err) + } + if handlerCalls != 2 || catchCalls != 1 || finallyCalls != 1 { + t.Fatalf("handler/catch/finally calls after recovery = %d/%d/%d, want 2/1/1", handlerCalls, catchCalls, finallyCalls) + } + if observedCatchErr == nil || observedCatchErr.Error() != committedErr.Error() { + t.Fatalf("catch error = %v, want committed cause %v", observedCatchErr, committedErr) + } + if errors.Is(observedCatchErr, replayedCause) { + t.Fatalf("catch error retained replayed cause: %v", observedCatchErr) + } + var failed, chainFailed, callbackSucceeded int + for _, event := range recorder.events { + switch event.Kind { + case EventJobFailed: + failed++ + if event.Err == nil || event.Err.Error() != committedErr.Error() || !busruntime.IsPermanent(event.Err) || errors.Is(event.Err, replayedCause) { + t.Fatalf("job failure cause = %v, want %v", event.Err, committedErr) + } + case EventChainFailed: + chainFailed++ + if event.Err == nil || event.Err.Error() != committedErr.Error() || !busruntime.IsPermanent(event.Err) || errors.Is(event.Err, replayedCause) { + t.Fatalf("chain failure cause = %v, want %v", event.Err, committedErr) + } + case EventCallbackSucceeded: + callbackSucceeded++ + } + } + if failed != 1 || chainFailed != 1 || callbackSucceeded != 2 { + t.Fatalf("job/chain/callback failure events = %d/%d/%d, want 1/1/2", failed, chainFailed, callbackSucceeded) + } +} + +// TestChainDoneRequiresTerminalState rejects custom stores that report a +// terminal transition while their readable state remains active. +func TestChainDoneRequiresTerminalState(t *testing.T) { + const ( + chainID = "chain-inconsistent-done" + nodeID = "node-inconsistent-done" + jobType = "workflow:chain:inconsistent-done" + ) + baseStore := NewMemoryStore() + if err := baseStore.CreateChain(context.Background(), ChainRecord{ + ChainID: chainID, + Nodes: []ChainNode{{NodeID: nodeID, Job: StoredJob{Type: jobType}}}, + }); err != nil { + t.Fatalf("create chain: %v", err) + } + faultStore := &workflowMutationFaultStore{Store: baseStore, advanceDoneWithoutState: true} + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, faultStore) + runtime.Register(jobType, func(context.Context, Context) error { return nil }) + err := queueRuntime.DispatchJSON(exhaustedWorkflowContext(), internalJobChainNode, envelope{ + SchemaVersion: schemaVersion, + DispatchID: "dispatch-inconsistent-done", + Kind: "chain_node", + ChainID: chainID, + NodeID: nodeID, + JobID: "job-inconsistent-done", + Job: StoredJob{Type: jobType}, + }) + if !busruntime.IsUncommitted(err) || !strings.Contains(err.Error(), "done without terminal state") { + t.Fatalf("inconsistent store error = %v, want uncommitted terminal-state validation", err) + } + assertNoCommittedEvents(t, recorder.events, EventJobSucceeded, EventChainAdvanced, EventChainCompleted) + state, stateErr := baseStore.GetChain(context.Background(), chainID) + if stateErr != nil { + t.Fatalf("get active chain: %v", stateErr) + } + if state.NextIndex != 0 || state.Completed || state.Failed { + t.Fatalf("inconsistent store changed chain state: %+v", state) + } +} + +// TestChainFailureRequiresTerminalState rejects compatibility stores that +// acknowledge failure while leaving the chain active and readable. +func TestChainFailureRequiresTerminalState(t *testing.T) { + const ( + chainID = "chain-inconsistent-failure" + nodeID = "node-inconsistent-failure" + jobType = "workflow:chain:inconsistent-failure" + ) + baseStore := NewMemoryStore() + if err := baseStore.CreateChain(context.Background(), ChainRecord{ + ChainID: chainID, + Nodes: []ChainNode{{NodeID: nodeID, Job: StoredJob{Type: jobType}}}, + }); err != nil { + t.Fatalf("create chain: %v", err) + } + faultStore := &workflowMutationFaultStore{Store: baseStore, failChainWithoutState: true} + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, faultStore) + runtime.Register(jobType, func(context.Context, Context) error { return errors.New("application failed") }) + err := queueRuntime.DispatchJSON(exhaustedWorkflowContext(), internalJobChainNode, envelope{ + SchemaVersion: schemaVersion, + DispatchID: "dispatch-inconsistent-failure", + Kind: "chain_node", + ChainID: chainID, + NodeID: nodeID, + JobID: "job-inconsistent-failure", + Job: StoredJob{Type: jobType}, + }) + if !busruntime.IsUncommitted(err) || !strings.Contains(err.Error(), "accepted failure without terminal state") { + t.Fatalf("inconsistent store error = %v, want uncommitted terminal-state validation", err) + } + assertNoCommittedEvents(t, recorder.events, EventJobFailed, EventChainFailed, EventCallbackStarted, EventCallbackSucceeded, EventCallbackFailed) + state, stateErr := baseStore.GetChain(context.Background(), chainID) + if stateErr != nil { + t.Fatalf("get active chain: %v", stateErr) + } + if state.NextIndex != 0 || state.Completed || state.Failed { + t.Fatalf("inconsistent store changed chain state: %+v", state) + } +} + +// TestChainFailureFallbackCommitsAndConfirmsState covers established custom +// stores that have not added the first-writer outcome capability. +func TestChainFailureFallbackCommitsAndConfirmsState(t *testing.T) { + const ( + chainID = "chain-compatibility-failure-fallback" + nodeID = "node-compatibility-failure-fallback" + ) + baseStore := NewMemoryStore() + if err := baseStore.CreateChain(context.Background(), ChainRecord{ + ChainID: chainID, + Nodes: []ChainNode{{NodeID: nodeID}}, + }); err != nil { + t.Fatalf("create chain: %v", err) + } + compatibilityStore := &workflowMutationFaultStore{Store: baseStore} + runtime, _, _ := newWorkflowMutationRuntime(t, compatibilityStore) + cause := errors.New("compatibility failure") + state, owned, err := runtime.failChainNode(context.Background(), chainID, nodeID, cause) + if err != nil || !owned || !state.Failed || state.Completed || state.Failure != cause.Error() { + t.Fatalf("fallback failure = state:%+v owned:%t err:%v", state, owned, err) + } + state, owned, err = runtime.failChainNode(context.Background(), chainID, nodeID, errors.New("replacement failure")) + if err != nil || !owned || state.Failure != cause.Error() { + t.Fatalf("fallback replay = state:%+v owned:%t err:%v", state, owned, err) + } +} + +// TestAtomicChainFailureRequiresTerminalState rejects a capable custom store +// that claims ownership without exposing the committed terminal transition. +func TestAtomicChainFailureRequiresTerminalState(t *testing.T) { + const ( + chainID = "chain-inconsistent-atomic-failure" + nodeID = "node-inconsistent-atomic-failure" + jobType = "workflow:chain:inconsistent-atomic-failure" + ) + baseStore := NewMemoryStore() + if err := baseStore.CreateChain(context.Background(), ChainRecord{ + ChainID: chainID, + Nodes: []ChainNode{{NodeID: nodeID, Job: StoredJob{Type: jobType}}}, + }); err != nil { + t.Fatalf("create chain: %v", err) + } + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, nonterminalWorkflowOutcomeStore{Store: baseStore}) + runtime.Register(jobType, func(context.Context, Context) error { return errors.New("application failed") }) + err := queueRuntime.DispatchJSON(exhaustedWorkflowContext(), internalJobChainNode, envelope{ + SchemaVersion: schemaVersion, + DispatchID: "dispatch-inconsistent-atomic-failure", + Kind: "chain_node", + ChainID: chainID, + NodeID: nodeID, + JobID: "job-inconsistent-atomic-failure", + Job: StoredJob{Type: jobType}, + }) + if !busruntime.IsUncommitted(err) || !strings.Contains(err.Error(), "accepted failure without terminal state") { + t.Fatalf("inconsistent atomic store error = %v, want uncommitted terminal-state validation", err) + } + assertNoCommittedEvents(t, recorder.events, EventJobFailed, EventChainFailed, EventCallbackStarted, EventCallbackSucceeded, EventCallbackFailed) +} + +// TestChainMutationFailuresRedeliverExhaustedAttempt verifies store outages cannot terminally settle a chain. +func TestChainMutationFailuresRedeliverExhaustedAttempt(t *testing.T) { + storeErr := errors.New("chain store unavailable") + tests := []struct { + name string + handlerErr error + configure func(*workflowMutationFaultStore) + }{ + { + name: "terminal failure does not commit", + handlerErr: errors.New("application failed"), + configure: func(store *workflowMutationFaultStore) { + store.failChainErr = storeErr + }, + }, + { + name: "successful node does not advance", + configure: func(store *workflowMutationFaultStore) { + store.advanceChainErr = storeErr + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + const ( + chainID = "chain_store_failure" + nodeID = "node_store_failure" + jobID = "job_store_failure" + jobType = "workflow:chain:store-failure" + ) + job := StoredJob{Type: jobType, Options: JobOptions{Retry: 2}} + baseStore := NewMemoryStore() + if err := baseStore.CreateChain(context.Background(), ChainRecord{ + ChainID: chainID, + Nodes: []ChainNode{{NodeID: nodeID, Job: job}}, + }); err != nil { + t.Fatalf("create chain: %v", err) + } + faultStore := &workflowMutationFaultStore{Store: baseStore} + test.configure(faultStore) + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, faultStore) + + handlerCalls := 0 + runtime.Register(jobType, func(context.Context, Context) error { + handlerCalls++ + return test.handlerErr + }) + catchCalls := 0 + finallyCalls := 0 + runtime.chainCallbacks[chainID] = chainCallbacks{ + catch: func(context.Context, ChainState, error) error { + catchCalls++ + return nil + }, + finally: func(context.Context, ChainState) error { + finallyCalls++ + return nil + }, + } + + err := queueRuntime.DispatchJSON(exhaustedWorkflowContext(), internalJobChainNode, envelope{ + SchemaVersion: schemaVersion, + DispatchID: "dispatch_store_failure", + Kind: "chain_node", + ChainID: chainID, + NodeID: nodeID, + JobID: jobID, + Job: job, + }) + assertUncommittedMutation(t, err, storeErr) + if handlerCalls != 1 { + t.Fatalf("handler calls = %d, want 1", handlerCalls) + } + if catchCalls != 0 || finallyCalls != 0 { + t.Fatalf("callbacks ran before state commit: catch=%d finally=%d", catchCalls, finallyCalls) + } + assertNoCommittedEvents(t, recorder.events, + EventJobSucceeded, + EventJobFailed, + EventChainAdvanced, + EventChainCompleted, + EventChainFailed, + EventCallbackSucceeded, + EventCallbackFailed, + ) + state, stateErr := baseStore.GetChain(context.Background(), chainID) + if stateErr != nil { + t.Fatalf("get chain: %v", stateErr) + } + if state.NextIndex != 0 || state.Completed || state.Failed { + t.Fatalf("chain state committed despite store failure: %+v", state) + } + }) + } +} + +// TestBatchMutationFailuresRedeliverExhaustedAttempt verifies every batch mutation gates execution and terminal facts. +func TestBatchMutationFailuresRedeliverExhaustedAttempt(t *testing.T) { + storeErr := errors.New("batch store unavailable") + tests := []struct { + name string + handlerErr error + wantHandlerCalls int + configure func(*workflowMutationFaultStore) + }{ + { + name: "started state does not commit", + wantHandlerCalls: 0, + configure: func(store *workflowMutationFaultStore) { + store.markBatchStartedErr = storeErr + }, + }, + { + name: "successful outcome does not commit", + wantHandlerCalls: 1, + configure: func(store *workflowMutationFaultStore) { + store.markBatchSucceededErr = storeErr + }, + }, + { + name: "failed outcome does not commit", + handlerErr: errors.New("application failed"), + wantHandlerCalls: 1, + configure: func(store *workflowMutationFaultStore) { + store.markBatchFailedErr = storeErr + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + const ( + batchID = "batch_store_failure" + jobID = "job_store_failure" + jobType = "workflow:batch:store-failure" + ) + job := StoredJob{Type: jobType, Options: JobOptions{Retry: 2}} + baseStore := NewMemoryStore() + if err := baseStore.CreateBatch(context.Background(), BatchRecord{ + BatchID: batchID, + Jobs: []BatchJob{{JobID: jobID, Job: job}}, + }); err != nil { + t.Fatalf("create batch: %v", err) + } + faultStore := &workflowMutationFaultStore{Store: baseStore} + test.configure(faultStore) + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, faultStore) + + handlerCalls := 0 + runtime.Register(jobType, func(context.Context, Context) error { + handlerCalls++ + return test.handlerErr + }) + progressCalls := 0 + thenCalls := 0 + catchCalls := 0 + finallyCalls := 0 + runtime.batchCallbacks[batchID] = batchCallbacks{ + progress: func(context.Context, BatchState) error { + progressCalls++ + return nil + }, + then: func(context.Context, BatchState) error { + thenCalls++ + return nil + }, + catch: func(context.Context, BatchState, error) error { + catchCalls++ + return nil + }, + finally: func(context.Context, BatchState) error { + finallyCalls++ + return nil + }, + } + + err := queueRuntime.DispatchJSON(exhaustedWorkflowContext(), internalJobBatchJob, envelope{ + SchemaVersion: schemaVersion, + DispatchID: "dispatch_store_failure", + Kind: "batch_job", + BatchID: batchID, + JobID: jobID, + Job: job, + }) + assertUncommittedMutation(t, err, storeErr) + if handlerCalls != test.wantHandlerCalls { + t.Fatalf("handler calls = %d, want %d", handlerCalls, test.wantHandlerCalls) + } + if progressCalls != 0 || thenCalls != 0 || catchCalls != 0 || finallyCalls != 0 { + t.Fatalf("callbacks ran before state commit: progress=%d then=%d catch=%d finally=%d", progressCalls, thenCalls, catchCalls, finallyCalls) + } + assertNoCommittedEvents(t, recorder.events, + EventJobSucceeded, + EventJobFailed, + EventBatchProgressed, + EventBatchCompleted, + EventBatchFailed, + EventBatchCancelled, + EventCallbackSucceeded, + EventCallbackFailed, + ) + state, stateErr := baseStore.GetBatch(context.Background(), batchID) + if stateErr != nil { + t.Fatalf("get batch: %v", stateErr) + } + if state.Pending != 1 || state.Processed != 0 || state.Failed != 0 || state.Completed || state.Cancelled { + t.Fatalf("batch terminal state committed despite store failure: %+v", state) + } + }) + } +} + +// TestCallbackStoreFailuresRedeliverWithoutTerminalFacts verifies state reads and idempotency writes remain retryable at exhaustion. +func TestCallbackStoreFailuresRedeliverWithoutTerminalFacts(t *testing.T) { + storeErr := errors.New("callback store unavailable") + tests := []struct { + name string + callbackEnv envelope + seed func(context.Context, Store) error + configure func(*workflowMutationFaultStore) + clear func(*workflowMutationFaultStore) + install func(*runtime, *int) + }{ + { + name: "chain state read", + callbackEnv: envelope{ + SchemaVersion: schemaVersion, + DispatchID: "dispatch_callback_store_failure", + JobID: "job_callback_store_failure", + ChainID: "chain_callback_store_failure", + CallbackKind: "chain_finally", + }, + seed: func(ctx context.Context, store Store) error { + if err := store.CreateChain(ctx, ChainRecord{ + ChainID: "chain_callback_store_failure", + Nodes: []ChainNode{{NodeID: "chain_callback_node", Job: StoredJob{Type: "callback:source"}}}, + }); err != nil { + return err + } + _, _, err := store.AdvanceChain(ctx, "chain_callback_store_failure", "chain_callback_node") + return err + }, + configure: func(store *workflowMutationFaultStore) { + store.getChainErr = storeErr + }, + clear: func(store *workflowMutationFaultStore) { + store.getChainErr = nil + }, + install: func(runtime *runtime, calls *int) { + runtime.chainCallbacks["chain_callback_store_failure"] = chainCallbacks{ + finally: func(context.Context, ChainState) error { + *calls = *calls + 1 + return nil + }, + } + }, + }, + { + name: "batch state read", + callbackEnv: envelope{ + SchemaVersion: schemaVersion, + DispatchID: "dispatch_callback_store_failure", + JobID: "job_callback_store_failure", + BatchID: "batch_callback_store_failure", + CallbackKind: "batch_then", + }, + seed: func(ctx context.Context, store Store) error { + if err := store.CreateBatch(ctx, BatchRecord{ + BatchID: "batch_callback_store_failure", + Jobs: []BatchJob{{JobID: "batch_callback_job", Job: StoredJob{Type: "callback:source"}}}, + }); err != nil { + return err + } + _, _, err := store.MarkBatchJobSucceeded(ctx, "batch_callback_store_failure", "batch_callback_job") + return err + }, + configure: func(store *workflowMutationFaultStore) { + store.getBatchErr = storeErr + }, + clear: func(store *workflowMutationFaultStore) { + store.getBatchErr = nil + }, + install: func(runtime *runtime, calls *int) { + runtime.batchCallbacks["batch_callback_store_failure"] = batchCallbacks{ + then: func(context.Context, BatchState) error { + *calls = *calls + 1 + return nil + }, + } + }, + }, + { + name: "callback idempotency write", + callbackEnv: envelope{ + SchemaVersion: schemaVersion, + DispatchID: "dispatch_callback_store_failure", + JobID: "job_callback_store_failure", + BatchID: "batch_callback_store_failure", + CallbackKind: "batch_then", + }, + seed: func(ctx context.Context, store Store) error { + if err := store.CreateBatch(ctx, BatchRecord{ + BatchID: "batch_callback_store_failure", + Jobs: []BatchJob{{JobID: "batch_callback_job", Job: StoredJob{Type: "callback:source"}}}, + }); err != nil { + return err + } + _, _, err := store.MarkBatchJobSucceeded(ctx, "batch_callback_store_failure", "batch_callback_job") + return err + }, + configure: func(store *workflowMutationFaultStore) { + store.markCallbackErr = storeErr + }, + clear: func(store *workflowMutationFaultStore) { + store.markCallbackErr = nil + }, + install: func(runtime *runtime, calls *int) { + runtime.batchCallbacks["batch_callback_store_failure"] = batchCallbacks{ + then: func(context.Context, BatchState) error { + *calls = *calls + 1 + return nil + }, + } + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + baseStore := NewMemoryStore() + if err := test.seed(context.Background(), baseStore); err != nil { + t.Fatalf("seed workflow state: %v", err) + } + faultStore := &workflowMutationFaultStore{Store: baseStore} + test.configure(faultStore) + runtime, queueRuntime, recorder := newWorkflowMutationRuntime(t, faultStore) + callbackCalls := 0 + test.install(runtime, &callbackCalls) + + err := queueRuntime.DispatchJSON(exhaustedWorkflowContext(), internalJobCallback, test.callbackEnv) + assertUncommittedMutation(t, err, storeErr) + if callbackCalls != 0 { + t.Fatalf("callback calls before store recovery = %d, want 0", callbackCalls) + } + assertNoCommittedEvents(t, recorder.events, EventCallbackSucceeded, EventCallbackFailed) + + test.clear(faultStore) + if err := queueRuntime.DispatchJSON(exhaustedWorkflowContext(), internalJobCallback, test.callbackEnv); err != nil { + t.Fatalf("callback retry after store recovery: %v", err) + } + if err := queueRuntime.DispatchJSON(exhaustedWorkflowContext(), internalJobCallback, test.callbackEnv); err != nil { + t.Fatalf("duplicate callback delivery: %v", err) + } + if callbackCalls != 1 { + t.Fatalf("callback calls after retry and duplicate = %d, want 1", callbackCalls) + } + succeeded := 0 + for _, event := range recorder.events { + if event.Kind == EventCallbackSucceeded { + succeeded++ + } + } + if succeeded != 1 { + t.Fatalf("callback success events after retry and duplicate = %d, want 1", succeeded) + } + }) + } +} diff --git a/job.go b/job.go index 0d8b5fe..5351346 100644 --- a/job.go +++ b/job.go @@ -7,6 +7,8 @@ import ( "fmt" "reflect" "time" + + "github.com/goforj/queue/busruntime" ) // Job is a pure queue payload value plus enqueue metadata. @@ -27,15 +29,32 @@ type Job struct { } type jobOptions struct { - queueName string - timeout *time.Duration - maxRetry *int - attempt int - backoff *time.Duration - delay time.Duration - uniqueTTL time.Duration + queueName string + timeout *time.Duration + maxRetry *int + attempt int + backoff *time.Duration + delay time.Duration + uniqueTTL time.Duration + logicalType string + logicalPayload []byte + logicalSet bool + metadata DriverJobMetadata } +// DriverJobMetadataVersion identifies the direct-delivery metadata understood by +// this version of the driver integration contract. +// @group Driver Integration +const DriverJobMetadataVersion = busruntime.DeliveryMetadataVersion + +// DriverJobMetadata carries stable correlation separately from an application's +// job type and payload. Drivers persist this metadata alongside their native +// delivery representation so ordinary jobs do not require a workflow envelope. +// +// This is an advanced type intended for optional driver integrations. +// @group Driver Integration +type DriverJobMetadata = busruntime.DeliveryMetadata + // DriverJobOptions exposes parsed job enqueue metadata for driver-module implementations. // // This is an advanced type intended for optional driver integrations. @@ -317,6 +336,34 @@ func DriverOptions(job Job) DriverJobOptions { } } +// DriverMetadata returns supported direct-delivery correlation attached to job. +// Unknown metadata versions are intentionally hidden so older workers do not +// trust fields whose semantics they cannot validate. +// +// This is an advanced helper intended for driver-module implementations. +// @group Driver Integration +func DriverMetadata(job Job) DriverJobMetadata { + metadata := job.options.metadata + if metadata.SchemaVersion != DriverJobMetadataVersion { + return DriverJobMetadata{} + } + return metadata +} + +// DriverWithMetadata returns a job carrying supported direct-delivery +// correlation. Unknown versions are ignored while the application payload and +// delivery policy remain usable. +// +// This is an advanced helper intended for driver-module implementations. +// @group Driver Integration +func DriverWithMetadata(job Job, metadata DriverJobMetadata) Job { + if metadata.SchemaVersion != DriverJobMetadataVersion { + return job + } + job.options.metadata = metadata + return job +} + func (t Job) withBuildErr(err error) Job { if t.buildErr == nil { t.buildErr = err @@ -329,6 +376,14 @@ func (t Job) withAttempt(attempt int) Job { return t } +// withLogicalIdentity carries application bytes separately from their physical delivery envelope. +func (t Job) withLogicalIdentity(jobType string, payload []byte) Job { + t.options.logicalType = jobType + t.options.logicalPayload = append([]byte(nil), payload...) + t.options.logicalSet = true + return t +} + // DriverWithAttempt returns a copy of the job with the attempt number set. // // This is an advanced helper intended for driver-module implementations. diff --git a/job_identity.go b/job_identity.go new file mode 100644 index 0000000..776a9ab --- /dev/null +++ b/job_identity.go @@ -0,0 +1,47 @@ +package queue + +import ( + "crypto/sha256" + "encoding/binary" + "encoding/hex" + + "github.com/goforj/queue/internal/jobidentity" +) + +const driverUniqueKeyVersion = "goforj:queue:unique:v1" + +// DriverUniqueKey returns a versioned queue-scoped identity for driver deduplication. +// Correlation IDs and delivery policy are excluded when a workflow envelope carries a logical job. +// @group Driver Integration +func DriverUniqueKey(job Job, queueName string) string { + jobType := job.Type + payload := job.PayloadBytes() + if job.options.logicalSet { + jobType = job.options.logicalType + payload = job.options.logicalPayload + } else { + logical := resolveLogicalJob(job.Type, payload) + jobType = logical.jobType + payload = logical.payload + } + payload = canonicalIdentityPayload(payload) + identity := []byte(driverUniqueKeyVersion) + identity = appendUniqueIdentityPart(identity, []byte(queueName)) + identity = appendUniqueIdentityPart(identity, []byte(jobType)) + identity = appendUniqueIdentityPart(identity, payload) + digest := sha256.Sum256(identity) + return "v1:" + hex.EncodeToString(digest[:]) +} + +// canonicalIdentityPayload keeps payload absence stable while the legacy workflow facade serializes nil as JSON null. +func canonicalIdentityPayload(payload []byte) []byte { + return jobidentity.CanonicalPayload(payload) +} + +// appendUniqueIdentityPart length-frames arbitrary bytes so delimiters inside names or payloads cannot collide. +func appendUniqueIdentityPart(dst, value []byte) []byte { + var size [8]byte + binary.BigEndian.PutUint64(size[:], uint64(len(value))) + dst = append(dst, size[:]...) + return append(dst, value...) +} diff --git a/job_identity_test.go b/job_identity_test.go new file mode 100644 index 0000000..889d3b7 --- /dev/null +++ b/job_identity_test.go @@ -0,0 +1,69 @@ +package queue + +import "testing" + +// TestDriverUniqueKeyGoldenVector pins persisted identity bytes across rolling producer versions. +func TestDriverUniqueKeyGoldenVector(t *testing.T) { + const want = "v1:b260f44c0b067a6a4b13214544d81291c3d58e43350c2e196bc4d7af39c11f5b" + direct := NewJob("reports:build").Payload([]byte(`{"id":1}`)) + if got := DriverUniqueKey(direct, "critical"); got != want { + t.Fatalf("direct unique key = %q, want golden %q", got, want) + } + envelope := NewJob("bus:job").Payload([]byte(`{"schema_version":1,"dispatch_id":"volatile","job_id":"volatile","job":{"type":"reports:build","payload":"eyJpZCI6MX0="}}`)) + if got := DriverUniqueKey(envelope, "critical"); got != want { + t.Fatalf("workflow unique key = %q, want golden %q", got, want) + } +} + +// TestDriverUniqueKeyEmptyPayloadGoldenVector pins the compatibility normalization required when direct jobs replace workflow envelopes. +func TestDriverUniqueKeyEmptyPayloadGoldenVector(t *testing.T) { + const want = "v1:c61e4fa70176e7ae023e4aae041317dbc8c8503b1ab07f9062cbfbcae1c328c7" + jobs := []Job{ + NewJob("reports:empty"), + NewJob("reports:empty").Payload([]byte{}), + NewJob("reports:empty").PayloadJSON(nil), + NewJob("reports:empty").Payload([]byte("null")), + NewJob("bus:job").Payload([]byte(`{"schema_version":1,"job":{"type":"reports:empty","payload":"bnVsbA=="}}`)), + NewJob("bus:job").Payload([]byte(`{"schema_version":1,"job":{"type":"reports:empty","payload":""}}`)), + } + for i, job := range jobs { + if got := DriverUniqueKey(job, "critical"); got != want { + t.Fatalf("empty payload variant %d unique key = %q, want golden %q", i, got, want) + } + } +} + +// TestDriverUniqueKeyExcludesWorkflowCorrelationAndPolicy verifies only logical job bytes and queue define identity. +func TestDriverUniqueKeyExcludesWorkflowCorrelationAndPolicy(t *testing.T) { + first := NewJob("bus:job").Payload([]byte(`{"schema_version":1,"dispatch_id":"dsp_1","job_id":"job_1","attempt":0,"job":{"type":"reports:build","payload":"eyJpZCI6MX0=","options":{"retry":1}}}`)) + second := NewJob("bus:job").Payload([]byte(`{"schema_version":1,"dispatch_id":"dsp_2","job_id":"job_2","attempt":7,"job":{"type":"reports:build","payload":"eyJpZCI6MX0=","options":{"retry":9}}}`)) + firstKey := DriverUniqueKey(first, "critical") + secondKey := DriverUniqueKey(second, "critical") + if firstKey == "" || firstKey != secondKey { + t.Fatalf("volatile workflow fields changed identity: %q != %q", firstKey, secondKey) + } + if firstKey == DriverUniqueKey(second, "default") { + t.Fatal("queue scope did not change identity") + } +} + +// TestDriverUniqueKeyFramesArbitraryBytes verifies delimiter-like values cannot alias each other. +func TestDriverUniqueKeyFramesArbitraryBytes(t *testing.T) { + first := DriverUniqueKey(NewJob("a").Payload([]byte("b:c")), "default") + second := DriverUniqueKey(NewJob("a:b").Payload([]byte("c")), "default") + if first == second { + t.Fatalf("length framing collided: %q", first) + } + if first != DriverUniqueKey(NewJob("a").Payload([]byte("b:c")), "default") { + t.Fatal("equal logical jobs produced unstable identities") + } +} + +// TestDriverUniqueKeyPrivateIdentityMatchesEnvelope verifies root dispatch can carry identity without exporting mutable driver options. +func TestDriverUniqueKeyPrivateIdentityMatchesEnvelope(t *testing.T) { + envelope := NewJob("bus:job").Payload([]byte(`{"schema_version":1,"dispatch_id":"dsp_1","job":{"type":"reports:build","payload":"eyJpZCI6MX0="}}`)) + physical := NewJob("bus:job").Payload([]byte("opaque")).withLogicalIdentity("reports:build", []byte(`{"id":1}`)) + if DriverUniqueKey(envelope, "critical") != DriverUniqueKey(physical, "critical") { + t.Fatal("private identity and decoded envelope disagreed") + } +} diff --git a/logical_job.go b/logical_job.go new file mode 100644 index 0000000..b870d24 --- /dev/null +++ b/logical_job.go @@ -0,0 +1,26 @@ +package queue + +import "github.com/goforj/queue/internal/workflow" + +// logicalJob is the root-facing view of identity and correlation resolved from one physical delivery. +type logicalJob struct { + jobType string + payload []byte + dispatchID string + jobID string + chainID string + batchID string +} + +// resolveLogicalJob decodes only the owned workflow schema so identity and telemetry cannot drift onto separate interpretations. +func resolveLogicalJob(rawType string, payload []byte) logicalJob { + metadata := workflow.ResolveDelivery(rawType, payload) + return logicalJob{ + jobType: metadata.JobType, + payload: metadata.Payload, + dispatchID: metadata.DispatchID, + jobID: metadata.JobID, + chainID: metadata.ChainID, + batchID: metadata.BatchID, + } +} diff --git a/observability.go b/observability.go index a3de5f3..c3282bc 100644 --- a/observability.go +++ b/observability.go @@ -2,12 +2,31 @@ package queue import ( "context" - "crypto/sha1" + "crypto/rand" + "encoding/hex" "errors" "fmt" "sort" "sync" "time" + + "github.com/goforj/queue/busruntime" + "github.com/goforj/queue/internal/observation" +) + +const eventSchemaVersion = observation.EventSchemaVersion + +// EventLayer identifies the subsystem that committed an observed fact. +// @group Observability +type EventLayer string + +const ( + // EventLayerQueue identifies dispatch, enqueue, and queue-control facts. + EventLayerQueue EventLayer = "queue" + // EventLayerWorker identifies physical delivery and handler-attempt facts. + EventLayerWorker EventLayer = "worker" + // EventLayerWorkflow identifies chain, batch, and continuation facts. + EventLayerWorkflow EventLayer = "workflow" ) // EventKind identifies a queue runtime event. @@ -15,11 +34,11 @@ import ( type EventKind string const ( - // EventDispatchStarted indicates workflow dispatch began. + // EventDispatchStarted indicates public dispatch began. EventDispatchStarted EventKind = "dispatch_started" - // EventDispatchSucceeded indicates workflow dispatch completed successfully. + // EventDispatchSucceeded indicates the backend accepted the public dispatch; synchronous execution may still return an application error. EventDispatchSucceeded EventKind = "dispatch_succeeded" - // EventDispatchFailed indicates workflow dispatch failed before handler execution. + // EventDispatchFailed indicates public dispatch failed before backend acceptance. EventDispatchFailed EventKind = "dispatch_failed" // EventEnqueueAccepted indicates a job was accepted for enqueue. EventEnqueueAccepted EventKind = "enqueue_accepted" @@ -33,11 +52,11 @@ const ( EventProcessStarted EventKind = "process_started" // EventProcessSucceeded indicates a handler completed successfully. EventProcessSucceeded EventKind = "process_succeeded" - // EventProcessFailed indicates a handler returned an error. + // EventProcessFailed indicates a handler returned an error or panicked; panics are reported before being rethrown. EventProcessFailed EventKind = "process_failed" - // EventProcessRetried indicates a failed attempt was requeued for retry. + // EventProcessRetried indicates processing began for a numbered application retry attempt; redelivery may repeat the fact. EventProcessRetried EventKind = "process_retried" - // EventProcessArchived indicates a failed attempt reached terminal state. + // EventProcessArchived indicates a driver confirmed terminal settlement for a failed attempt. EventProcessArchived EventKind = "process_archived" // EventQueuePaused indicates queue consumption was paused. EventQueuePaused EventKind = "queue_paused" @@ -47,25 +66,65 @@ const ( EventProcessRecovered EventKind = "process_recovered" // EventRepublishFailed indicates an internal delay/retry republish attempt failed. EventRepublishFailed EventKind = "republish_failed" + // EventSettlementFailed indicates a broker acknowledgement or deletion failed after handler or replacement work completed. + EventSettlementFailed EventKind = "settlement_failed" + // EventJobStarted indicates logical job execution began. + EventJobStarted EventKind = "job_started" + // EventJobSucceeded indicates logical job execution succeeded. + EventJobSucceeded EventKind = "job_succeeded" + // EventJobFailed indicates logical job execution reached a failed outcome. + EventJobFailed EventKind = "job_failed" + // EventChainStarted indicates a chain was created and started. + EventChainStarted EventKind = "chain_started" + // EventChainAdvanced indicates a chain advanced to its next job. + EventChainAdvanced EventKind = "chain_advanced" + // EventChainCompleted indicates a chain completed successfully. + EventChainCompleted EventKind = "chain_completed" + // EventChainFailed indicates a chain reached a failed outcome. + EventChainFailed EventKind = "chain_failed" + // EventBatchStarted indicates a batch was created and started. + EventBatchStarted EventKind = "batch_started" + // EventBatchProgressed indicates a batch job reached a terminal outcome. + EventBatchProgressed EventKind = "batch_progressed" + // EventBatchCompleted indicates a batch completed successfully. + EventBatchCompleted EventKind = "batch_completed" + // EventBatchFailed indicates a batch reached a failed outcome. + EventBatchFailed EventKind = "batch_failed" + // EventBatchCancelled indicates remaining batch work was cancelled. + EventBatchCancelled EventKind = "batch_cancelled" + // EventCallbackStarted indicates an ephemeral workflow callback began. + EventCallbackStarted EventKind = "callback_started" + // EventCallbackSucceeded indicates an ephemeral workflow callback succeeded. + EventCallbackSucceeded EventKind = "callback_succeeded" + // EventCallbackFailed indicates an ephemeral workflow callback failed. + EventCallbackFailed EventKind = "callback_failed" ) -// Event is emitted through Observer hooks for queue/worker activity. +// Event is emitted through Observer hooks for queue, worker, and workflow activity. // @group Driver Integration type Event struct { - Kind EventKind - Driver Driver - Queue string - JobType string - JobKey string - Attempt int - MaxRetry int - Scheduled bool - Duration time.Duration - Err error - Time time.Time -} - -// Observer receives queue runtime events. + SchemaVersion int + EventID string + Layer EventLayer + Kind EventKind + Driver Driver + Queue string + JobType string + JobKey string + DispatchID string + JobID string + ChainID string + BatchID string + Attempt int + MaxRetry int + Scheduled bool + Duration time.Duration + Err error + Time time.Time +} + +// Observer receives queue, worker, and workflow events. +// Implementations must be safe for concurrent calls from dispatchers and workers. // @group Observability type Observer interface { // Observe handles a queue runtime event. @@ -114,6 +173,45 @@ func (f ObserverFunc) Observe(ctx context.Context, event Event) { f(ctx, event) } +// observerSink is the construction-time extension point shared with optional driver modules. +type observerSink interface { + Observer + Add(func(context.Context, Event)) + HasObservers() bool +} + +// ensureObserverSink provides one mutable fan-out point that drivers can retain before root options are applied. +func ensureObserverSink(observer Observer) Observer { + if sink, ok := observer.(observerSink); ok { + return sink + } + if observer == nil { + return observation.NewSink[Event]() + } + return observation.NewSink[Event](observer.Observe) +} + +// addObserverToSink extends the shared sink without nesting queue wrappers or creating a second event identity. +func addObserverToSink(current Observer, observer Observer) Observer { + current = ensureObserverSink(current) + if observer == nil { + return current + } + current.(observerSink).Add(observer.Observe) + return current +} + +// observerHasRecipients preserves the no-observer fast path even when drivers retain an initially empty sink. +func observerHasRecipients(observer Observer) bool { + if observer == nil { + return false + } + if sink, ok := observer.(interface{ HasObservers() bool }); ok { + return sink.HasObservers() + } + return true +} + type multiObserver struct { observers []Observer } @@ -142,6 +240,7 @@ func MultiObserver(observers ...Observer) Observer { } func (m *multiObserver) Observe(ctx context.Context, event Event) { + event = normalizeObservedEvent(event) for _, observer := range m.observers { safeObserve(ctx, observer, event) } @@ -168,6 +267,7 @@ func (c ChannelObserver) Observe(_ context.Context, event Event) { if c.Events == nil { return } + event = normalizeObservedEvent(event) if c.DropIfFull { select { case c.Events <- event: @@ -478,14 +578,110 @@ type StatsCollector struct { } type collectorQueueState struct { - counters QueueCounters - processedAt []time.Time - failedAt []time.Time - pendingByKey map[string][]time.Time - waitSum time.Duration - waitCount int64 - runSum time.Duration - runCount int64 + counters QueueCounters + processedAt []time.Time + failedAt []time.Time + pendingByKey map[string][]time.Time + activeSettlements map[busruntime.DeliverySettlementIdentity]struct{} + activeByKey map[string]int64 + uncorrelatedActive int64 + waitSum time.Duration + waitCount int64 + runSum time.Duration + runCount int64 +} + +// openActive records one physical handler invocation so later process and +// settlement facts can close that exact execution at most once. +func (s *collectorQueueState) openActive(ctx context.Context, event Event) { + if settlement, ok := busruntime.DeliverySettlementIdentityFromContext(ctx); ok { + if s.activeSettlements == nil { + s.activeSettlements = make(map[busruntime.DeliverySettlementIdentity]struct{}) + } + if _, exists := s.activeSettlements[settlement]; exists { + return + } + s.activeSettlements[settlement] = struct{}{} + s.counters.Active++ + return + } + key := collectorExecutionKey(event) + if key != "" { + if s.activeByKey == nil { + s.activeByKey = make(map[string]int64) + } + s.activeByKey[key]++ + } else { + s.uncorrelatedActive++ + } + s.counters.Active++ +} + +// removeSettlement deletes one exact physical boundary without changing the +// queue-wide gauge. +func (s *collectorQueueState) removeSettlement(settlement busruntime.DeliverySettlementIdentity) bool { + _, exists := s.activeSettlements[settlement] + if !exists { + return false + } + delete(s.activeSettlements, settlement) + return true +} + +// closeByKey removes one execution only from the identity-less correlation +// domain; losing an exact context cannot consume a different physical owner. +func (s *collectorQueueState) closeByKey(key string) bool { + removed := false + if key == "" && s.uncorrelatedActive > 0 { + s.uncorrelatedActive-- + removed = true + } else if key != "" && s.activeByKey[key] > 0 { + s.activeByKey[key]-- + if s.activeByKey[key] == 0 { + delete(s.activeByKey, key) + } + removed = true + } + if !removed { + return false + } + if s.counters.Active > 0 { + s.counters.Active-- + } + return true +} + +// closeActive removes one matching physical invocation and reports whether +// the queue-wide gauge changed, using the strongest available correlation. +func (s *collectorQueueState) closeActive(ctx context.Context, event Event) bool { + if settlement, ok := busruntime.DeliverySettlementIdentityFromContext(ctx); ok { + if !s.removeSettlement(settlement) { + return false + } + if s.counters.Active > 0 { + s.counters.Active-- + } + return true + } + return s.closeByKey(collectorExecutionKey(event)) +} + +// closeSettlementActive requires exact physical identity because event fields +// cannot distinguish a late settlement from a newer execution of the same job. +func (s *collectorQueueState) closeSettlementActive(ctx context.Context, event Event) bool { + if _, ok := busruntime.DeliverySettlementIdentityFromContext(ctx); !ok { + return false + } + return s.closeActive(ctx, event) +} + +// collectorExecutionKey derives the strongest event-only execution identity +// available when a driver does not attach a settlement boundary. +func collectorExecutionKey(event Event) string { + if event.JobID == "" { + return "" + } + return fmt.Sprintf("%s\x00%s\x00%d", event.DispatchID, event.JobID, event.Attempt) } // NewStatsCollector creates an event collector for queue counters. @@ -511,7 +707,7 @@ func NewStatsCollector() *StatsCollector { // Queue: "default", // Time: time.Now(), // }) -func (c *StatsCollector) Observe(_ context.Context, event Event) { +func (c *StatsCollector) Observe(ctx context.Context, event Event) { queue := event.Queue if queue == "" { queue = "default" @@ -550,7 +746,7 @@ func (c *StatsCollector) Observe(_ context.Context, event Event) { if state.counters.Scheduled > 0 && event.Scheduled { state.counters.Scheduled-- } - state.counters.Active++ + state.openActive(ctx, event) if event.JobKey != "" { if entries, ok := state.pendingByKey[event.JobKey]; ok && len(entries) > 0 { enqueuedAt := entries[0] @@ -578,23 +774,22 @@ func (c *StatsCollector) Observe(_ context.Context, event Event) { state.counters.Paused-- } case EventProcessSucceeded: - if state.counters.Active > 0 { - state.counters.Active-- - } + state.closeActive(ctx, event) state.counters.Processed++ state.processedAt = append(state.processedAt, now) state.runSum += event.Duration state.runCount++ state.counters.AvgRun = state.runSum / time.Duration(state.runCount) case EventProcessFailed: - if state.counters.Active > 0 { - state.counters.Active-- - } + state.closeActive(ctx, event) state.counters.Failed++ state.failedAt = append(state.failedAt, now) state.runSum += event.Duration state.runCount++ state.counters.AvgRun = state.runSum / time.Duration(state.runCount) + case EventSettlementFailed: + // Built-in drivers attach exact identity. Identity-less settlement facts cannot safely choose between late and current executions that share event fields. + state.closeSettlementActive(ctx, event) } c.pruneThroughputLocked(state, now) @@ -700,7 +895,7 @@ type observedQueue struct { } func newObservedQueue(inner queueBackend, driver Driver, observer Observer) queueBackend { - if observer == nil { + if !observerHasRecipients(observer) { return inner } return &observedQueue{ @@ -753,6 +948,7 @@ func (q *observedQueue) Pause(ctx context.Context, queueName string) error { return err } safeObserve(ctx, q.observer, Event{ + Layer: EventLayerQueue, Kind: EventQueuePaused, Driver: q.driver, Queue: normalizeQueueName(queueName), @@ -770,6 +966,7 @@ func (q *observedQueue) Resume(ctx context.Context, queueName string) error { return err } safeObserve(ctx, q.observer, Event{ + Layer: EventLayerQueue, Kind: EventQueueResumed, Driver: q.driver, Queue: normalizeQueueName(queueName), @@ -827,21 +1024,38 @@ func (q *observedQueue) History(ctx context.Context, queueName string, window Qu } func (q *observedQueue) Dispatch(ctx context.Context, job Job) error { - err := q.inner.Dispatch(ctx, job) + ctx, acceptance := ensureDispatchAcceptance(ctx) opts := job.jobOptions() + metadata := ResolveObservedJobMetadataFromJob(job) base := Event{ - Driver: q.driver, - Queue: jobQueueName(job), - JobType: job.Type, - JobKey: jobEventKey(job), - MaxRetry: optionInt(opts.maxRetry), - Scheduled: opts.delay > 0, - Time: time.Now(), + Layer: EventLayerQueue, + Driver: q.driver, + Queue: jobQueueName(job), + JobType: metadata.JobType, + JobKey: metadata.JobKey, + DispatchID: metadata.DispatchID, + JobID: metadata.JobID, + ChainID: metadata.ChainID, + BatchID: metadata.BatchID, + MaxRetry: optionInt(opts.maxRetry), + Scheduled: opts.delay > 0, + } + acceptance.onAccepted(func() { + accepted := base + accepted.Kind = EventEnqueueAccepted + accepted.Time = time.Now() + safeObserve(ctx, q.observer, accepted) + }) + + err := q.inner.Dispatch(ctx, job) + if err == nil { + acceptance.markAccepted() } + if acceptance.isAccepted() { + return err + } + base.Time = time.Now() switch { - case err == nil: - base.Kind = EventEnqueueAccepted - safeObserve(ctx, q.observer, base) case errors.Is(err, ErrDuplicate): base.Kind = EventEnqueueDuplicate base.Err = err @@ -876,31 +1090,54 @@ func (q *observedQueue) Driver() Driver { return q.driver } +// wrapObservedHandler emits physical attempt facts while deferring success to any driver-owned settlement boundary. func wrapObservedHandler(observer Observer, driver Driver, queueName string, jobType string, ctxDecorator func(context.Context) context.Context, handler Handler) Handler { return func(ctx context.Context, job Job) error { if ctxDecorator != nil { if decorated := ctxDecorator(ctx); decorated != nil { - ctx = decorated + ctx = busruntime.PreserveDeliveryContext(ctx, decorated) } } opts := job.jobOptions() + metadata := ResolveObservedJobMetadataFromJob(job) effectiveQueue := queueName if effectiveQueue == "" { effectiveQueue = jobQueueName(job) } start := time.Now() base := Event{ - Driver: driver, - Queue: effectiveQueue, - JobType: jobType, - JobKey: jobEventKey(job), - Attempt: opts.attempt, - MaxRetry: optionInt(opts.maxRetry), - Scheduled: opts.delay > 0, - Time: start, + Layer: EventLayerWorker, + Driver: driver, + Queue: effectiveQueue, + JobType: metadata.JobType, + JobKey: metadata.JobKey, + DispatchID: metadata.DispatchID, + JobID: metadata.JobID, + ChainID: metadata.ChainID, + BatchID: metadata.BatchID, + Attempt: opts.attempt, + MaxRetry: optionInt(opts.maxRetry), + Scheduled: opts.delay > 0, + Time: start, } base.Kind = EventProcessStarted + if base.Attempt > 0 { + retry := base + retry.Kind = EventProcessRetried + safeObserve(ctx, observer, retry) + } safeObserve(ctx, observer, base) + defer func() { + if recovered := recover(); recovered != nil { + finish := base + finish.Kind = EventProcessFailed + finish.Duration = time.Since(start) + finish.Time = time.Now() + finish.Err = handlerPanicError(recovered) + safeObserve(ctx, observer, finish) + panic(recovered) + } + }() err := handler(ctx, job) finish := base @@ -909,27 +1146,31 @@ func wrapObservedHandler(observer Observer, driver Driver, queueName string, job finish.Err = err if err == nil { finish.Kind = EventProcessSucceeded + if busruntime.DeferUntilDeliveryCommitted(ctx, func() { + safeObserve(ctx, observer, finish) + }) { + return nil + } safeObserve(ctx, observer, finish) return nil } finish.Kind = EventProcessFailed safeObserve(ctx, observer, finish) - if finish.Attempt < finish.MaxRetry { - retry := finish - retry.Kind = EventProcessRetried - retry.Err = nil - safeObserve(ctx, observer, retry) - } else { - archive := finish - archive.Kind = EventProcessArchived - archive.Err = nil - safeObserve(ctx, observer, archive) - } return err } } +// handlerPanicError preserves error identity for telemetry while the wrapper +// re-panics so each backend retains its established panic behavior. +func handlerPanicError(recovered any) error { + if err, ok := recovered.(error); ok { + return fmt.Errorf("handler panicked: %w", err) + } + return fmt.Errorf("handler panicked: %v", recovered) +} + +// safeObserve keeps internal and driver event delivery on the same panic-isolated path. func safeObserve(ctx context.Context, observer Observer, event Event) { SafeObserve(ctx, observer, event) } @@ -939,18 +1180,78 @@ func safeObserve(ctx context.Context, observer Observer, event Event) { // This is an advanced helper intended for driver-module implementations. // @group Observability func SafeObserve(ctx context.Context, observer Observer, event Event) { - if observer == nil { + if !observerHasRecipients(observer) { return } if ctx == nil { ctx = context.Background() } + event = normalizeObservedEvent(event) defer func() { _ = recover() }() observer.Observe(ctx, event) } +// normalizeObservedEvent fills portable metadata at the observation boundary so drivers and workflows cannot diverge on envelope construction. +func normalizeObservedEvent(event Event) Event { + if event.SchemaVersion == 0 { + event.SchemaVersion = eventSchemaVersion + } + if event.EventID == "" { + event.EventID = newEventID() + } + if event.Layer == "" { + event.Layer = eventLayerForKind(event.Kind) + } + if event.Time.IsZero() { + event.Time = time.Now() + } + return event +} + +// eventLayerForKind preserves meaningful delivery and workflow boundaries inside the single public event model. +func eventLayerForKind(kind EventKind) EventLayer { + switch kind { + case EventProcessStarted, + EventProcessSucceeded, + EventProcessFailed, + EventProcessRetried, + EventProcessArchived, + EventProcessRecovered, + EventRepublishFailed, + EventSettlementFailed: + return EventLayerWorker + case EventJobStarted, + EventJobSucceeded, + EventJobFailed, + EventChainStarted, + EventChainAdvanced, + EventChainCompleted, + EventChainFailed, + EventBatchStarted, + EventBatchProgressed, + EventBatchCompleted, + EventBatchFailed, + EventBatchCancelled, + EventCallbackStarted, + EventCallbackSucceeded, + EventCallbackFailed: + return EventLayerWorkflow + default: + return EventLayerQueue + } +} + +// newEventID gives each observed fact a process-independent identifier without imposing an observer cost when no observer is installed. +func newEventID() string { + var value [8]byte + if _, err := rand.Read(value[:]); err == nil { + return "evt_" + hex.EncodeToString(value[:]) + } + return fmt.Sprintf("evt_%d", time.Now().UnixNano()) +} + func jobQueueName(job Job) string { queueName := job.jobOptions().queueName if queueName == "" { @@ -967,8 +1268,7 @@ func optionInt(v *int) int { } func jobEventKey(job Job) string { - hash := sha1.Sum(append([]byte(job.Type+":"), job.PayloadBytes()...)) - return fmt.Sprintf("%x", hash[:]) + return observedJobKey(job.Type, job.PayloadBytes()) } func normalizeQueueName(name string) string { diff --git a/observability_branches_test.go b/observability_branches_test.go index 44efc31..a9d65eb 100644 --- a/observability_branches_test.go +++ b/observability_branches_test.go @@ -5,6 +5,8 @@ import ( "errors" "testing" "time" + + "github.com/goforj/queue/busruntime" ) type queueBackendStub struct { @@ -122,8 +124,8 @@ func TestObservedQueue_DispatchClassifiesErrors(t *testing.T) { } } -func TestWrapObservedHandler_EmitsRetriedAndArchived(t *testing.T) { - t.Run("retry path", func(t *testing.T) { +func TestWrapObservedHandler_EmitsRetryOnlyWhenAttemptBegins(t *testing.T) { + t.Run("initial failure does not claim retry settlement", func(t *testing.T) { recorder := &observerRecorder{} h := wrapObservedHandler(recorder, DriverSync, "", "job:retry", nil, func(context.Context, Job) error { return errors.New("boom") @@ -133,15 +135,15 @@ func TestWrapObservedHandler_EmitsRetriedAndArchived(t *testing.T) { if err == nil { t.Fatal("expected handler error") } - if len(recorder.events) != 3 { - t.Fatalf("expected 3 events (started/failed/retried), got %d", len(recorder.events)) + if len(recorder.events) != 2 { + t.Fatalf("expected 2 events (started/failed), got %d", len(recorder.events)) } - if recorder.events[2].Kind != EventProcessRetried { - t.Fatalf("expected retried event, got %q", recorder.events[2].Kind) + if recorder.events[0].Kind != EventProcessStarted || recorder.events[1].Kind != EventProcessFailed { + t.Fatalf("unexpected initial attempt events: %+v", recorder.events) } }) - t.Run("archive path", func(t *testing.T) { + t.Run("later attempt proves retry began", func(t *testing.T) { recorder := &observerRecorder{} h := wrapObservedHandler(recorder, DriverSync, "", "job:archive", nil, func(context.Context, Job) error { return errors.New("boom") @@ -152,14 +154,78 @@ func TestWrapObservedHandler_EmitsRetriedAndArchived(t *testing.T) { t.Fatal("expected handler error") } if len(recorder.events) != 3 { - t.Fatalf("expected 3 events (started/failed/archived), got %d", len(recorder.events)) + t.Fatalf("expected 3 events (retried/started/failed), got %d", len(recorder.events)) } - if recorder.events[2].Kind != EventProcessArchived { - t.Fatalf("expected archived event, got %q", recorder.events[2].Kind) + if recorder.events[0].Kind != EventProcessRetried || recorder.events[1].Kind != EventProcessStarted || recorder.events[2].Kind != EventProcessFailed { + t.Fatalf("unexpected later attempt events: %+v", recorder.events) } }) } +// TestWrapObservedHandlerDefersSuccessUntilSettlement verifies a broker-backed handler cannot report success before acknowledgement. +func TestWrapObservedHandlerDefersSuccessUntilSettlement(t *testing.T) { + recorder := &observerRecorder{} + handler := wrapObservedHandler(recorder, DriverSQS, "", "job:settled", nil, func(context.Context, Job) error { + return nil + }) + ctx, settlement := busruntime.WithDeliverySettlement(context.Background()) + if err := handler(ctx, NewJob("job:settled").OnQueue("default")); err != nil { + t.Fatalf("wrapped handler returned error: %v", err) + } + if len(recorder.events) != 1 || recorder.events[0].Kind != EventProcessStarted { + t.Fatalf("events before settlement = %+v, want process_started only", recorder.events) + } + settlement.Commit() + if len(recorder.events) != 2 || recorder.events[1].Kind != EventProcessSucceeded { + t.Fatalf("events after settlement = %+v, want process_succeeded", recorder.events) + } +} + +// TestWrapObservedHandlerReplacementDecoratorPreservesSettlement verifies a +// decorator cannot replace the driver's settlement boundary with its own. +func TestWrapObservedHandlerReplacementDecoratorPreservesSettlement(t *testing.T) { + recorder := &observerRecorder{} + ctx, settlement := busruntime.WithDeliverySettlement(context.Background()) + wantIdentity, ok := busruntime.DeliverySettlementIdentityFromContext(ctx) + if !ok { + t.Fatal("driver context did not retain its settlement identity") + } + var replacementSettlement *busruntime.DeliverySettlement + handler := wrapObservedHandler( + recorder, + DriverSQS, + "", + "job:settled-replacement", + func(context.Context) context.Context { + decorated, spoofed := busruntime.WithDeliverySettlement(context.Background()) + replacementSettlement = spoofed + return decorated + }, + func(handlerCtx context.Context, _ Job) error { + gotIdentity, identityOK := busruntime.DeliverySettlementIdentityFromContext(handlerCtx) + if !identityOK || gotIdentity != wantIdentity { + t.Fatalf("handler settlement identity = %+v, %t; want driver identity", gotIdentity, identityOK) + } + return nil + }, + ) + + if err := handler(ctx, NewJob("job:settled-replacement").OnQueue("default")); err != nil { + t.Fatalf("wrapped handler returned error: %v", err) + } + if len(recorder.events) != 1 || recorder.events[0].Kind != EventProcessStarted { + t.Fatalf("events before settlement = %+v, want process_started only", recorder.events) + } + replacementSettlement.Commit() + if len(recorder.events) != 1 { + t.Fatalf("replacement settlement published driver success: %+v", recorder.events) + } + settlement.Commit() + if len(recorder.events) != 2 || recorder.events[1].Kind != EventProcessSucceeded { + t.Fatalf("events after driver settlement = %+v, want process_succeeded", recorder.events) + } +} + func TestWrapObservedHandler_DecoratesObserverContext(t *testing.T) { recorder := &observerContextRecorder{} h := wrapObservedHandler(recorder, DriverSync, "", "job:decorated", func(ctx context.Context) context.Context { @@ -361,7 +427,9 @@ func TestObservabilityHelpers_ResolveAndSnapshotFallbacks(t *testing.T) { cfg: Config{Driver: DriverNull}, driver: DriverNull, }, - registered: map[string]Handler{}, + externalQueueRuntimeState: &externalQueueRuntimeState{ + registered: map[string]Handler{}, + }, } if err := Ready(context.Background(), readyRuntime); !errors.Is(err, wantErr) { t.Fatalf("expected ready error %v, got %v", wantErr, err) diff --git a/observability_job_type.go b/observability_job_type.go index 9c0db53..2ad93c2 100644 --- a/observability_job_type.go +++ b/observability_job_type.go @@ -1,32 +1,71 @@ package queue -import "encoding/json" +import "github.com/goforj/queue/internal/jobidentity" -type observedBusEnvelope struct { - Job struct { - Type string `json:"type"` - } `json:"job"` +// ObservedJobMetadata contains application identity and correlation resolved +// from direct driver metadata or a legacy internal delivery envelope. +// JobKey groups the same logical type and payload used by UniqueFor without becoming its persisted, queue-scoped key. +// @group Driver Integration +type ObservedJobMetadata struct { + JobType string + JobKey string + DispatchID string + JobID string + ChainID string + BatchID string +} + +// ResolveObservedJobMetadata bridges legacy versioned workflow envelopes into +// the event model. Direct-aware drivers should use ResolveObservedJobMetadataFromJob. +// Malformed, unknown-version, and non-internal payloads remain observable as their raw physical job. +// @group Driver Integration +func ResolveObservedJobMetadata(rawType string, payload []byte) ObservedJobMetadata { + metadata, _ := resolveObservedJobMetadata(rawType, payload) + return metadata +} + +// ResolveObservedJobMetadataFromJob returns correlation from the direct driver +// metadata channel when present and otherwise decodes a legacy workflow delivery. +// +// This is an advanced helper intended for optional driver integrations. +// @group Driver Integration +func ResolveObservedJobMetadataFromJob(job Job) ObservedJobMetadata { + metadata := ResolveObservedJobMetadata(job.Type, job.PayloadBytes()) + direct := DriverMetadata(job) + if direct.SchemaVersion == 0 { + return metadata + } + metadata.DispatchID = direct.DispatchID + metadata.JobID = direct.JobID + metadata.ChainID = direct.ChainID + metadata.BatchID = direct.BatchID + return metadata +} + +// resolveObservedJobMetadata returns both correlation fields and the exact logical payload used by delivery policy. +func resolveObservedJobMetadata(rawType string, payload []byte) (ObservedJobMetadata, []byte) { + logical := resolveLogicalJob(rawType, payload) + metadata := ObservedJobMetadata{ + JobType: logical.jobType, + JobKey: observedJobKey(logical.jobType, logical.payload), + DispatchID: logical.dispatchID, + JobID: logical.jobID, + ChainID: logical.chainID, + BatchID: logical.batchID, + } + return metadata, logical.payload } // ResolveObservedJobType returns the effective application job type that should -// be emitted to observers. External workers may process internal bus wrapper -// jobs (for example, "bus:job") whose payload embeds the real application job +// be emitted to observers. External workers may process private workflow delivery +// envelopes (for example, "bus:job") whose payload embeds the real application job // type. When possible, this helper unwraps that payload so dashboards and // metrics reflect the user-facing job type instead of the transport wrapper. func ResolveObservedJobType(rawType string, payload []byte) string { - if rawType == "" { - return "" - } - if len(payload) == 0 || len(rawType) < 4 || rawType[:4] != "bus:" { - return rawType - } + return ResolveObservedJobMetadata(rawType, payload).JobType +} - var env observedBusEnvelope - if err := json.Unmarshal(payload, &env); err != nil { - return rawType - } - if env.Job.Type == "" { - return rawType - } - return env.Job.Type +// observedJobKey keeps telemetry correlation stable when volatile workflow IDs surround the application payload. +func observedJobKey(jobType string, payload []byte) string { + return jobidentity.ObservedKey(jobType, payload) } diff --git a/observability_job_type_test.go b/observability_job_type_test.go index d1cf4ec..b19876e 100644 --- a/observability_job_type_test.go +++ b/observability_job_type_test.go @@ -1,7 +1,11 @@ package queue -import "testing" +import ( + "bytes" + "testing" +) +// TestResolveObservedJobType verifies the compatibility helper keeps raw and internal job names meaningful. func TestResolveObservedJobType(t *testing.T) { t.Run("plain job type passes through", func(t *testing.T) { got := ResolveObservedJobType("monitoring:check", []byte(`{"anything":"ok"}`)) @@ -11,14 +15,14 @@ func TestResolveObservedJobType(t *testing.T) { }) t.Run("bus wrapper unwraps nested job type", func(t *testing.T) { - got := ResolveObservedJobType("bus:job", []byte(`{"job":{"type":"monitoring:check"}}`)) + got := ResolveObservedJobType("bus:job", []byte(`{"schema_version":1,"job":{"type":"monitoring:check"}}`)) if got != "monitoring:check" { t.Fatalf("expected unwrapped job type, got %q", got) } }) t.Run("other bus wrappers also unwrap", func(t *testing.T) { - got := ResolveObservedJobType("bus:batch:job", []byte(`{"job":{"type":"reports:build"}}`)) + got := ResolveObservedJobType("bus:batch:job", []byte(`{"schema_version":1,"job":{"type":"reports:build"}}`)) if got != "reports:build" { t.Fatalf("expected unwrapped batch job type, got %q", got) } @@ -31,10 +35,137 @@ func TestResolveObservedJobType(t *testing.T) { } }) + t.Run("missing envelope version falls back to raw type", func(t *testing.T) { + got := ResolveObservedJobType("bus:job", []byte(`{"job":{"type":"monitoring:check"}}`)) + if got != "bus:job" { + t.Fatalf("expected raw type for missing schema, got %q", got) + } + }) + t.Run("missing nested job type falls back to raw type", func(t *testing.T) { - got := ResolveObservedJobType("bus:job", []byte(`{"job":{}}`)) + got := ResolveObservedJobType("bus:job", []byte(`{"schema_version":1,"job":{}}`)) if got != "bus:job" { t.Fatalf("expected fallback raw job type, got %q", got) } }) + + t.Run("unknown internal-looking job type cannot supply metadata", func(t *testing.T) { + got := ResolveObservedJobType("bus:tenant", []byte(`{"schema_version":1,"job":{"type":"monitoring:check"}}`)) + if got != "bus:tenant" { + t.Fatalf("expected raw unknown type, got %q", got) + } + }) + + t.Run("unknown envelope version cannot supply metadata", func(t *testing.T) { + got := ResolveObservedJobType("bus:job", []byte(`{"schema_version":2,"job":{"type":"monitoring:check"}}`)) + if got != "bus:job" { + t.Fatalf("expected raw type for unknown schema, got %q", got) + } + }) + + t.Run("application payload resembling an envelope is not decoded", func(t *testing.T) { + got := ResolveObservedJobType("monitoring:check", []byte(`{"schema_version":1,"job":{"type":"spoofed"}}`)) + if got != "monitoring:check" { + t.Fatalf("expected application type, got %q", got) + } + }) +} + +// TestResolveObservedJobMetadata verifies every observable layer can join one internal delivery to its logical job. +func TestResolveObservedJobMetadata(t *testing.T) { + payload := []byte(`{"schema_version":1,"dispatch_id":"dsp_1","job_id":"job_1","chain_id":"chn_1","batch_id":"bat_1","job":{"type":"monitoring:check","payload":"eyJpZCI6MX0="}}`) + metadata := ResolveObservedJobMetadata("bus:job", payload) + if metadata.JobType != "monitoring:check" { + t.Fatalf("job type = %q, want monitoring:check", metadata.JobType) + } + if metadata.DispatchID != "dsp_1" || metadata.JobID != "job_1" || metadata.ChainID != "chn_1" || metadata.BatchID != "bat_1" { + t.Fatalf("correlation metadata is incomplete: %+v", metadata) + } + wantKey := ResolveObservedJobMetadata("monitoring:check", []byte(`{"id":1}`)).JobKey + if metadata.JobKey != wantKey { + t.Fatalf("logical job key = %q, want %q", metadata.JobKey, wantKey) + } + + fallback := ResolveObservedJobMetadata("bus:job", []byte(`{`)) + if fallback.JobType != "bus:job" || fallback.DispatchID != "" || fallback.JobKey == "" { + t.Fatalf("invalid envelope fallback = %+v", fallback) + } + + unknownType := ResolveObservedJobMetadata("bus:tenant", payload) + if unknownType.JobType != "bus:tenant" || unknownType.DispatchID != "" || unknownType.JobID != "" { + t.Fatalf("unknown internal-looking type decoded metadata: %+v", unknownType) + } + + unknownVersion := ResolveObservedJobMetadata("bus:job", []byte(`{"schema_version":2,"dispatch_id":"spoofed","job":{"type":"monitoring:check"}}`)) + if unknownVersion.JobType != "bus:job" || unknownVersion.DispatchID != "" || unknownVersion.JobID != "" { + t.Fatalf("unknown schema decoded metadata: %+v", unknownVersion) + } +} + +// TestResolveObservedJobMetadataInternalTypes locks decoding to the four version-one workflow envelopes already on the wire. +func TestResolveObservedJobMetadataInternalTypes(t *testing.T) { + tests := []struct { + name string + jobType string + payload string + wantType string + wantChain string + wantBatch string + }{ + { + name: "direct job", + jobType: "bus:job", + payload: `{"schema_version":1,"dispatch_id":"dsp_direct","job_id":"job_direct","job":{"type":"reports:build","payload":"e30="}}`, + wantType: "reports:build", + }, + { + name: "chain node", + jobType: "bus:chain:node", + payload: `{"schema_version":1,"dispatch_id":"dsp_chain","job_id":"job_chain","chain_id":"chn_1","job":{"type":"reports:build","payload":"e30="}}`, + wantType: "reports:build", + wantChain: "chn_1", + }, + { + name: "batch job", + jobType: "bus:batch:job", + payload: `{"schema_version":1,"dispatch_id":"dsp_batch","job_id":"job_batch","batch_id":"bat_1","job":{"type":"reports:build","payload":"e30="}}`, + wantType: "reports:build", + wantBatch: "bat_1", + }, + { + name: "callback without application job", + jobType: "bus:callback", + payload: `{"schema_version":1,"dispatch_id":"dsp_callback","job_id":"job_callback","batch_id":"bat_1","job":{}}`, + wantType: "bus:callback", + wantBatch: "bat_1", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + metadata := ResolveObservedJobMetadata(test.jobType, []byte(test.payload)) + if metadata.JobType != test.wantType || metadata.DispatchID == "" || metadata.JobID == "" { + t.Fatalf("decoded metadata = %+v, want type %q and correlation IDs", metadata, test.wantType) + } + if metadata.ChainID != test.wantChain || metadata.BatchID != test.wantBatch { + t.Fatalf("workflow correlation = chain:%q batch:%q, want chain:%q batch:%q", metadata.ChainID, metadata.BatchID, test.wantChain, test.wantBatch) + } + }) + } +} + +// TestResolveObservedJobMetadataLogicalKey verifies random envelope IDs cannot fragment telemetry identity or mutate delivered bytes. +func TestResolveObservedJobMetadataLogicalKey(t *testing.T) { + first := []byte(`{"schema_version":1,"dispatch_id":"dsp_1","job_id":"job_1","job":{"type":"reports:build","payload":"eyJpZCI6MX0="}}`) + second := []byte(`{"schema_version":1,"dispatch_id":"dsp_2","job_id":"job_2","job":{"type":"reports:build","payload":"eyJpZCI6MX0="}}`) + original := append([]byte(nil), first...) + + firstMetadata := ResolveObservedJobMetadata("bus:job", first) + secondMetadata := ResolveObservedJobMetadata("bus:job", second) + if firstMetadata.JobKey == "" || firstMetadata.JobKey != secondMetadata.JobKey { + t.Fatalf("logical keys differ across correlation IDs: %q != %q", firstMetadata.JobKey, secondMetadata.JobKey) + } + if !bytes.Equal(first, original) { + t.Fatalf("decoder mutated payload: got %q, want %q", first, original) + } } diff --git a/observability_test.go b/observability_test.go index 920b48c..0419e16 100644 --- a/observability_test.go +++ b/observability_test.go @@ -6,6 +6,8 @@ import ( "sync/atomic" "testing" "time" + + "github.com/goforj/queue/busruntime" ) func startTestQueue(t *testing.T, q queueRuntime) { @@ -71,6 +73,417 @@ func TestStatsCollector_CapturesProcessingFailure(t *testing.T) { t.Fatal("expected failed counter to be incremented") } +// TestStatsCollector_WorkerpoolPanicClosesActive verifies panic recovery does +// not leave the observer gauge or its per-delivery correlation state live. +func TestStatsCollector_WorkerpoolPanicClosesActive(t *testing.T) { + collector := NewStatsCollector() + q, err := New( + Config{Driver: DriverWorkerpool}, + WithObserver(collector), + WithWorkers(1), + ) + if err != nil { + t.Fatalf("new workerpool queue: %v", err) + } + q.Register("job:observer:panic", func(context.Context, Message) error { + panic("handler panic") + }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workerpool: %v", err) + } + t.Cleanup(func() { + if shutdownErr := q.Shutdown(context.Background()); shutdownErr != nil { + t.Errorf("shutdown workerpool: %v", shutdownErr) + } + }) + if _, err := q.Dispatch(NewJob("job:observer:panic")); err != nil { + t.Fatalf("dispatch panicking job: %v", err) + } + + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + counters, ok := collector.Snapshot().Queue("default") + if ok && counters.Failed == 1 && counters.Active == 0 { + collector.mu.RLock() + state := collector.byQueue["default"] + activeKeys := len(state.activeByKey) + activeSettlements := len(state.activeSettlements) + uncorrelatedActive := state.uncorrelatedActive + collector.mu.RUnlock() + if activeKeys != 0 || activeSettlements != 0 || uncorrelatedActive != 0 { + t.Fatalf("panic correlation state = keys:%d settlements:%d uncorrelated:%d, want empty", activeKeys, activeSettlements, uncorrelatedActive) + } + return + } + time.Sleep(10 * time.Millisecond) + } + t.Fatalf("panic counters = %+v, want failed=1 active=0", collector.Snapshot().ByQueue["default"]) +} + +// TestHandlerPanicErrorPreservesErrorIdentity verifies panic telemetry remains +// useful to errors.Is callers without changing non-error panic formatting. +func TestHandlerPanicErrorPreservesErrorIdentity(t *testing.T) { + sentinel := errors.New("panic sentinel") + if err := handlerPanicError(sentinel); !errors.Is(err, sentinel) { + t.Fatalf("error panic = %v, want wrapped sentinel", err) + } + if err := handlerPanicError("panic value"); err == nil || err.Error() != "handler panicked: panic value" { + t.Fatalf("value panic = %v, want stable diagnostic", err) + } +} + +// TestWrapObservedHandlerReportsAndRepanics pins failure telemetry without +// turning a synchronous backend panic into an ordinary returned error. +func TestWrapObservedHandlerReportsAndRepanics(t *testing.T) { + var events []Event + wrapped := wrapObservedHandler( + ObserverFunc(func(_ context.Context, event Event) { events = append(events, event) }), + DriverSync, + "default", + "job:panic", + nil, + func(context.Context, Job) error { panic("panic value") }, + ) + + var recovered any + func() { + defer func() { recovered = recover() }() + _ = wrapped(context.Background(), NewJob("job:panic")) + }() + if recovered != "panic value" { + t.Fatalf("recovered panic = %#v, want original value", recovered) + } + if len(events) != 2 || events[0].Kind != EventProcessStarted || events[1].Kind != EventProcessFailed { + t.Fatalf("panic events = %+v, want process_started then process_failed", events) + } + if events[1].Err == nil || events[1].Err.Error() != "handler panicked: panic value" { + t.Fatalf("panic failure error = %v, want stable diagnostic", events[1].Err) + } +} + +// TestStatsCollector_SettlementFailureClosesActive verifies unresolved broker settlement cannot leak active delivery gauges or fabricate an application outcome. +func TestStatsCollector_SettlementFailureClosesActive(t *testing.T) { + collector := NewStatsCollector() + now := time.Now() + ctx, _ := busruntime.WithDeliverySettlement(context.Background()) + collector.Observe(ctx, Event{ + Kind: EventProcessStarted, + Driver: DriverSQS, + Queue: "default", + JobID: "job-settlement", + JobKey: "job-settlement", + Time: now, + }) + collector.Observe(ctx, Event{ + Kind: EventProcessStarted, + Driver: DriverSQS, + Queue: "default", + JobID: "job-settlement", + JobKey: "job-settlement", + Time: now, + }) + if active := collector.Snapshot().Active("default"); active != 1 { + t.Fatalf("duplicate start active = %d, want 1 for the same physical identity", active) + } + collector.Observe(ctx, Event{ + Kind: EventSettlementFailed, + Driver: DriverSQS, + Queue: "default", + JobID: "job-settlement", + JobKey: "job-settlement", + Err: errors.New("delete failed"), + Time: now.Add(time.Millisecond), + }) + counters, ok := collector.Snapshot().Queue("default") + if !ok { + t.Fatal("expected settlement queue counters") + } + if counters.Active != 0 || counters.Processed != 0 || counters.Failed != 0 { + t.Fatalf("settlement counters = %+v, want active closed without terminal application count", counters) + } +} + +// TestStatsCollector_SettlementFailureDoesNotGuessUncorrelatedActive verifies +// settlement facts without physical identity cannot consume a live gauge. +func TestStatsCollector_SettlementFailureDoesNotGuessUncorrelatedActive(t *testing.T) { + collector := NewStatsCollector() + now := time.Now() + collector.Observe(context.Background(), Event{ + Kind: EventProcessStarted, + Queue: "default", + Time: now, + }) + collector.Observe(context.Background(), Event{ + Kind: EventSettlementFailed, + Queue: "default", + Err: errors.New("settlement failed"), + Time: now.Add(time.Millisecond), + }) + + counters, ok := collector.Snapshot().Queue("default") + if !ok { + t.Fatal("expected default queue counters") + } + if counters.Active != 1 || counters.Processed != 0 || counters.Failed != 0 { + t.Fatalf("context-free settlement counters = %+v, want active unchanged without physical identity", counters) + } +} + +// TestStatsCollector_LateIdentitylessSettlementCannotCloseNewExecution pins +// sequential tuple reuse, where event fields cannot identify the old receipt. +func TestStatsCollector_LateIdentitylessSettlementCannotCloseNewExecution(t *testing.T) { + collector := NewStatsCollector() + now := time.Now() + oldCtx, _ := busruntime.WithDeliverySettlement(context.Background()) + newCtx, _ := busruntime.WithDeliverySettlement(context.Background()) + base := Event{ + Driver: DriverSQS, + Queue: "default", + DispatchID: "dispatch-reused", + JobID: "job-reused", + Attempt: 1, + } + + started := base + started.Kind = EventProcessStarted + started.Time = now + collector.Observe(oldCtx, started) + failed := base + failed.Kind = EventProcessFailed + failed.Err = errors.New("old handler failed") + failed.Time = now.Add(time.Millisecond) + collector.Observe(oldCtx, failed) + started.Time = now.Add(2 * time.Millisecond) + collector.Observe(newCtx, started) + + lateSettlement := base + lateSettlement.Kind = EventSettlementFailed + lateSettlement.Err = errors.New("old acknowledgement failed") + lateSettlement.Time = now.Add(3 * time.Millisecond) + collector.Observe(context.Background(), lateSettlement) + if active := collector.Snapshot().Active("default"); active != 1 { + t.Fatalf("active after late identity-less settlement = %d, want newer execution retained", active) + } + + succeeded := base + succeeded.Kind = EventProcessSucceeded + succeeded.Time = now.Add(4 * time.Millisecond) + collector.Observe(newCtx, succeeded) + counters, _ := collector.Snapshot().Queue("default") + if counters.Active != 0 || counters.Failed != 1 || counters.Processed != 1 { + t.Fatalf("reused tuple terminal counters = %+v, want both physical executions closed once", counters) + } +} + +// TestStatsCollector_SettlementFailureClosesOnlyItsExecution verifies a failed +// acknowledgement cannot consume another delivery's active gauge after the +// same handler attempt already emitted process_failed. +func TestStatsCollector_SettlementFailureClosesOnlyItsExecution(t *testing.T) { + collector := NewStatsCollector() + now := time.Now() + failedCtx, _ := busruntime.WithDeliverySettlement(context.Background()) + runningCtx, _ := busruntime.WithDeliverySettlement(context.Background()) + + collector.Observe(failedCtx, Event{ + Kind: EventProcessStarted, + Driver: DriverSQS, + Queue: "default", + DispatchID: "dispatch-failed", + JobID: "job-failed", + Time: now, + }) + collector.Observe(runningCtx, Event{ + Kind: EventProcessStarted, + Driver: DriverSQS, + Queue: "default", + DispatchID: "dispatch-running", + JobID: "job-running", + Time: now.Add(time.Millisecond), + }) + collector.Observe(failedCtx, Event{ + Kind: EventProcessFailed, + Driver: DriverSQS, + Queue: "default", + DispatchID: "dispatch-failed", + JobID: "job-failed", + Err: errors.New("handler failed"), + Time: now.Add(2 * time.Millisecond), + }) + collector.Observe(failedCtx, Event{ + Kind: EventSettlementFailed, + Driver: DriverSQS, + Queue: "default", + DispatchID: "dispatch-failed", + JobID: "job-failed", + Err: errors.New("delete failed"), + Time: now.Add(3 * time.Millisecond), + }) + + counters, ok := collector.Snapshot().Queue("default") + if !ok { + t.Fatal("expected default queue counters") + } + if counters.Active != 1 || counters.Failed != 1 || counters.Processed != 0 { + t.Fatalf("counters after failed settlement = %+v, want one unrelated active execution and one handler failure", counters) + } + + collector.Observe(runningCtx, Event{ + Kind: EventProcessSucceeded, + Driver: DriverSQS, + Queue: "default", + DispatchID: "dispatch-running", + JobID: "job-running", + Time: now.Add(4 * time.Millisecond), + }) + counters, _ = collector.Snapshot().Queue("default") + if counters.Active != 0 || counters.Failed != 1 || counters.Processed != 1 { + t.Fatalf("terminal counters = %+v, want both executions closed exactly once", counters) + } +} + +// TestStatsCollector_SettlementFailureRequiresIdentityForDuplicateTuple +// verifies event correlation cannot masquerade as physical delivery identity. +func TestStatsCollector_SettlementFailureRequiresIdentityForDuplicateTuple(t *testing.T) { + collector := NewStatsCollector() + now := time.Now() + base := Event{ + Driver: DriverSQS, + Queue: "default", + DispatchID: "dispatch-duplicate", + JobID: "job-duplicate", + Attempt: 2, + } + started := base + started.Kind = EventProcessStarted + started.Time = now + collector.Observe(context.Background(), started) + started.Time = now.Add(time.Millisecond) + collector.Observe(context.Background(), started) + ambiguousSettlement := base + ambiguousSettlement.Kind = EventSettlementFailed + ambiguousSettlement.Err = errors.New("ambiguous ack failed") + ambiguousSettlement.Time = now.Add(1500 * time.Microsecond) + collector.Observe(context.Background(), ambiguousSettlement) + if active := collector.Snapshot().Active("default"); active != 2 { + t.Fatalf("ambiguous settlement active = %d, want both indistinguishable executions retained", active) + } + + failed := base + failed.Kind = EventProcessFailed + failed.Err = errors.New("handler failed") + failed.Time = now.Add(2 * time.Millisecond) + collector.Observe(context.Background(), failed) + settlementFailed := base + settlementFailed.Kind = EventSettlementFailed + settlementFailed.Err = errors.New("ack failed") + settlementFailed.Time = now.Add(3 * time.Millisecond) + collector.Observe(context.Background(), settlementFailed) + + counters, ok := collector.Snapshot().Queue("default") + if !ok { + t.Fatal("expected default queue counters") + } + if counters.Active != 1 || counters.Failed != 1 { + t.Fatalf("duplicate tuple counters after settlement failure = %+v, want one unrelated physical duplicate active", counters) + } + + succeeded := base + succeeded.Kind = EventProcessSucceeded + succeeded.Time = now.Add(4 * time.Millisecond) + collector.Observe(context.Background(), succeeded) + counters, _ = collector.Snapshot().Queue("default") + if counters.Active != 0 || counters.Failed != 1 || counters.Processed != 1 { + t.Fatalf("duplicate tuple terminal counters = %+v, want both physical deliveries closed once", counters) + } +} + +// TestStatsCollector_SettlementFailureRequiresPhysicalIdentity verifies the +// collector fails closed when an older or custom driver loses settlement context. +func TestStatsCollector_SettlementFailureRequiresPhysicalIdentity(t *testing.T) { + for _, test := range []struct { + name string + startWithIdentity bool + settleWithIdentity bool + wantActive int64 + }{ + {name: "context-free", wantActive: 1}, + {name: "mixed-version lost identity", startWithIdentity: true, wantActive: 1}, + {name: "current identity", startWithIdentity: true, settleWithIdentity: true}, + } { + t.Run(test.name, func(t *testing.T) { + collector := NewStatsCollector() + now := time.Now() + startCtx := context.Background() + if test.startWithIdentity { + startCtx, _ = busruntime.WithDeliverySettlement(startCtx) + } + settlementCtx := context.Background() + if test.settleWithIdentity { + settlementCtx = startCtx + } + started := Event{ + Kind: EventProcessStarted, + Driver: DriverSQS, + Queue: "default", + DispatchID: "dispatch-legacy", + JobID: "job-legacy", + Attempt: 1, + Time: now, + } + collector.Observe(startCtx, started) + settlementFailed := started + settlementFailed.Kind = EventSettlementFailed + settlementFailed.Err = errors.New("legacy delete failed") + settlementFailed.Time = now.Add(time.Millisecond) + collector.Observe(settlementCtx, settlementFailed) + + counters, ok := collector.Snapshot().Queue("default") + if !ok { + t.Fatal("expected default queue counters") + } + if counters.Active != test.wantActive || counters.Processed != 0 || counters.Failed != 0 { + t.Fatalf("settlement counters = %+v, want active=%d and no terminal application count", counters, test.wantActive) + } + }) + } + + collector := NewStatsCollector() + ctx, _ := busruntime.WithDeliverySettlement(context.Background()) + started := Event{ + Kind: EventProcessStarted, + Driver: DriverSQS, + Queue: "default", + DispatchID: "dispatch-lost-process-context", + JobID: "job-lost-process-context", + Time: time.Now(), + } + collector.Observe(ctx, started) + succeeded := started + succeeded.Kind = EventProcessSucceeded + succeeded.Time = started.Time.Add(time.Millisecond) + collector.Observe(context.Background(), succeeded) + if counters, _ := collector.Snapshot().Queue("default"); counters.Active != 1 || counters.Processed != 1 { + t.Fatalf("identity-less process terminal counters = %+v, want exact execution retained", counters) + } + settlementFailed := started + settlementFailed.Kind = EventSettlementFailed + settlementFailed.Err = errors.New("late exact settlement") + settlementFailed.Time = started.Time.Add(2 * time.Millisecond) + collector.Observe(ctx, settlementFailed) + if active := collector.Snapshot().Active("default"); active != 0 { + t.Fatalf("late exact settlement active = %d, want already-closed execution unchanged", active) + } +} + +// TestCollectorQueueStateRejectsMissingActiveClose verifies a terminal fact +// cannot decrement a queue that has no matching execution state. +func TestCollectorQueueStateRejectsMissingActiveClose(t *testing.T) { + state := collectorQueueState{} + if state.closeByKey("missing") { + t.Fatal("missing execution unexpectedly closed") + } +} + func TestStatsSnapshot_Getters(t *testing.T) { collector := NewStatsCollector() now := time.Now() diff --git a/permanent_error_contract_test.go b/permanent_error_contract_test.go new file mode 100644 index 0000000..210bbd7 --- /dev/null +++ b/permanent_error_contract_test.go @@ -0,0 +1,42 @@ +package queue_test + +import ( + "context" + "errors" + "testing" + + "github.com/goforj/queue" +) + +// TestPublicPermanentStopsRetries verifies applications can declare terminal errors without depending on an internal runtime package. +func TestPublicPermanentStopsRetries(t *testing.T) { + q, err := queue.NewSync() + if err != nil { + t.Fatalf("new sync queue: %v", err) + } + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + t.Cleanup(func() { _ = q.Shutdown(context.Background()) }) + + cause := errors.New("payload cannot be processed") + calls := 0 + q.Register("contract:permanent", func(context.Context, queue.Message) error { + calls++ + return queue.Permanent(cause) + }) + + _, err = q.Dispatch(queue.NewJob("contract:permanent").Retry(5)) + if !queue.IsPermanent(err) || !errors.Is(err, cause) { + t.Fatalf("dispatch error = %v, want permanent cause", err) + } + if calls != 1 { + t.Fatalf("handler calls = %d, want 1", calls) + } + if queue.Permanent(nil) != nil { + t.Fatal("Permanent(nil) must remain nil") + } + if marked := queue.Permanent(cause); queue.Permanent(marked) != marked { + t.Fatal("Permanent must be idempotent") + } +} diff --git a/plan.md b/plan.md new file mode 100644 index 0000000..56aafc5 --- /dev/null +++ b/plan.md @@ -0,0 +1,642 @@ +# Queue Unification Plan + +Status: Active + +Last updated: 2026-07-20 + +Baseline: `origin/main` at `18a7647` + +Working branch: `refactor/unify-queue-workflow` + +## Goal + +Make `queue` a coherent, dependable queue and workflow library with one normal application model, explicit ownership between delivery and orchestration, truthful backend guarantees, compatibility-conscious evolution, and validation that covers every module and supported deployment shape. + +This is the living execution plan. Keep it current as work lands. A task is complete only when its acceptance criteria and applicable validation pass. + +## Working Agreement + +- Work from the highest-priority unblocked item in the current milestone. +- Add a regression test before or with every correctness fix. +- Exercise changes through the public `*queue.Queue` path, not only private runtimes or driver internals. +- Preserve source/API, configuration, persisted-data, runtime, operational, and minimum-Go-version compatibility by default. +- Record any necessary incompatibility in the decision log before implementation, including why compatibility cannot be preserved and how users migrate. +- Treat generators, GoDoc examples, and templates as authoritative; regenerate checked-in documentation and verify a second generation produces no diff. +- Validate every affected Go module independently. Workspace success alone is insufficient. +- Preserve intentional sibling `replace` directives used for repository testing. +- Use `/tmp` for all test renders and generated application compositions. +- Keep this file focused on decisions, executable work, evidence, and remaining risk. Move lengthy design specifications into dedicated documents and link them here. + +## North-Star Model + +The intended architecture has four explicit layers: + +1. **Application facade** — one public `*queue.Queue`, one canonical `Job`, one handler `Message`, and workflow builders. +2. **Workflow engine** — chain/batch state transitions, correlation, continuation scheduling, and durable workflow policy. +3. **Worker runtime** — handler registration, execution, concurrency, retry coordination, draining, and lifecycle state. +4. **Driver SPI** — enqueue, delivery metadata, acknowledgement or settlement, backend resources, and explicit capabilities. + +Cross-cutting administration and observability may span layers, but their events and capabilities must identify which layer produced them. + +The normal application path remains `*queue.Queue`. Advanced packages may expose extension points, but they must not create a second contradictory application model. + +## Unified Public Surface + +The following direction is accepted and governs implementation order: + +- `queue` owns the canonical public `Queue`, `Job`, `Message`, handler, middleware, workflow builder/state, event, observer, store, and capability types. +- `Queue.Dispatch`, `Queue.Chain`, and `Queue.Batch` compose the same `Job` and handler model. A workflow is not a second dispatch runtime. +- Ordinary jobs remain ordinary queue jobs. They are not wrapped in a workflow envelope merely to pass through the public facade. +- There is one `queue.Observer` receiving one extensible `queue.Event` model for dispatch, enqueue, attempt, queue-control, chain, batch, and continuation facts. +- Observation is best-effort telemetry and never controls retries, workflow transitions, or business continuations. +- Reliable workflow continuations are named `Job` values persisted and dispatched through the queue. Function callbacks may remain only as explicitly ephemeral compatibility helpers. +- `bus` becomes a deprecated compatibility facade over the canonical queue model. Compatibility declarations may remain while callers migrate, but the package must not retain a second orchestration engine, store implementation, event producer, or lifecycle owner. Its independent fake remains explicit debt under M5-07. +- Internal delivery and workflow components may remain separate, but that separation is an implementation detail with explicit outcome contracts rather than duplicated application APIs. + +The target application experience is intentionally small: + +```go +q, err := queue.NewWorkerpool( + queue.WithWorkers(4), + queue.WithObserver(observer), + queue.WithStore(store), +) +if err != nil { + return err +} + +q.Register("reports:build", buildReport) +q.Register("reports:publish", publishReport) +q.Register("reports:failed", recordFailure) + +_, err = q.Dispatch(queue.NewJob("reports:build").Payload(payload)) +if err != nil { + return err +} + +_, err = q.Chain( + queue.NewJob("reports:build").Payload(payload), + queue.NewJob("reports:publish"), +). + OnFailure(queue.NewJob("reports:failed")). + Dispatch(ctx) +return err +``` + +`OnFailure` illustrates the target durable continuation API and does not exist yet. Existing closure-based `Catch`, `Then`, `Progress`, and `Finally` methods remain compatibility surfaces until the durable replacements are implemented and documented. + +The canonical event model remains flat and easy to log. It carries the union of useful transport and workflow correlation fields without nested type hierarchies: + +```go +type Observer interface { + Observe(context.Context, Event) +} + +type Event struct { + SchemaVersion int + EventID string + Layer EventLayer + Kind EventKind + Driver Driver + Queue string + DispatchID string + JobID string + ChainID string + BatchID string + JobType string + Attempt int + MaxRetry int + Scheduled bool + Duration time.Duration + Time time.Time + Err error +} +``` + +Exact field evolution remains compatibility-sensitive. Existing fields and event string values should be retained or adapted during migration where doing so does not preserve contradictory semantics. + +## Compatibility Guardrails + +- Preserve the root `Queue`, `Job`, `Message`, handler, and builder APIs wherever viable. +- Prefer forwarding aliases and deprecation periods over immediate removal of public `bus` APIs. +- Do not silently change the meaning of an existing option. Where semantics are currently broken or inconsistent, document the corrected contract and add focused compatibility tests. +- Version internal transport/workflow envelopes before changing their persisted or wire representation. +- Define mixed-version producer/worker behavior for every envelope change. +- Define forward and rollback behavior for SQL schema changes before applying them. +- Validate root and optional driver module versions independently before release. +- Do not raise the minimum Go version unless an implementation or required dependency demands it; record the exact constraint here. + +## Baseline Findings + +The 2026-07-18 audit established the following starting point: + +- [x] Inventory all 12 Go modules. +- [x] Root unit tests pass. +- [x] Root race tests pass. +- [x] Root `go vet` fails because mutex-bearing runtime values are copied. +- [x] NATS, SQS, and RabbitMQ test packages fail to compile after the observer signature change. +- [x] The examples module does not validate independently with `GOWORK=off` because dependency sums are incomplete. +- [x] CI root tests do not cover nested driver modules. +- [x] Public-path reproductions confirm incorrect workflow retry state and broken `UniqueFor` behavior. +- [x] Driver, workflow, lifecycle, observability, capability, fake, and documentation contracts have been mapped. + +The audit found release-critical semantic risks. Until the relevant milestones complete, do not make stronger durability, uniqueness, workflow recovery, or cross-backend equivalence claims. + +## Decision Gates + +Resolve each decision before implementing the dependent architectural milestone. Record the chosen option and rationale in the decision log. + +### D-001: Retry Ownership + +Status: Accepted + +Required before: M1 workflow retry implementation and M2 SPI design + +Decision: the worker runtime coordinates retries, drivers commit settlement or retry scheduling, and workflows transition only from committed attempt outcomes. Public handlers continue to return `error`; internal adapters classify retryable, permanent, and exhausted outcomes. An attempted retry is not a workflow fact until the responsible driver confirms it was scheduled. + +The shared internal classifier uses zero-based attempt numbers and four decisions: + +- success commits application success; +- retry preserves nonterminal workflow state and consumes the next application attempt only after the settlement owner schedules or begins it; +- failure commits a permanent or exhausted application outcome; +- redelivery means workflow/infrastructure state did not commit and must retry the same attempt without consuming business retry budget. + +Core NATS cannot fully satisfy the durable committed-outcome contract while it uses ephemeral pub/sub. Publish plus flush may prove only an ephemeral republish; D-004 remains a required reliability decision rather than weakening the contract for durable drivers. + +Asynq v0.26 checks retry exhaustion before consulting its `IsFailure` predicate. The behavior is an upstream ordering regression: revocation, skip-retry, and non-failure classification should precede exhaustion. New explicit-retry Redis tasks reserve one Asynq transport slot and carry their application retry budget in a task header; workers classify against the application budget, explicitly archive its terminal outcome, and reuse the reserve for uncommitted or lease-recovery redelivery without incrementing the application attempt. This preserves one Asynq settlement owner and queued-task decoding, but requires workers to roll out before producers and cannot repair an already-exhausted legacy task. An upstream fix remains preferred so the compatibility reserve can eventually be removed. + +### D-002: Workflow Durability Contract + +Status: Open + +Required before: M3 + +Decide whether distributed backends: + +- require an explicitly durable workflow store; +- automatically derive a compatible store where possible; or +- permit memory workflows only with a clear ephemeral-mode diagnostic. + +Preferred direction: allow memory mode for local development and explicitly ephemeral workflows, but require or loudly diagnose a durable store when durable workflows run across processes. + +### D-003: Public `bus` Compatibility + +Status: Accepted + +Required before: M2 public API consolidation + +Decision: make `*queue.Queue` the sole documented application facade. Retain `bus` temporarily as a deprecated forwarding compatibility package, then remove its independent implementation only after root equivalents and migration documentation exist. + +### D-004: NATS Product Contract + +Status: Open + +Required before: claiming queue durability for NATS + +Choose between: + +- replacing Core NATS pub/sub with JetStream durable work-queue semantics; or +- retaining the existing adapter under explicitly ephemeral pub/sub semantics and naming. + +Preferred direction: use JetStream durable consumers and queue-group delivery for the queue driver. + +### D-005: Queue Targeting + +Status: Open + +Required before: M1 default-queue corrections + +Define whether one runtime consumes: + +- only one configured queue; +- every queue targeted through it; or +- an explicit configured queue set with weights/concurrency. + +Preferred direction: distinguish producer target selection from worker subscription configuration, apply `DefaultQueue` centrally to empty targets, and reject dispatches that the configured consumer model cannot service only when that is an explicit contract. + +### D-006: Observer and Event Model + +Status: Accepted + +Required before: M2 public model consolidation + +Decision: use one root `queue.Observer` and one root `queue.Event` superset. `queue.WithObserver` observes both delivery and workflow facts. Existing root `WorkflowObserver`, `WorkflowObserverFunc`, `WorkflowEvent`, and `WorkflowEventKind` names become deprecated aliases or adapters. The compatibility `bus` package translates to its legacy event representation for a deprecation period instead of retaining a second event producer. + +### D-007: Workflow Continuations + +Status: Accepted + +Required before: M3 callback durability work + +Decision: observers are never workflow callbacks. Durable success, failure, progress, and completion continuations are named queue jobs persisted with workflow state. Existing function callbacks are explicitly ephemeral and remain only for compatibility and local convenience until a future compatibility boundary permits removal. + +### D-008: Direct Job Execution + +Status: Accepted + +Required before: M2 runtime consolidation + +Decision: direct `Queue.Dispatch` does not create a workflow envelope. Chains and batches attach private workflow correlation to the same canonical job/delivery model, and the worker runtime reports committed outcomes back to the workflow engine. + +### D-009: Observer Compatibility Boundary + +Status: Accepted + +Required before: publishing the unified observer release + +Decision: the observer collapse is an intentional source/API and runtime-behavior compatibility boundary in the next pre-v1 feature release. `WithObserver` accepts the canonical root `Observer`, legacy root workflow names alias that root model, and one observer receives all layers. Preserving the exact old type identity would require retaining the second public event contract or accepting an untyped option, both of which conflict with the requested collapse and reliable compile-time contracts. + +Migration requirements: + +- replace unkeyed `queue.Event` and `bus.Event` literals with keyed literals because the event envelopes gained correlation fields; +- adapt custom `bus.Observer` implementations with `queue.ObserverFunc`, or keep raw-runtime legacy bus consumers on `bus.WithObserver`; an existing `*queue.Queue` must receive its observer at root construction; +- filter `Event.Layer` when an existing workflow observer should retain workflow-only volume; +- make observer-owned mutable state concurrency-safe because dispatchers and workers may call the same observer concurrently. + +This boundary does not itself change configuration files, persisted workflow data, wire envelopes, operational rollout, or the minimum Go version. + +### D-010: Public Ownership and Dependency Direction + +Status: Accepted + +Required before: M2 public API consolidation + +Decision: root `queue` owns the canonical application model. The orchestration implementation moves behind `internal/workflow`, root composes that internal engine directly, and public `bus` becomes a forwarding compatibility package. Do not create another public workflow-model package. Preserve the legacy raw-`busruntime.Runtime` construction path temporarily while normal `bus.New(*queue.Queue)` calls route to the root facade. + +The migration is staged to avoid an import cycle and an all-at-once source break: + +1. extract the existing bus engine behind `internal/workflow` without changing event names, JSON envelopes, SQL schemas, stores, retry behavior, or public type identity; +2. switch root production code from public `bus` imports to the internal engine and add an import-direction guard; +3. define canonical root `Message`, dispatch/state, middleware, store, and builder contracts one model at a time; +4. turn compatible `bus` declarations into deprecated aliases/adapters, retaining legacy composite-literal fields until a separately approved compatibility boundary; +5. route option-free `bus.New(*queue.Queue)` through root and reject construction-only options explicitly instead of constructing another independently configured engine. + +The extraction itself is source/API, configuration, persisted-data, runtime-behavior, operational, wire, and minimum-Go-version neutral. The facade conversion deliberately changes runtime behavior for every `bus.New(*queue.Queue)`: compatibility views now share the root handler registry, store, observer, middleware, and lifecycle instead of constructing independent state over the same physical runtime. Code that needs isolation must use distinct runtimes. It also changes configuration and runtime behavior for option-bearing `bus.New(*queue.Queue, ...)` and `bus.NewWithStore(*queue.Queue, ...)`: those calls now return `bus.ErrQueueOptionsUnsupported` because options cannot apply only to a shared view. Callers migrate those options to root queue construction and then use option-free `bus.New(existingQueue)`; the raw-`busruntime.Runtime` route retains its legacy options. Later alias conversions require focused source-compatibility fixtures before landing. + +## Milestone M0: Restore a Trustworthy Baseline + +Objective: every module builds and its non-network validation runs independently, while CI exposes rather than hides module drift. + +- [x] **M0-01 — Repair nested driver test compilation.** Update NATS, SQS, and RabbitMQ tests to the current observer contract and verify the existing all-module compile guard covers every optional driver module. +- [x] **M0-02 — Make examples independently reproducible.** Repair the examples module sums and validate it with `GOWORK=off`. +- [x] **M0-03 — Eliminate mutex-copy hazards.** Replace runtime value cloning with shared lifecycle state and verify context-bound handles retain synchronized worker state. +- [x] **M0-04 — Expand CI across module boundaries.** Run unit tests for every module, root race tests, vet, README snippet checks, generated example checks, and independent `GOWORK=off` passes where applicable. +- [x] **M0-05 — Fix known stale documentation sources.** Correct the batch callback example, Go-version badge, and sync lifecycle examples in authoritative sources, then regenerate documentation. +- [x] **M0-06 — Establish public-path contract fixtures.** Ensure shared fixtures construct and exercise public `*queue.Queue` values rather than private runtime adapters. +- [x] **M0-07 — Add a module/version inventory guard.** Check `go.mod` files, workspace membership, sibling replacements, module tags, and release-script coverage. + +Exit criteria: + +- Every module's non-network unit suite compiles and passes independently. +- Root unit, race, and vet passes are green. +- CI cannot pass while an optional driver test package fails to compile. +- Generated documentation is reproducible with no second-run diff. + +## Milestone M1: Correct Existing Public Semantics + +Objective: repair behavior already promised by the public API without first requiring a broad redesign. + +### Retry and terminal outcomes + +- [ ] **M1-01 — Add retry-state regressions.** Cover chain and batch transient success, terminal exhaustion, fatal errors, attempt numbers, callbacks, and event ordering through public queues. Public transient, exhaustion, permanent, attempt, callback, ordering, downstream-node, allowed-batch-failure ordering, and workflow-store fault cases are in place. Real SQLite proves a surviving same-attempt row can use an exact transition receipt to recover terminal-chain, completed-predecessor, batch-member, and aggregate-completion facts after forced finalization failure without executing application code twice. Active predecessor recovery now re-dispatches the immediate successor after definite enqueue rejection without replaying the predecessor; the same liveness-only behavior covers a missing receipt, a decorated store without receipt capability, and supported receipts with different or legacy generation provenance. Those weaker paths emit no predecessor facts or callbacks, and progressed or terminal state is a no-op. The remaining already-enqueued-but-not-progressed ambiguity makes continuation at-least-once, not exactly-once. Focused built-in contracts prove a receipt-backed terminal chain failure returns its first persisted permanent cause across exact, different, or legacy recovered-generation provenance without repeating the handler, callbacks, or logical failure facts; invalid receipts fail closed, while receipt-absent legacy failure rows retain weaker replay. A real SQLite archive-failure fixture extends that proof across multiple recovery finalization faults, preserves the cause and receipt lineage, and reaches `dead` with one application/workflow occurrence. The two-member terminal-owner case proves the completing member's receipt recovers `BatchCompleted` without crediting the earlier member. Repeated SQLite success and failed-batch finalization faults prove the same fenced best-effort lineage repair; failed batch recovery uses a generic permanent cause and reaches SQL's `dead` archive without fabricating the unpersisted application error. A separate race proves application retry clears the earlier generation link, while the later attempt can own a new receipt and recover a subsequent finalization failure of that same attempt. Real MySQL and PostgreSQL successful terminal-chain scenarios prove the versioned receipt-to-generation contract, and twelve-worker fail-fast races on both dialects prove one serialized aggregate terminal owner. No-redelivery publication and remaining settlement-owner `WithStore` gates remain. +- [x] **M1-02 — Stop premature workflow failure.** Chain and batch state, callbacks, and terminal logical events now wait for a permanent or exhausted outcome; retryable attempts remain worker-layer failures. +- [x] **M1-03 — Make `FailOnError` operational.** `queue.Permanent`, `FailOnError`, and the shared classifier now stop application retries across local, Redis, SQL, NATS, SQS, and RabbitMQ paths. +- [ ] **M1-04 — Emit only committed outcome events.** Workflow mutation failures are classified as uncommitted before terminal facts publish, retry facts appear only when a numbered retry delivery begins, and generic/archive predictions were removed. SQL, SQS, and RabbitMQ defer positive process/workflow facts until durable row finalization, deletion, or acknowledgement; failed settlement emits a correlated worker fact. SQL claims carry opaque generation provenance, while direct built-in workflow stores separately record exact transition ownership plus receipt and observer-event schema versions. The observer schema is independent from the workflow-envelope protocol, and an unsupported receipt or event version fails recovery closed with an uncommitted error before acknowledgement, application execution, state-commit signaling, or fact publication. A supported identity-matching receipt suppresses duplicate application execution; exact recovered-generation ownership additionally gates republishing successful chain-node, batch-member, and aggregate facts with deterministic IDs after the new settlement. A failed chain receipt returns the authoritative persisted permanent cause across generation variants without repeating occurrence-based failure facts or callbacks. Failed batch recovery returns a generic permanent cause because its original error is not persisted; both failure paths archive without emitting replacement member facts. Real SQLite, MySQL, and PostgreSQL finalization-failure tests cover the supported successful terminal-chain path; focused memory/SQLite contracts plus a real repeated SQLite archive-failure fixture cover atomic failed-chain receipts; repeated SQLite finalization faults cover best-effort lineage repair; and SQLite plus real MySQL/PostgreSQL concurrency gates prove aggregate completion/terminal effects belong to one serialized receipt owner rather than aggregate state or an earlier member. When a generation claims a receipt-backed transition but later workflow infrastructure still needs same-attempt redelivery, the delivery-settlement application-state signal makes SQL retain that current generation instead of inherited provenance; focused signal, token-selection, and chain post-transition tests cover the handoff. Queue provenance, aggregate state, and response-local `claimedNow` are not durable proof; application retry clears the earlier generation link. Callback redelivery no longer turns an at-most-once marker into false success, premature callback envelopes cannot consume the marker, callback panics become failures, and new Redis tasks preserve final-attempt uncommitted redelivery through one header-marked transport reserve. Custom/decorated/raw stores have weaker private-capability guarantees. A driver-owned settlement outbox is still required when finalization applied but its deferred observer calls were lost; durable callback/continuation intents, physical commit/readback ambiguity, cross-driver provenance, Redis post-`Done` success, legacy Redis task rollout, and NATS's explicit nonconformance also remain. + +### Identity and uniqueness + +- [x] **M1-05 — Define logical job identity.** The versioned identity length-frames effective queue, logical type, and canonical payload while excluding volatile correlation IDs and delivery options. Golden vectors pin direct/workflow parity and normalize absent, zero-byte, and exact JSON `null` payloads before direct execution replaces legacy envelopes. +- [x] **M1-06 — Repair public `UniqueFor`.** Public workflows use canonical logical identity across every backend; concurrent public SQL/Redis clients prove one backend-wide winner, restart persistence, and documented instance-versus-backend scope. SQL rolling upgrades atomically claim both the exact historical physical identity and canonical version-one identity with the queue row, so an outstanding matching historical or current lock rejects the dispatch and either collision rolls back the companion claim. Pre-version high-level envelopes carried volatile correlation, so their independently generated physical keys still require a longest-live-TTL producer cutover or an explicitly accepted transient duplicate window. +- [ ] **M1-07 — Make uniqueness acquisition atomic with acceptance.** SQL claims and queue rows now share one transaction; local and broker paths compensate known pre-acceptance failures with token-owned claims. Redis compensates only definite physical duplicates and intentionally retains other ambiguous enqueue outcomes, but its separate claim/enqueue operations leave an unavoidable crash window until a stronger atomic protocol exists. + +### Configuration and lifecycle behavior + +- [ ] **M1-08 — Apply `DefaultQueue` centrally.** Preserve explicit queue names and ensure empty names follow one documented rule across every driver. +- [ ] **M1-09 — Make worker targeting explicit.** Implement D-005 and test that every accepted target is consumed by the intended runtime configuration. +- [ ] **M1-10 — Make `WithWorkers` effective.** Workerpool now applies the configured count to execution concurrency and derives its default buffer from that count. Verify the same end-to-end behavior for SQL, Redis, NATS, SQS, and RabbitMQ rather than wrapper state; Core NATS plain subscriptions currently make higher counts duplicate broadcast consumers rather than queue workers. +- [ ] **M1-11 — Clarify sync startup semantics.** Either register bus handlers immediately for synchronous dispatch or consistently require startup and correct every example and contract. +- [ ] **M1-12 — Validate registrations and options.** Nil handler registration is now a consistent no-op across root, workflow-engine, and deprecated bus boundaries, including after a valid registration. Reject nil options deterministically; preserve explicit zero versus unset retry, timeout, and backoff values. +- [ ] **M1-13 — Normalize payload contracts.** Give `Payload` and `PayloadJSON` distinct, documented behavior and consistent binding errors. + +### Readiness, capabilities, and shutdown + +- [ ] **M1-14 — Restore backend readiness.** Readiness now survives the internal driver bridge, managed SQL checks the complete runtime table shape without DDL, and NATS supplies the deadline required for a real server roundtrip while preserving shorter caller deadlines. A public NATS negative uses a reachable non-NATS endpoint, and the live integration covers `Ready(context.Background())`. Remaining work includes negative unreachable tests for every external backend and composing worker-factory configuration into readiness, including invalid NATS default-subject detection and recovery after permanent producer reconnect exhaustion. +- [ ] **M1-15 — Replace wrapper-inflated capability checks.** Report actual capabilities independently of observers and adapters. +- [ ] **M1-16 — Make shutdown retryable and context-aware.** Root native/external lifecycles serialize startup with shutdown, latch drain intent before waiting on startup, retain partially started workers for cleanup or retry, keep one replaceable handler slot per backend registration, lease dispatch/readiness/control/admin operations before resources can close, retain state after failed cleanup, reject restart while draining, and close successfully exactly once. Native cleanup now drains workers without closing resources, waits the stable late-operation generation after every handler permit expires, and only then closes producer/storage resources. Redis owned resources close at most once, report joined close diagnostics to the caller that performs cleanup, and let a later root shutdown converge to terminal success instead of remaining permanently draining. Scoped continuation permits expire when their handler returns, while Sync and Workerpool reserve accepted delayed descendants and drain them within the caller's shutdown deadline. Replacement decorators retain runtime-owned continuation, settlement, provenance, attempt, and metadata state while keeping their own cancellation, deadlines, and user values. Context-unaware broker dialing and RabbitMQ resource closure, plus real broker deadline evidence, remain. +- [x] **M1-17 — Close producer-owned resources without worker startup.** Native and external shutdown now reach producer cleanup without worker startup, Redis closes every owned producer client exactly once, SQL closes only internally opened handles, and a real SQLite test proves caller-owned handles remain usable. +- [ ] **M1-18 — Drain before broker settlement resources close.** Root operation leases prevent producer closure during admitted public work; SQS and RabbitMQ workers wait for active delivery loops before closing settlement resources; and NATS startup/drain coordination waits for admitted callbacks. Real in-flight delete, acknowledgement, replacement-publication, and connection-close shutdown scenarios remain. Core NATS can still accept a replacement after its own ephemeral subscription has drained, so D-004 remains a correctness boundary rather than a shutdown guarantee. + +Exit criteria: + +- The retry, uniqueness, default queue, worker count, readiness, capability, and shutdown contracts pass for every applicable driver. +- No option changes meaning during queue-to-workflow-to-driver conversion. +- Enabling an observer cannot change reported capabilities or job outcomes. + +## Milestone M2: Establish Clean Internal Boundaries + +Objective: introduce a stable internal architecture while preserving the root application API. + +- [ ] **M2-01 — Specify the driver SPI.** Define immutable enqueue input, option presence, logical identity, delivery attempt metadata, settlement outcomes, lifecycle, and capabilities. + - [x] Add the additive attempt-classification foundation shared by orchestration and delivery: zero-based attempt metadata, success/retry/failure/redelivery decisions, and distinct permanent versus uncommitted error markers. + - [x] Propagate attempt metadata through every root/driver handler path before changing workflow transitions; Redis now preserves queue/attempt/retry metadata even without observers, and SQL reconstruction has a direct contract test. + - [x] Add versioned, driver-neutral direct-delivery metadata for correlation outside application payloads; local jobs, Redis headers, broker messages, and nullable SQL storage now round-trip the same record and reject unknown versions without rejecting the job. + - [ ] Move committed retry/archive emission to each settlement owner and define enqueue acceptance receipts. +- [ ] **M2-02 — Create a domain-neutral core/SPI package.** Drivers depend on this package rather than importing root helpers through re-export and global hook layers. +- [ ] **M2-03 — Introduce adapters alongside existing drivers.** Migrate one local and one durable driver first, keeping compatibility tests on both paths. +- [ ] **M2-04 — Remove the mutable runtime hook bridge.** Retire `any`-based global initialization only after every driver uses the new SPI. +- [x] **M2-05 — Consolidate the Job model.** Root `queue.Job` is the sole canonical public application specification, `queue.Message` is the delivered handler model, and `queue.StoredJob` is the persisted workflow model. Direct dispatch freezes exact root payload bytes instead of passing through the private fluent workflow DTO. `bus.Job` remains only as the documented source-compatible boundary DTO whose deferred JSON conversion occurs once at facade dispatch. +- [ ] **M2-06 — Version the envelope.** Define schema evolution, unknown-version behavior, mixed producer/worker deployments, and rollback. +- [x] **M2-07 — Resolve the public `bus` direction.** Implemented D-003 and D-010 in bounded slices: extracted `internal/workflow`, removed root production imports of public `bus`, established physical root models, then made `bus` a deprecated forwarding facade with source- and wire-compatibility fixtures. One internal engine now owns orchestration; root composes it directly through physical root messages, middleware, workflow records, and stores; and `bus.New(existingQueue)` wraps that exact engine without registering another. Legacy `bus.Job`, `Event`, `Observer`, `Bus`, `Option`, and self-returning builder interfaces remain physical boundary contracts, while compatible model names forward to root. Literal protocol, transport-boundary, legacy SQLite, source, construction, package-identity, import-direction, adapter, and deferred-encoding fixtures pin the migration. The full module, race, generated-documentation, local/SQLite, Redis, and NATS validation matrix passes. +- [ ] **M2-08 — Separate producer and worker lifecycle.** Model start, running, draining, stopped, and failed states without `sync.Once` poisoning. +- [x] **M2-09 — Collapse root observers.** One root observer receives delivery and workflow events through a shared sink without duplicate execution events. The legacy public `bus` observer is now translated only at the deprecated raw-runtime compatibility boundary. +- [x] **M2-10 — Stop enveloping direct jobs as workflows.** Root dispatch now sends the application type and exact payload with versioned out-of-payload correlation. One engine executor still owns middleware, handler lookup, logical events, attempt classification, and settlement deferral. Every backend round-trips supported metadata, old version-one envelopes remain readable, reserved protocol-name applications retain the legacy route, raw-runtime `bus` bytes remain frozen, and `WithLegacyDirectEnvelope` supports a safe workers-first rollout. Direct/envelope uniqueness parity remains on the existing `v1` key. The additive SQL column, asymmetric mixed-worker boundary, exact payload correction, and rollback procedure are documented in `docs/direct-delivery-migration.md`. + +Exit criteria: + +- Optional drivers no longer depend on mutable global runtime hooks. +- The root public API is backed by one explicit composition path. +- Mixed supported module/envelope versions have documented and tested behavior. + +## Milestone M3: Make Workflow Semantics Durable and Atomic + +Objective: chains, batches, callbacks, and stores behave correctly across retries, concurrency, restarts, and multiple processes. + +- [ ] **M3-01 — Implement the workflow durability contract.** Apply D-002 at construction/readiness time. +- [ ] **M3-02 — Replace durable closure callbacks.** Persist named continuation jobs or registered callback identifiers; keep closures only as explicit ephemeral compatibility behavior. +- [ ] **M3-03 — Add store lifecycle ownership.** Close internally opened SQL resources and never close caller-owned resources. +- [x] **M3-04 — Make chain advancement atomic.** Memory and SQL validate immutable node order, then compare-and-swap both success and failure against the current `NextIndex`, so the first committed outcome for a node remains authoritative. Late contradictory deliveries cannot change state or publish opposite logical facts, legacy completion-then-failure rows retain completion precedence, and built-in `FailChain` preserves the first terminal cause instead of allowing a later call to rewrite authoritative recovery state. Unknown, future, duplicate-ID, and caller-aliasing branches are covered. Thirty-two-way contracts pass repeatedly on SQLite, MySQL, and PostgreSQL. Recovery re-dispatches an immediate still-pending successor without replaying the predecessor when exact receipt ownership, a missing receipt, a store without receipt capability, or different/legacy receipt provenance accompanies durable success state. Only exact ownership reconstructs predecessor facts; progressed and terminal state is a no-op. Supported receipt cancellation/completion shape is validated before liveness dispatch, corruption fails uncommitted without effects, and a rejected successor remains uncommitted for another recovery. The post-commit ambiguity where enqueue may have succeeded but workflow state has not progressed remains an M3-06 concern. +- [x] **M3-05 — Make batch aggregation atomic.** SQL conditionally claims each member and applies arithmetic aggregate updates in the same transaction, preventing both duplicate decrements and distinct-member counter overwrites. Built-in memory settlement holds one mutex, while MySQL/PostgreSQL lock the parent row after the member claim so only one false-to-true transition owns terminal aggregate effects. The additive `WorkflowOutcomeStore` reports first-writer outcome-category ownership without breaking established custom `WorkflowStore` implementations, and the runtime suppresses contradictory job/batch facts, progress, and callbacks. Concurrent mixed success/failure contracts pass on memory, SQLite, MySQL, and PostgreSQL; twelve-worker fail-fast races on each server dialect prove one aggregate receipt and one terminal failed/cancelled fact pair. Aggregate recovery now fails uncommitted when a SQL aggregate row omits completion, cancellation does not own a failed outcome, the row's incarnation is stale, or a row naming the requested logical member disagrees with that member receipt's complete owner/outcome; runtime validation also requires the aggregate flags to agree with live terminal state. `TestSQLStoreBatchAggregateOwnershipMismatchFailsClosed`, `TestSQLStoreBatchAggregateIncarnationMismatchFailsClosed`, and `TestBatchRecoveryRejectsInvalidAggregateReceiptShape` cover those branches. The established batch model does not persist per-member cause text, so restart recovery uses a generic permanent cause to archive an already-committed failed member rather than fabricating the original error. +- [ ] **M3-06 — Recover partial dispatch.** Ensure a partially enqueued batch or chain reaches a recoverable terminal or resumable state rather than permanent pending state. Validated chain success now conservatively re-dispatches the live immediate successor after definite rejection even when the receipt is missing, hidden by a custom/decorated store, or owned by different/legacy generation provenance; those weaker paths restore only liveness and do not reconstruct predecessor facts or callbacks. Recovery still cannot distinguish a missing successor from one already enqueued and not yet progressed, so persisted successor intent is required for exact recovery. Persist per-member batch failure detail if recovered callbacks and facts must reproduce the first physical cause exactly. The aggregate receipt proves which member transaction made a batch terminal, including the SQLite two-member recovery and real MySQL/PostgreSQL terminal-owner races, but it does not retain fan-out, callback, progress, or observer-publication intent. Keep those durable intents separate from the completed state-transition provenance gate. +- [ ] **M3-07 — Make callback invocation truthful.** State validation now precedes idempotency claims, missing process-local closures fail visibly, panics become callback failures, duplicate envelopes emit no orphan start, and reverse-order serialized sibling callbacks each complete once. The marker still precedes application success and callback enqueue errors have no durable recovery path, so named persisted continuations remain required. +- [ ] **M3-08 — Unify memory and SQL store contracts.** Align missing-ID, terminal transition, clock, copy/ownership, and validation behavior. +- [ ] **M3-09 — Make schema migration configurable.** `DisableAutoMigrate` is the additive queue-schema opt-out while the established queue default remains enabled. Managed queue readiness and startup perform no DDL and validate both operational objects as base-table relations, including PostgreSQL partitioned tables, plus every column used by the current runtime; an empty, view-backed, or incomplete schema fails closed, and the same runtime can retry after external provisioning. A real SQLite empty-schema scenario proves the no-DDL and retry contract, while canonical preprovisioned SQLite, MySQL, and PostgreSQL scenarios prove readiness, uniqueness, dispatch, and consumption. This presence gate does not certify write permissions, exact SQL types, constraints, or performance indexes. Automatic startup remains retryable after a real SQLite DDL lock failure. Workflow schema has a separate explicit constructor boundary: `NewSQLStore` preserves legacy migration-on-first-use behavior regardless of the false `SQLStoreConfig.AutoMigrate` zero value, while `NewSQLStoreWithManagedSchema` performs no DDL. Workflow auto-schema creates versioned `bus_workflow_transition_receipts` beside the established state tables; non-null `receipt_version` and `event_schema_version` fence durable interpretation, and unknown values fail recovery closed rather than becoming indistinguishable from absence. The event schema versions the shared observer envelope independently from the workflow-envelope protocol. Real SQLite, MySQL, and PostgreSQL recovery scenarios exercise each dialect's fresh auto-created supported-version receipt path. Managed-schema callers must precreate every column, and rollback should quiesce new workers and retain the receipt table because old binaries ignore it. A wholly fresh MySQL schema uses 255-byte workflow/member and receipt identities plus 512-byte callback keys. When only the receipt table is missing beside established state, ordinary startup derives its shared workflow width from the larger effective chain-or-batch capacity and its member width from the larger chain-node-or-batch-job capacity; the real legacy-width upgrade fixture proves the 512/512 path with identities above fresh defaults. Existing receipt tables are never altered and continue to intersect connected-schema limits. An incompatible existing table therefore needs a quiescent managed migration and a fresh store. If derived widths exceed the server's composite-key budget, startup fails with both widths and schema-first guidance rather than narrowing live tables; operators must provision a compatible indexed schema or explicitly migrate supported identity limits and existing data. Every caller-managed non-`VARBINARY` identity schema still needs the migration recorded in DL-019. Managed-schema rollout/rollback, real cross-dialect pruning and physical commit/readback ambiguity, concurrent startup, permission gates, and queue uniqueness-index performance evidence remain open. +- [ ] **M3-10 — Reclassify the Temporal adapter.** Either implement a real external workflow-engine contract or clearly separate the current façade from queue-backed workflow guarantees. + +Exit criteria: + +- Durable workflows survive process restart and producer/worker separation. +- Duplicate deliveries cannot duplicate state transitions or callbacks. +- Store behavior is consistent and concurrency-tested on SQLite, MySQL, and PostgreSQL. + +## Milestone M4: Converge Driver Guarantees + +Objective: every advertised capability has a conformance test and every semantic difference is explicit. + +- [ ] **M4-01 — Resolve NATS semantics.** Implement D-004 and test two workers, crash recovery, delayed work, broker restart, and poison handling. +- [ ] **M4-02 — Define poison/dead-letter behavior.** Prevent silent deletion/acknowledgement of malformed, unhandled, and terminally failed jobs in SQS and RabbitMQ. +- [ ] **M4-03 — Harden SQS delivery.** Configure visibility/redrive behavior, extend visibility for long handlers, validate credential pairs, and add deterministic receive-failure coverage. Missing receipts and delete failures now surface as settlement failures without false success. +- [ ] **M4-04 — Harden RabbitMQ delivery.** Preserve positive publisher confirms for dispatch/retry while adding worker reconnect, context-aware dialing, and real safe-settlement drain scenarios. +- [ ] **M4-05 — Harden Redis resources and admin.** Owned producer clients and state stores now close exactly once; align totals/windows, remove unreachable queue-resolution branches, and define bounded clear behavior. +- [ ] **M4-06 — Harden SQL execution and admin.** Finalization retries are bounded, require one affected row, and surface settlement failure; constrain active-job admin races, preserve timeout precision, and finish claim/recovery indexing evidence. +- [ ] **M4-07 — Make uniqueness claims precise.** Label each driver as process-local or distributed and test exactly that scope. +- [ ] **M4-08 — Separate guarantees from evidence.** Maintain a portable contract plus a driver evidence matrix whose cells link to executable scenarios. + +Exit criteria: + +- No driver is described as at-least-once unless acceptance, persistence, settlement, retry, and crash boundaries support that claim. +- Unsupported capabilities report false before invocation rather than failing after an optimistic support check. +- Multi-process and broker-fault scenarios cover every durable backend. + +## Milestone M5: Unify Observability, Administration, and Test Doubles + +Objective: operational surfaces describe real state consistently and testing APIs model production behavior. + +- [x] **M5-01 — Define a shared root event envelope.** The normal facade now includes stable correlation, layer/source, queue, logical job, delivery attempt, event identity, and timestamps. Settlement-owner completion remains M1-04, and the legacy `bus` envelope survives only as a translated compatibility shape. +- [x] **M5-02 — Preserve distinct event vocabularies without duplicate observer models.** The root observer retains effective queue, logical job key/type, delivery attempt, and workflow correlation across queue, worker, aggregate, and callback facts. The legacy `bus.Event` shape is now translated only at the deprecated compatibility boundary; it no longer owns an event producer or orchestration runtime. +- [x] **M5-03 — Correct event ordering.** Local acceptance callbacks and workerpool delivery gates ensure synchronous/in-process processing cannot begin before enqueue acceptance appears to observers; distributed arrival order remains correlation-based rather than globally ordered. +- [ ] **M5-04 — Make stats semantically comparable.** Define pending, scheduled, retry, active, processed, failed, and throughput windows for each capability level. +- [ ] **M5-05 — Make history instance-scoped and truthful.** Do not present process-wide sampled memory as durable backend history. +- [ ] **M5-06 — Consolidate admin APIs.** Remove `any`-based duplicate paths over time and align not-found, unsupported, and active-operation behavior. +- [x] **M5-07 — Consolidate fakes.** `queue.NewFake` now owns one concurrency-safe direct/workflow recorder; deprecated `bus.Fake` and `queuefake.Fake` are typed compatibility views over that state. Direct dispatch shares production conversion and validation, while chain/batch builders run through the production workflow engine, record only accepted `Dispatch` calls, retain policy, and expose isolated canonical records and lookup state. +- [ ] **M5-08 — Add observer failure policy.** Define panic/error handling instead of silently swallowing observer failures without diagnostics. + +Exit criteria: + +- Events correspond to committed state transitions. +- Stats and admin operations have documented cross-driver meanings. +- Tests using the public fake exercise the same job conversion and validation rules as production queues. + +## Milestone M6: Documentation, Compatibility, and Release Readiness + +Objective: documentation and release mechanics accurately describe the implemented system. + +- [ ] **M6-01 — Replace stale architecture snapshots.** Reconcile `design.md`, `bus-design.md`, and the one-path rationale; label historical proposals clearly. +- [ ] **M6-02 — Publish a delivery contract.** Define acceptance, durability, retries, duplicates, ordering, poison handling, and failure boundaries. +- [ ] **M6-03 — Publish a workflow/store contract.** Define durability modes, callback recovery, retry ownership, concurrency, pruning, and retention. +- [ ] **M6-04 — Expand compatibility policy.** Cover source API, configuration, persisted data, wire envelopes, SQL schemas, mixed module versions, operations, and minimum Go version. +- [ ] **M6-05 — Make all examples executable.** Expected output must immediately follow producing calls and generated examples must run successfully where behavior is being demonstrated. +- [ ] **M6-06 — Complete GoDoc quality pass.** Add compliant comments for exported and private entities while explaining constraints and intent rather than syntax. +- [ ] **M6-07 — Validate the largest generated composition.** Use repository-pinned versions and render outside the repository. +- [ ] **M6-08 — Exercise release scripts and module tags.** Verify every independently published module and a downstream `GOWORK=off` integration against published versions. + +Exit criteria: + +- Documentation contains one normal application model. +- Every public guarantee links to tests or a clearly scoped limitation. +- Every module can be released and consumed independently through its intended tag convention. + +## Required Validation Matrix + +Run the applicable subset continuously and the full matrix before milestone completion or release. + +### Root module + +```bash +GOCACHE=/tmp/gocache GOMODCACHE=/tmp/gomodcache go test ./... +GOCACHE=/tmp/gocache GOMODCACHE=/tmp/gomodcache go test -race ./... +GOCACHE=/tmp/gocache GOMODCACHE=/tmp/gomodcache go vet ./... +``` + +### All modules + +- Root +- `docs` +- `examples` +- `integration` +- `driver/sqlqueuecore` +- `driver/mysqlqueue` +- `driver/postgresqueue` +- `driver/sqlitequeue` +- `driver/redisqueue` +- `driver/natsqueue` +- `driver/sqsqueue` +- `driver/rabbitmqqueue` + +For each relevant module, run unit and vet passes independently with `GOWORK=off`. Use the repository's all-module script when it covers the required mode, and keep a direct module inventory guard so newly nested modules cannot be omitted. + +### Shared semantic contracts + +Each applicable backend must cover: + +- public dispatch and processing; +- empty/default and explicit queue targeting; +- retry attempt metadata, backoff, fatal errors, and exhaustion; +- uniqueness at its declared scope, including a failed first dispatch; +- delayed work and restart behavior; +- malformed, unhandled, poison, and terminal failures; +- negative readiness; +- actual concurrency; +- cancellation, drain deadlines, and resource closure; +- two worker processes where distributed behavior is claimed; +- workflow recovery, duplicate delivery, and callbacks where workflows are supported. + +Network-backed integration validation should use the required elevated execution path. Test renders must always be created under `/tmp`, never inside the repository. + +## Definition of Done for Every Change + +- The behavior and compatibility category are identified. +- New validation branches and failure modes have direct tests. +- Every affected module passes unit tests and vet independently. +- Race testing is run when concurrency or lifecycle is involved. +- Generated mirrors are regenerated from their authoritative source and a second generation is clean. +- Documentation describes the implemented behavior without overstating guarantees. +- `git status` and the staged diff contain only intended files. +- This plan's checkboxes, decision log, baseline, and progress log are updated in the same change when applicable. + +## Decision Log + +Record accepted decisions here using the next stable ID. + +| ID | Date | Decision | Compatibility and migration notes | +| --- | --- | --- | --- | +| DL-001 | 2026-07-18 | Use this file as the living execution ledger and preserve compatibility by default. | Architectural cleanup alone does not authorize public API, configuration, persisted-data, runtime, or Go-version breaks. | +| DL-002 | 2026-07-18 | Make `queue` the only canonical application model and reduce `bus` to a deprecated forwarding facade. | Root APIs remain the migration target. Existing `bus` users receive adapters and deprecation guidance before independent implementations are removed. | +| DL-003 | 2026-07-18 | Use one root observer and event model across delivery and workflows. | Legacy workflow observer names become aliases/adapters; legacy `bus` event consumers receive translated events during migration. | +| DL-004 | 2026-07-18 | Keep observation separate from reliable workflow continuation. | Durable continuations become persisted jobs; closure callbacks are explicitly ephemeral compatibility behavior. | +| DL-005 | 2026-07-18 | Dispatch direct jobs without a workflow envelope. | Envelope/schema changes require mixed-version tests; logical job identity must remain stable for uniqueness. | +| DL-006 | 2026-07-18 | Make worker settlement the authoritative retry boundary. | Workflow state and events follow only committed retry, permanent failure, exhaustion, or success outcomes. | +| DL-007 | 2026-07-18 | Take an explicit pre-v1 observer compatibility boundary instead of retaining two typed models or using `any`. | Migration covers keyed `queue.Event` and `bus.Event` literals, custom bus observers, layer filtering, and concurrent observer calls; no persisted-data or wire change is implied. | +| DL-008 | 2026-07-18 | Invert orchestration dependencies through `internal/workflow`; root owns public models and `bus` becomes a compatibility facade. | Extract behavior first, preserve the legacy raw-runtime route, and migrate type ownership incrementally with compile and wire fixtures. | +| DL-009 | 2026-07-18 | Preserve an explicit zero workflow retry budget and require positive settlement before SQL/SQS/RabbitMQ success facts. | This corrects runtime behavior: Redis no longer substitutes Asynq's 25-retry default, NATS dispatch gains flush latency, RabbitMQ dispatch gains confirmation latency, SQS rejects an SDK success without a service `MessageId`, and observer success timing moves later. Deploy Redis workers before producers, set `.Retry(25)` explicitly if the old fallback was intentional, update SQS test doubles to return a message ID, and treat canonical uniqueness cutover as the documented operational migration. | +| DL-010 | 2026-07-18 | Preserve database migration-on-start as the default and add `DisableAutoMigrate` as the explicit externally managed-schema opt-out. | Existing keyed configurations retain their runtime behavior; the prior `AutoMigrate: false` value was normalized to enabled and therefore could not express an opt-out. The additive public field can break unkeyed struct literals at compile time, so those callers must migrate to keyed literals before upgrading. No persisted-data format or minimum-Go-version change is implied. | +| DL-011 | 2026-07-18 | Treat accepted local work, including delayed workflow descendants, as a shutdown drain obligation bounded by the supplied context. | Shutdown may now wait for accepted delayed work and return the context error when the deadline expires; cleanup remains retryable and a later call converges. This is a runtime-lifecycle correction with no API, configuration, persisted-data, or minimum-Go-version change. | +| DL-012 | 2026-07-18 | Make successful queue shutdown terminal while keeping incomplete cleanup retryable. | `Dispatch` and `StartWorkers` now reject use after successful shutdown instead of reporting false success over closed resources; construct a new queue instance to restart. Repeated shutdown is idempotent. This is a runtime-behavior correction with no source/API, configuration, persisted-data, operational-migration, or minimum-Go-version change. | +| DL-013 | 2026-07-18 | Fence every SQL processing generation with a random token and require that exact claim to finalize the row. | This is a persisted-schema and operational rollout change, not a source/API, configuration, wire-envelope, or minimum-Go-version break. The nullable `processing_token` column preserves existing rows and old producer-only binaries. Externally managed schemas must add it before new workers start. Quiesce every old SQL worker before migration and then start the new worker fleet; mixed old/new workers are unsafe because old workers settle by row ID and cannot honor the generation fence. Rollback likewise requires quiescing new workers before running an old worker binary. | +| DL-014 | 2026-07-18 | Keep the canonical public workflow models physically owned by root `queue`, with explicit private adapters around one `internal/workflow.Engine`; never expose an `internal` package as the real owner of a public alias. | Source-compatible `bus` model, middleware, and store names now resolve to root types. Legacy `bus.Job`, `Event`, `Observer`, `Bus`, `Option`, and self-returning builder interfaces remain physical compatibility contracts. Existing code that keys behavior on `%T`, `reflect.Type.PkgPath`, gob/interface registration names, generated type registries, DI keys, or custom type-sensitive persistence must migrate applicable names from `github.com/goforj/queue/bus` to `github.com/goforj/queue`. JSON/wire envelopes, SQL schemas/data, runtime outcomes outside DL-015, operational rollout, and the minimum Go version do not change. | +| DL-015 | 2026-07-18 | Make every `bus.New(*queue.Queue)` a view of the root engine, reject view-specific construction options, and preserve the option-bearing raw-runtime route. | This is a configuration and runtime-behavior incompatibility, not a source/API break. Root and bus views now share registrations, store, observer, middleware, and lifecycle; code requiring isolation must use distinct runtimes. Move observer, store, clock, and middleware options to root queue construction, then call option-free `bus.New(existingQueue)`. Preserving the old behavior would preserve the second engine this milestone removes. | +| DL-016 | 2026-07-18 | Dispatch ordinary root jobs by their application type and exact payload, carrying correlation in one versioned driver metadata record instead of the workflow envelope. | Existing root and bus signatures remain source-compatible; advanced metadata helpers and `WithLegacyDirectEnvelope` are additive. Wire behavior changes for root direct jobs, SQL adds nullable `queue_jobs.metadata_json`, and absent payload no longer becomes JSON `null`; arbitrary raw bytes reach handlers without a dispatch-time JSON re-marshal. New workers read old and new deliveries, but old workers cannot safely consume new direct types. Deploy new workers while producers retain legacy emission, switch producers only after all consumers are upgraded, and restore legacy emission plus drain direct backlog before worker rollback. Raw-runtime bus v1 bytes, workflow envelopes, uniqueness keys, configuration files, and the minimum Go version remain stable. | +| DL-017 | 2026-07-18 | Make `queue.NewFake` the sole fake-state owner and reduce `bus.Fake` and `queuefake.Fake` to compatibility views over its direct and workflow records. | Constructor and method signatures, the usable `bus.Fake` zero value, value copyability, and physical `bus.Fake`/`bus.BatchSpec` identities remain source-compatible. Testing runtime behavior is intentionally corrected: queue and workflow compatibility views share direct history and effective default queues; abandoned builders, invalid jobs, and canceled dispatches do not record; builder options survive; chain/batch IDs are opaque lookup identifiers instead of `fake-chain`/`fake-batch`; `FindChain`/`FindBatch` expose pending fake state; `Reset` clears direct, workflow, and store state; and fluent closure callbacks are not retained in fake runtime state or executed. Preserving separate histories or constructor-time records would preserve the duplicate owners and false-positive assertions this milestone removes. Tests that require isolated histories must use distinct fake instances; tests must treat returned IDs as opaque and may inspect them through `FindChain` or `FindBatch`. No configuration, persisted-data, wire-format, operational-rollout, or minimum-Go-version change is implied. | +| DL-018 | 2026-07-18 | Reject ambiguous workflow creation records and make memory-store chain snapshots caller-independent. | This intentionally tightens runtime input validation for direct `WorkflowStore` calls without changing source/API, configuration, persisted schema, wire format, or the minimum Go version. Chain and batch IDs must be non-empty, each record must contain at least one member, and member IDs must be non-empty and unique; the public builders already satisfy these constraints. Direct callers must correct invalid records before upgrading. Memory-store callers must no longer rely on mutating input or returned chain payloads to mutate stored state. Existing persisted rows are not rewritten, but deployments that previously wrote ambiguous chain records directly should audit and repair them before further processing. | +| DL-019 | 2026-07-18 | Require `VARBINARY` for every MySQL workflow identity column and derive accepted widths from the complete connected schema. | This is a MySQL persisted-schema, runtime-behavior, and operational migration correction; it is not a source/API, configuration-file, wire-envelope, or minimum-Go-version break. Fresh auto-schema uses byte-exact 255-byte workflow/member and receipt identities plus 512-byte callback keys. When a legacy schema has no receipt table, automatic startup validates the existing `VARBINARY` identity columns and derives a shared receipt wide enough for both effective workflow-ID capacities and both member-ID capacities. `TestWorkflowStoreIntegration_MySQLAutoMigratesMissingReceiptAtLegacyWidths` proves ordinary startup preserves a live 512-byte legacy schema and accepts long chain, batch, member, callback, and receipt identities. Existing receipt tables are never altered; their capacities intersect the accepted limits of the complete connected schema. Deployments with an incompatible existing receipt must quiesce workflow writers, audit comparison-equivalent and over-limit identities, migrate the table, and construct a fresh store for capacity rediscovery. Extremely wide legacy identities can produce a derived primary key beyond the server's indexed-key budget; startup then reports both widths and schema-first guidance instead of silently narrowing or altering tables. Operators must precreate a compatible indexed schema or explicitly migrate supported limits and existing data before rollout. Managed `VARCHAR`, `TEXT`, and fixed-width `BINARY` columns still fail instead of silently conflating or padding identities. | +| DL-020 | 2026-07-18 | Separate queue-generation provenance, workflow-transition receipts, and the future settlement/continuation outbox. | Each SQL claim carries an opaque generation ID. Same-attempt redelivery normally retains inherited recovery provenance; after the current generation durably claims a transition receipt, the additive delivery-settlement application-state signal makes SQL retain that current generation if later infrastructure still requires redelivery. The signal is not queue settlement or observer delivery, and application retry clears the link. Direct built-in stores record the private immutable receipt in the same transaction as the chain-node or batch-member transition. `receipt_version` versions durable ownership and `event_schema_version` versions the shared observer facts independently from workflow-envelope protocol; both start at `1`, and unsupported values fail closed with an uncommitted outcome. Logical receipt proof requires a complete persisted owner, including a nonnegative owner attempt, matching workflow incarnation/member, dispatch, and job fingerprint plus nonempty current dispatch/`JobID`. The current attempt is physical provenance and may differ from the owner or be negative; chain physical `JobID` may also differ, while batch `JobID` remains the logical member key and must match. That logical proof suppresses handler replay. Reconstructing facts additionally requires the exact prior recovered generation, current attempt, and physical `JobID` owner tuple. A SQL aggregate row must own completion and a cancelled aggregate must own failure; when it names the requested logical member, its incarnation, complete physical owner, and outcome must match that member receipt, and its flags must agree with live terminal state. Contradictions fail uncommitted before effects. Validated durable predecessor success restores its immediate live continuation without handler replay when receipts are absent/hidden or carry non-exact physical provenance, but those paths emit no predecessor facts or callbacks; progressed and terminal state is a no-op, and duplicate successor enqueue remains possible. Failed chain receipts return the first persisted cause as permanent across physical nonowners without replaying callbacks or occurrence-based failure facts; an empty cause becomes a permanent diagnostic. Failed batch receipts return a generic permanent cause across the same physical variants because their original cause is not persisted. Both keep duplicate physical rows on the archive path without fabricated failure facts. Built-in `FailChain` now preserves the first terminal cause rather than allowing later calls to overwrite the authoritative value used by failed-receipt recovery. Direct store callers that relied on late failure-cause replacement must retain that metadata separately. This is a runtime-behavior tightening, not a source/API, configuration, persisted-schema, wire, or minimum-Go-version change. `claimedNow` is response-local, and queue provenance or aggregate state alone is insufficient. Real SQLite, MySQL, and PostgreSQL finalization-failure tests prove supported-version successful terminal-chain recovery; focused memory/SQLite contracts plus a repeated real SQLite archive-failure fixture prove atomic terminal-failure receipts, first-cause archive, and lineage repair. SQLite additionally proves definite chain-successor rejection recovery and failed-batch archive, while the compatibility-focused successor test covers receipt-absent, decorated, and non-exact provenance. Focused corruption contracts pin aggregate incarnation, completion, cancellation, owner, outcome, and member-presence checks. Real twelve-worker MySQL/PostgreSQL races and the SQLite two-member recovery prove only one serialized member owns aggregate terminal effects. This slice otherwise adds `busruntime` API and root runtime behavior without removing a root API or changing configuration files, application wire, or the minimum Go version. It adds `NewSQLStoreWithManagedSchema` because legacy `NewSQLStore` continues enabling migration despite the false `AutoMigrate` zero value, and it adds persisted `bus_workflow_transition_receipts`; managed deployments require schema-first rollout and quiescent worker rollback. Old binaries ignore a retained table, while dropping it loses provenance and old pruning can leave orphan rows. Custom/decorated/raw stores keep public compatibility but have weaker private guarantees outside state-confirmed chain-successor liveness. Successful recovery facts use deterministic IDs; failures remain occurrence-based. The receipt is not a settlement outbox or durable continuation/callback intent. Server-dialect failed-chain finalization evidence, conservative successor re-dispatch's duplicate ambiguity, and physical commit/readback ambiguity remain open. | +| DL-021 | 2026-07-19 | Treat coverage as a repository-wide multi-module and backend fan-in contract. | Unit coverage runs every buildable module independently with `GOWORK=off`; each existing backend matrix leg emits one integration profile from the actual integration module. A final guard rejects missing, extra, malformed, duplicate, or non-executing backend evidence before one explicit Codecov upload. This changes CI evidence only; it does not change source/API, configuration, persisted data, runtime behavior, wire formats, operations, or the minimum Go version. | +| DL-022 | 2026-07-19 | Close coverage gaps with deterministic behavioral tests without adding production-only test seams. | Scripted failures, focused driver contracts, and a targeted RabbitMQ integration scenario now prove reachable migration, lifecycle, settlement, uniqueness, retry, and shutdown boundaries. Explicit defensive tests pin fail-closed behavior for malformed collaborator results without presenting those results as production client behavior. Fixed-structure JSON marshal failures, entropy-source failures, and failures emitted only inside concrete broker clients remain uncovered where exercising them would require global hooks or production-only indirection. This preserves runtime design and test isolation while making the remaining coverage limits explicit. | +| DL-023 | 2026-07-19 | Require opaque physical identity before `settlement_failed` can close a `StatsCollector` active execution. | The source-compatible `busruntime.DeliverySettlementIdentity` type and context accessor are additive public API. Every built-in settlement-aware driver forwards the same handler context through start, process, and settlement facts. Older or custom drivers that omit it may conservatively overcount `Active`; guessing from event fields could undercount a newer execution after a late settlement. Release and upgrade the settlement-aware driver modules with root when exact gauges matter. This changes metrics runtime behavior and adds an operational rollout consideration, but does not change configuration, persisted data, wire formats, or the minimum Go version. | +| DL-024 | 2026-07-19 | Treat a Codecov upload as complete only after the exact workflow upload, commit report, and pull request comparison are processed. | The uploader returning success proves acceptance, not report freshness. Each run now has an explicit commit, build, and upload identity, and the coverage job polls Codecov with a bounded timeout before it can pass. Pull requests additionally require the exact base/head comparison. Fast contracts pin same-SHA aggregate sessions, pagination, stale and incomplete comparisons, transient API responses, and push behavior. The default badge remains intentionally tied to `main` until merge, and the mutable pull request comment remains eventually consistent. This changes CI evidence only; it does not change source/API, configuration, persisted data, runtime behavior, wire formats, operations, or the minimum Go version. | +| DL-025 | 2026-07-19 | Honor `WithHandlerContextDecorator` independently of whether an observer has recipients, while retaining Redis's single native execution boundary. | This corrects documented runtime behavior for Sync, Workerpool, SQL, NATS, SQS, and RabbitMQ without changing source/API, configuration, persisted data, wire formats, operational rollout, or the minimum Go version. Redis continues decorating exactly once in its worker, and a decorator that returns `nil` preserves the original context. | +| DL-026 | 2026-07-19 | Let already-drained Sync cleanup succeed with an expired context and track every admitted Sync dispatch in one mutex-protected work generation. | This corrects runtime lifecycle behavior without changing source/API, configuration, persisted data, wire formats, operational rollout, or the minimum Go version. Immediate and delayed work share a stable completion channel, pending work still honors caller cancellation, shutdown remains latched, and retries converge without waiter goroutines or late continuation admission. Workerpool performs the same continuation recheck while reserving its existing work generation. | +| DL-027 | 2026-07-19 | Make a multi-module release tag family dependency-complete and commit-consistent before any tag mutation, then publish it atomically. | This changes operational release tooling only. Real release validation and tag planning run from an archive of the exact captured commit, followed by final HEAD and cleanliness checks. Shared validation rejects malformed versions and release majors that do not match each module path. A release also fails closed for unresolved sibling pins, dependency-incomplete exclusions, dirty non-dry-run state, an unreadable Git status, stale existing tags, remote inspection failures, or a remote that cannot accept the complete family atomically. Release operators must commit synchronized sibling pins first, use `--allow-dirty` only for dry runs, and reuse existing tags only when they resolve to the captured release commit. There is no library source/API, configuration, persisted-data, runtime, wire-format, or minimum-Go-version change. | +| DL-028 | 2026-07-19 | Make externally managed queue readiness reject an absent or structurally incomplete schema. | `DisableAutoMigrate` still performs no queue DDL, but `Ready` and `StartWorkers` now fail when either operational object is not a base-table relation, including an ordinary or partitioned PostgreSQL table, or any runtime-used column is absent. Views are rejected because the runtime requires inserts, updates, deletes, transactions, and row locking. This intentionally tightens managed-mode runtime behavior without changing source/API, configuration shape, persisted data, wire formats, or the minimum Go version. Operators must provision the complete dialect-correct schema before declaring a deployment ready. The same runtime may retry after provisioning. Presence validation does not certify write permissions, exact SQL types, constraints, or performance indexes. | +| DL-029 | 2026-07-19 | Give every Core NATS server roundtrip a bounded context when the caller supplies no deadline. | The NATS client rejects `FlushWithContext` calls without a deadline, while public queue readiness accepts ordinary background contexts. Readiness, initial dispatch, and worker replacement publication now share a five-second bound and retain any shorter caller deadline. This corrects NATS runtime behavior without changing source/API, configuration, wire formats, persisted data, or the minimum Go version. Initial connection establishment remains context-unaware under M1-16. | +| DL-030 | 2026-07-20 | Preserve exact outstanding pre-version SQL `UniqueFor` claims by atomically acquiring historical and canonical identities. | Current SQL producers claim the unprefixed physical queue/type/payload digest and canonical version-one logical identity in the same transaction as the queue row. A collision or storage failure on either identity rolls back both claims and the row. This is a runtime and persisted-data compatibility correction with no source/API, configuration, schema, wire, or minimum-Go-version change. Each accepted unique dispatch stores two rows in the existing lock table; existing rows need no rewrite and normal expiry pruning removes both formats. Keep dual claims until all old SQL producers are gone and at least their longest possible `UniqueFor` duration has elapsed; this release retains them unconditionally. Historical high-level envelopes included generated correlation, so independently generated logical equivalents still require a longest-live-TTL cutover or an accepted transient duplicate window. The historical delimiter format can also preserve false collisions between distinct canonical jobs until retirement. | +| DL-031 | 2026-07-20 | Split native shutdown into worker drain, stable late-operation wait, and resource close, and preserve runtime delivery state across replacement context decorators. | Native backends stop delivery and drain accepted handlers without closing producer or storage resources. Root then waits the post-drain operation generation before final cleanup, so an admitted continuation cannot be overtaken. `PreserveDeliveryContext` is an additive advanced API that carries the source settlement, provenance, attempt, metadata, and expiring continuation permits into a replacement while retaining the replacement's cancellation, deadlines, and user values; source runtime facts override replacement facts. Redis snapshots Asynq retry and queue facts before decoration. This corrects runtime behavior without changing configuration, persisted data, wire formats, operational rollout, or the minimum Go version. | +| DL-032 | 2026-07-20 | Linearize handler registration with worker activation. | Native runtimes install the current logical handler generation before backend activation. External runtimes publish and catch up the constructed worker before activation. A non-nil registration completed while startup is in flight is therefore present on the potentially consuming backend, while stable handler slots and generation-owned ledgers retain one physical registration across replacements and failed-start retries. Nil and empty registrations remain no-ops and cannot erase pending handlers. This corrects runtime behavior without changing source/API, configuration, persisted data, wire formats, operational rollout, or the minimum Go version. | +| DL-033 | 2026-07-20 | Cache only successful workflow SQL schema initialization. | Schema DDL and MySQL key-limit discovery remain serialized, but caller cancellation, connectivity, locking, permission, and idempotent partial-DDL failures no longer poison the store instance permanently. A later operation retries the complete initialization sequence and publishes MySQL limits only after full success. Permanent failures are retried until operators repair the underlying condition, so repeated operations may produce repeated DDL or catalog attempts. This corrects runtime recovery behavior without changing source/API, configuration, persisted schema, wire formats, operational rollout, or the minimum Go version. | +| DL-034 | 2026-07-20 | Upgrade pgx to the first release that fixes GO-2026-5004 without raising unrelated module baselines. | `driver/postgresqueue` and the integration module pin pgx v5.9.2 or newer through a checked dependency policy. Because pgx v5.9.2 requires Go 1.25, the PostgreSQL driver, examples, integration tooling, and repository workspace now require Go 1.25. The root library and every non-PostgreSQL published driver remain on Go 1.24.4. An exact per-module policy prevents accidental baseline drift, while the workspace must match the highest module version. This is a minimum-Go-version incompatibility for PostgreSQL driver consumers and repository contributors using the workspace. It does not change queue source/API, configuration shape, persisted data, schema, or wire formats. It does incorporate pgx v5.9 runtime changes, including reduced prepared-statement protocol traffic, discarding pooled connections left in transactions during reset, and defaulting an omitted database user to the current operating-system user. PostgreSQL consumers must build with Go 1.25 or newer and should set the database user explicitly when they do not want that upstream default; consumers of the root or other driver modules need no migration. | + +## Progress Log + +### 2026-07-20 + +- Upgraded the PostgreSQL driver and integration suite to pgx v5.9.2, the first release that fixes GO-2026-5004. Added exact mixed-version module policy, overflow-safe semantic dependency floors across every direct owner, replacement rejection, and manifest-derived minimum-toolchain CI that compiles tagged integration, generated examples, and documentation tooling. The root and unrelated driver modules remain on Go 1.24.4, while the PostgreSQL-only Go 1.25 consumer requirement is explicit. +- Replaced one-shot workflow SQL schema failure caching with serialized success-only initialization. A deterministic SQLite regression starts the same store with a canceled first use, then proves a healthy retry completes migration and persists state; focused fix-only reviews verified MySQL limit publication, managed-schema continuity, idempotent DDL replay, and connection ownership. +- Closed the handler-registration startup gap for native and external runtimes. Deterministic race-enabled tests cover new and replaced types during live startup, multiple late types, context-derived handles, concurrent start callers, external factory delay, failed-start retry, and shutdown latching while proving one physical registration per type. Corrected the production guide to name the actual SQL driver recovery settings and regenerated the executed unit count to 969. +- Closed the exact-identity SQL rolling-producer uniqueness gap by claiming historical and canonical identities atomically with queue acceptance. Focused transaction fixtures cover both collision and failure phases for SQLite, PostgreSQL, and MySQL, while the shared real-dialect scenario proves outstanding, expired, rolled-back, and concurrent legacy/current transition behavior. Pre-version high-level envelopes retain the documented longest-live-TTL cutover because their generated correlation prevents exact historical matching. +- Split native shutdown into an explicit worker drain, stable post-drain lease wait, and resource close. Replacement handler contexts now retain runtime-owned continuation and settlement state without inheriting source cancellation or user values, and Redis snapshots Asynq transport facts before decoration. + +### 2026-07-19 + +- Closed the three findings from the fourth fresh-context review. Handler context decoration no longer depends on observer configuration, idle Sync shutdown no longer loses to an already-canceled cleanup context, and the release command refuses unresolved sibling-module tag families before mutation. +- Closed the follow-up lifecycle and release-script edges found while cross-reviewing those fixes. Sync immediate and delayed jobs now share one locked work generation without waiter goroutines, escaped Sync and Workerpool continuations cannot reserve after their permit expires, exclusions preserve module dependency closure, malformed or path-incompatible versions are rejected, unreadable repository status fails closed, dirty trees cannot be tagged, existing local and remote tags must resolve to the captured release commit, remote lookup failures are distinct from absent tags, and the complete tag family is pushed atomically. Offline fixtures cover every failure before touching real repository tags. +- Closed the Codecov freshness race in which the GitHub coverage job could pass before Codecov merged the upload or processed the pull request comparison. Uploads now carry a unique run identity and the job waits for the exact SHA's aggregate commit totals and, on pull requests, the exact base/head comparison. Fast mock contracts cover reruns, pagination, stale or incomplete results, malformed responses, and push behavior. +- Fixed all five primary findings from the third fresh-context review: canonical physical queue labels now agree across queue, worker, and workflow facts; every driver module has an independent parallel race job; manual observer snippets compile against the context-aware signature; settlement failure closes only its exact active execution; and SQL shutdown retries share one bounded drain waiter. +- Closed the post-fix continuity edges found by three specialist audits. Whitespace-only explicit queues retain existing physical routing across every event layer, identity-less late settlements cannot consume a newer execution, handler panics emit truthful failure telemetry before rethrow, metrics documentation uses the canonical physical queue contract, and the historical aggregate `race` check remains available for branch protection while the module checks stay parallel. +- Audited every remaining changed-line coverage marker with fresh NATS, RabbitMQ, Redis, and documentation reviewers. Fixed-structure JSON errors, invalid injected wiring, entropy-source failure, concrete-client transport faults, generator process exits, and the bounded integration timeout remain explicit rather than adding global test hooks. The one deterministic production gap now has a real NATS startup test proving an invalid subscription leaves retryable lifecycle state; focused unit contracts also pin connection-drain diagnostic convergence, expired delivery readiness, and same-attempt uncommitted republish. The authoritative all-backend run passes and records 613 executed integration tests, and the complete 12-module test/vet plus two-pass generation gate remains clean. +- Fixed all six findings from the second fresh-context PR review. Nil handler registration is a no-op at every root, workflow, and deprecated facade boundary; direct, legacy-envelope, and raw compatibility paths return the normal missing-handler error instead of panicking, and nil cannot replace a valid handler. +- Made Redis owned-resource shutdown converge after a close diagnostic. The first cleanup caller receives every joined resource error, later calls return success without closing anything twice, concurrent callers remain race-safe, and the public queue leaves draining on retry. +- Corrected observer migration guidance so legacy workflow sinks retain the three queue-layer dispatch facts, and aligned the metrics taxonomy with the runtime's deliberate event-layer mapping. +- Added an executable generated-documentation guard that runs every deterministic generator, verifies checked-in README, examples, test-count badges, and benchmark dashboards, and proves second-run idempotency in both CI and the all-module gate. Unit counts come from the current executed suite; integration counts combine the all-backend integration module with root integration-tagged tests. Their full-run manifest hashes every Go and module input in the integration module plus root tagged sources and module inputs. Full regeneration rejects partial backend selection and disables optional chaos/soak modes, so unit CI cannot silently preserve stale, reduced, or out-of-module integration evidence. +- Replaced the duplicate-idempotency scenario's two independent dispatches with one job whose first handler attempt commits its keyed side effect and forces a real driver retry. Redis advances its isolated real retry entry deterministically instead of sleeping through randomized production backoff. +- Made chaos and flake-repeat jobs actually run on the weekly schedule. The repeat harness now requires one named backend and validates the exact scenario's `go test -json` terminal event, records capability skips separately, and fails missing execution instead of reporting a false pass. The Redis chaos test stops the broker while a handler is active, proves its successful result cannot be acknowledged, retains the same active task, exercises Asynq's real lease-expiration recovery, preserves the zero application retry budget, redelivers exactly once, and settles with one side effect. The exact scheduled Redis subset and three repeated lost-ack runs pass locally. +- Completed three independent fresh-context reviews of public compatibility, runtime correctness, and test/CI evidence. No new runtime correctness defect survived validation. +- Corrected the README-linked direct-delivery guide so managed SQL schemas add both queue columns and old/new SQL worker generations never overlap during upgrade or rollback. +- Replaced root-only Codecov input with deterministic coverage from every buildable module and all ten parallel backend jobs. The fan-in guard now proves exact artifacts, normalized unique ranges, complete module inventory, and backend-specific executed functions before upload. +- Restored the repository's established Codecov project and patch status policy, made upload failures fail CI, and documented the multi-module collector without implying that coverage replaces behavioral guarantees. +- Used the first complete fan-in report to add focused SQL, NATS, Redis, RabbitMQ, and SQS failure-path tests. SQL combined changed-statement coverage reached 98.3%, NATS 93.3%, Redis 95.7%, SQS 96.3%, and RabbitMQ 90.7% without production hooks or local containers. +- Followed the fan-in report with an honest boundary pass that proves NATS shutdown waits for accepted work, Redis state ownership reaches real command semantics, and RabbitMQ immediate retry advances, commits, and leaves its broker queue empty. Defensive invariant tests verify ambiguous SQL claim results roll back and absent SQS client results fail closed. Fresh-context review separated supported behavior from defensive coverage and removed tests whose only premise was invalid injected collaborators. +- Added the lightweight root integration-tagged `bus` fixture suite to unit collection and made the fan-in guard require proof that it executed. + +### 2026-07-18 + +- Completed the initial architecture, public API, workflow, lifecycle, driver, observability, documentation, and multi-module audit. +- Reproduced the transient workflow retry contradiction and public `UniqueFor` failure. +- Established the north-star model, compatibility guardrails, decision gates, milestones, and validation matrix. +- Accepted one canonical queue/workflow surface, one observer/event model, job-based durable continuations, and direct job execution without workflow wrapping. +- Repaired the NATS, SQS, and RabbitMQ observer test drift; all three full module suites and the repository-wide module compile guard pass independently with `GOWORK=off` for nested modules. +- Added the examples module's independently required dependency graph and verified every generated/manual example build with `GOWORK=off` without further module edits. +- Introduced the first unified-observer slice: root `Event` now carries every layer's correlation fields, `WithObserver` receives queue/worker/workflow events, and the root `Workflow*` observer names are deprecated canonical aliases. +- Replaced copied mutex-bearing runtimes with shared lifecycle state across context-bound handles; the root vet pass is now clean. +- Expanded the all-module validation entrypoint to cover tests, vet, examples, integration, and the docs tooling module, and made CI run the full independent-module pass. +- Added black-box public `Queue` contract fixtures for direct jobs, chains, batches, retries, uniqueness validation, unified observation, lookup, and lifecycle sharing. +- Replaced nested observer wrappers with one concurrency-safe sink retained by root, Redis, SQL, NATS, SQS, and RabbitMQ paths; config and option observers now share event identity and late options reach native driver events. +- Added transitional, schema-gated logical-envelope decoding so queue, worker, workflow, Redis, and broker-republish facts share job type and dispatch/job/chain/batch correlation without leaking volatile wrapper IDs into observability `JobKey`. Driver-enforced `UniqueFor` remains open under M1-05/M1-06. +- Classified dispatch/enqueue/control as queue facts, physical attempts as worker facts, and logical job/chain/batch/callback transitions as workflow facts so `Event.Layer` follows one semantic rule instead of the package that happened to emit an event. +- Accepted the acyclic consolidation sequence `queue -> internal/workflow`, with `bus` retained only as a staged compatibility facade and no new public workflow model. +- Added the first D-001/M2-01 foundation in `busruntime`: one tested attempt classifier now distinguishes retryable application failures, terminal failures, and infrastructure outcomes that require same-attempt redelivery without changing existing handler signatures or runtime behavior. +- Propagated physical attempt metadata through both root adapters and every worker reconstruction path, then corrected chain/batch transitions so public Sync workflows can fail transiently and later complete without stale failed state or premature callbacks. +- Split local enqueue acceptance from inline execution, gated workerpool delivery until acceptance observation completes, and locked the exact Sync success/failure sequence so handler errors are no longer mislabeled as enqueue rejection. +- Added a 12-module workspace/replacement/release/tag guard and brought the docs tooling module into `go.work`. +- Found a release blocker: published nested modules require siblings at nonexistent `v0.0.0` versions. Keep relative replacements for repository testing, but pin every sibling requirement to the prospective release before the next tag family is created. +- Removed the blanket at-least-once documentation claim: the evidence matrix now distinguishes fixture coverage from production guarantees, calls Core NATS explicitly ephemeral, records RabbitMQ's missing publisher-confirm boundary, and no longer presents fixture contention tests as proof of public `UniqueFor`. +- Fast-forwarded to `origin/main` at `18a7647`, preserved its retired-badge removal during regeneration, and moved the reconciled work to `refactor/unify-queue-workflow` for scoped commits and later PR/CI validation. +- Made permanent outcomes operational across local, Redis, SQL, NATS, SQS, and RabbitMQ workers, and introduced a distinct uncommitted outcome for infrastructure/workflow-state failures. Redis's original final-attempt gap is now covered for newly reserved tasks, while legacy queued tasks retain the upstream limitation. +- Deferred logical job and chain/batch terminal facts until the owning workflow mutation commits. Chain, batch, and callback store failures now return the uncommitted outcome, suppress premature callbacks/events, preserve the store cause, and have exhausted-attempt recovery/idempotency regressions. +- Separated synchronous continuation failure from its predecessor's physical outcome: a downstream chain node can return its exact error to the caller without retrying the already-successful node or corrupting failed state into completion. +- Verified the reconciled branch with the full 12-module test-and-vet matrix, independent nested modules under `GOWORK=off`, root race tests, README snippet compilation, the module inventory guard, and stable README/example generation. Integration test-count discovery timed out at its bounded 30-second limit and deliberately retained the existing integration badge rather than fabricating a count. +- Defined one canonical, versioned logical uniqueness identity from effective queue, application job type, and exact payload; workflow correlation IDs and delivery options no longer defeat public `UniqueFor`. +- Replaced duplicated in-memory uniqueness maps with one token-owned store, added pre-acceptance compensation across local and broker drivers, coupled SQL claims to queue-row insertion, and added a backend-shared public facade scenario. +- Added backend-shared Redis logical claims while retaining Asynq physical claims for direct-job rollout compatibility; documented the non-atomic claim/enqueue boundary and coordinated public-workflow rollout requirement. +- Preserved explicit workflow `Retry(0)` instead of falling through to backend defaults, required SQS message receipts and RabbitMQ publisher confirms before replacement settlement, surfaced RabbitMQ/SQS settlement ambiguity, and made closure callback failures observable instead of silently successful. +- Added a header-marked Asynq transport reserve so new Redis tasks can redeliver an uncommitted final application attempt without inflating handler-visible retry counts; terminal outcomes explicitly skip the reserve and lease recovery does not consume it. A container-backed test now proves `Retry(0)` redelivers the same application attempt through the real v0.26 processor. +- Pinned the Redis retry-budget boundary through a real broker: task storage carries `application retries + 1`, the versioned header retains the application value, and worker observation restores that original value for handlers and users. +- Proved canonical Redis uniqueness across concurrent public clients, producer shutdown/restart, and TTL expiry; proved the same public logical composition across concurrent and restarted SQLite clients, and pinned the persisted `v1` key with a golden vector. +- Deferred SQL, SQS, and RabbitMQ positive process/workflow facts until their settlement owner commits. Missing or failed settlement now suppresses success and emits `settlement_failed` with the original delivery attempt; SQL finalization retries are bounded and require exactly one affected row. +- Prevented callback redelivery from converting a prior callback failure into success, skipped absent optional callback deliveries, and made missing ephemeral callback state fail visibly instead of reporting a no-op success. +- Made the Redis timeline/uniqueness store structurally required, validated Asynq's one-second unique TTL before claiming, and closed all owned producer state resources. +- Added an explicit MySQL expiry-index migration probe so existing uniqueness tables receive bounded-pruning support rather than only new installations. +- Normalized absent, zero-byte, and exact JSON `null` payloads into one pinned uniqueness identity so the eventual direct-job cutover cannot split existing workflow claims. +- Made allowed-failure batches derive completion from aggregate state regardless of failure order, proved the behavior through the public facade, and ensured Catch, Then, and Finally each execute once. +- Validated callback workflow state before consuming idempotency markers, isolated callback and Progress panics, removed duplicate orphan starts, and exercised real serialized Catch/Then/Finally envelopes in reverse order. +- Reworked root lifecycle coordination so concurrent starts and shutdown share attempts, failed cleanup remains retryable, never-started producers close, and post-close work is rejected. +- Preserved caller ownership of supplied SQL handles, propagated owned close errors, and added a real SQLite ownership proof alongside Redis owned-resource coverage. +- Leased every root operation that can touch runtime resources, including dispatch, readiness, pause/resume, stats, administration, and history, so shutdown cannot close resources underneath an admitted call. +- Replaced the process-global continuation marker with runtime-scoped, non-transferable permits that expire when a handler returns; foreign and escaped contexts can no longer enqueue after drain begins. +- Added a post-worker quiescence barrier so a descendant admitted during drain finishes before producer cleanup, and gave direct SQL runtimes their own scoped permit rather than trusting a caller-forgeable generic marker. +- Retained partially started external workers for retryable cleanup and installed one stable, replaceable handler slot per job type, so canceled Redis/Asynq startup can retry, same-key replacement remains continuous across startup races, and a started strict mux never receives a duplicate pattern. +- Latched shutdown intent before waiting on in-flight startup, preventing a fresh start or dispatch from overtaking cleanup while the original start is still blocked. +- Made Sync and Workerpool reserve accepted delayed descendants through shutdown, gave bounded workerpool callbacks a reentrant relay that avoids one-worker Catch/Finally deadlock, and made `WithWorkers` control local execution concurrency. +- Made NATS worker startup retryable and subscription-flush-gated, synchronized real Core NATS drain completion, retained the producer connection for admitted callback/delay work, and proved a real queued callback backlog drains. Core NATS remains an ephemeral broadcast adapter whose retry replacement can be accepted without a subscriber, so D-004 is still open. +- Kept workflow queue, job type, and logical `JobKey` correlation across aggregate/callback events, and made allowed batch failures emit progress rather than a false terminal failure before later completion. +- Propagated the first workflow job's queue, type, and logical key into chain/batch start facts, and preserved the triggering job's physical payload metadata through callback envelopes so the unified observer sees one continuous identity. +- Made `settlement_failed` end the collector's active attempt without inventing a processed or application-failed count. +- Added an explicit `DisableAutoMigrate` path across database wrappers, removed poisoned one-shot SQL startup, and proved startup can recover from a real SQLite schema lock. Managed readiness and startup now fail closed without mutating empty or incomplete queue schemas, remain retryable on the same runtime after external provisioning, and run canonical preprovisioned schemas through readiness, uniqueness, dispatch, and consumption on SQLite, MySQL, and PostgreSQL. +- Made Core NATS readiness usable through the public background-context contract by sharing the bounded server-roundtrip context already required by dispatch and worker publication. A public negative rejects a reachable non-NATS endpoint, and the live NATS readiness integration proves the exact background-context path. +- Added per-claim SQL processing tokens, invalidated them on stale recovery and administrative transitions, and required token-matched finalization so an expired handler cannot delete, retry, archive, overwrite, or report success for a row already reclaimed by another worker. Real SQLite tests cover legacy-schema migration plus stale success and stale failure races across two runtimes. +- Began M2-07 with one private workflow-protocol owner shared by root identity/observability and orchestration dispatch. Literal version-one fixtures freeze schema/type names, malformed and unknown-version fallback, legacy JSON payload semantics, transport options, and all chain/batch callback delivery routes before the engine moves. +- Seeded the exact legacy workflow SQLite DDL and persisted records from literal SQL, then proved the current store reads and mutates them without rewriting `nodes_json`, losing active/recent state, duplicating callbacks, or changing terminal-prune semantics. +- Validated the frozen slice with the full 12-module test/vet gate, root and every concurrency-sensitive driver under the race detector, README snippets, stable two-pass generation, the complete local/SQLite integration matrix, and real container-backed Redis and NATS lifecycle scenarios. +- Moved the cohesive chain, batch, callback, middleware, event, and store implementation behind one `internal/workflow.Engine`; root production no longer imports public `bus`, and an architecture guard prevents that dependency from returning. +- Established physical root workflow records, stores, middleware, messages, and results with explicit root-to-engine adapters so GoDoc, reflection, code generation, and custom stores see `queue` rather than an inaccessible internal package. +- Rebuilt public `bus` as a deprecated compatibility facade: option-free `bus.New(existingQueue)` shares the configured root engine, the raw-`busruntime.Runtime` seam delegates to the same internal engine, construction-only options are rejected explicitly on existing queues, and source fixtures retain custom Bus/Store/Middleware/builders, Temporal, fake, composite-literal, and payload behavior. +- Removed the second observer producer. Legacy `bus.Event` values are translated only for compatibility consumers while one root observer receives delivery and workflow facts. +- Completed M2-07 after direct tests covered both store-adapter directions, physical middleware branches, package identities, queue/raw facade builders and lifecycle, exact error propagation, nil/empty payload ownership, and v1 Dispatch-time payload encoding. Revalidated all 12 modules with vet and independent nested-module resolution, root and concurrency-sensitive drivers under the race detector, stable two-pass example/README generation, README snippets, the local/SQLite integration matrix, and real container-backed Redis and NATS suites. +- Completed M2-05/M2-10: ordinary root jobs now retain their application type and exact payload, while one versioned correlation record travels through in-memory jobs, Redis headers, NATS/SQS/RabbitMQ messages, and nullable SQL storage. The engine registers both direct application handlers and legacy envelope handlers, so middleware, retry classification, logical events, settlement deferral, queued v1 work, reserved protocol names, and raw-bus wire fixtures remain unified. +- Added the workers-first migration gate `WithLegacyDirectEnvelope`, additive concurrent-safe SQL metadata migration, caller-managed-schema diagnostics, malformed/future metadata fallback, backend retry/republication identity tests, and shared integration assertions that the dispatch receipt matches the delivered `Message` across local, SQLite, Redis, and NATS execution. +- Completed M5-07 by moving all fake state into `queue.FakeQueue`, adapting the deprecated bus and queuefake surfaces onto it, and running fake workflows through the production engine without retaining non-executing closure callbacks in fake runtime state. Focused tests cover execution-time recording, validation and cancellation failures, builder reuse and policy, payload isolation, lookup/reset behavior, the legacy zero value, shared compatibility views, and concurrent access under the race detector. +- Completed M3-04/M3-05 with one additive first-writer `WorkflowOutcomeStore` used by memory, SQL, fake, root, and deprecated compatibility paths. Chain success/failure compare-and-swap the same ordered node, batch member outcome categories remain immutable, and losing redeliveries publish no contradictory job/workflow facts, progress, or callbacks. Concurrent outcome races, claim rollback faults, legacy dual-terminal rows, fail-fast settlement, callback claims, caller ownership, and repeated real SQLite, MySQL, and PostgreSQL contracts now cover the transition boundary. MySQL auto-schema uses byte-exact dialect types, rejects fresh-schema truncation, and rejects non-`VARBINARY` managed identity columns. A real legacy-width integration now drops only the receipt table beside 512-byte state identities, proves ordinary startup derives a 512/512 replacement without altering existing tables, and exercises identities above the fresh defaults. The separate managed-width fixture proves complete pre-existing schema discovery. +- Added opaque SQL delivery-generation provenance and separate built-in workflow transition receipts without conflating either with continuations or observer delivery. Forced SQLite, MySQL, and PostgreSQL finalization failures prove supported receipts suppress duplicate application execution while exact recovered-generation ownership gates reconstructed success facts. Focused memory/SQLite store and runtime contracts add terminal chain-failure receipts atomically, return the first persisted permanent cause across exact, different, or legacy generation provenance, suppress repeated handlers/callbacks/failure facts, roll back parent failure when receipt insertion fails, and fail invalid receipts closed. Built-in `FailChain` now preserves that first cause; direct callers that used late calls as replacement metadata must retain those diagnostics separately. A real SQLite chain-failure fixture forces the initial archive and multiple recovery archives to fail, retains owner/attempt-zero/cause through fenced best-effort repair, and then reaches `dead` at attempt one with a single application/workflow occurrence. SQLite additionally covers completed predecessors, later-attempt ownership, aggregate non-inference, a two-member completing-owner recovery, and a failed batch member that archives with a generic durable cause. Active chain recovery re-dispatches an immediate successor after definite rejection without replaying its predecessor. A compatibility matrix now proves the same liveness-only behavior for missing receipts, decorated stores, and different/legacy generation provenance, with no predecessor facts/callbacks and no dispatch after successor progress or terminal state. Duplicate enqueue remains possible while successor progress is not durable intent. Real twelve-worker MySQL and PostgreSQL fail-fast races prove the locked parent transition produces one aggregate owner and one terminal fact pair. `receipt_version` and shared observer `event_schema_version` both start at `1`, evolve independently from the workflow-envelope protocol, and fail recovery closed when unsupported. Application retry clears the earlier generation link, while the delivery-settlement application-state signal preserves a current receipt owner when later infrastructure requests another same-attempt redelivery. Server-dialect failed-chain finalization evidence, remaining custom/decorated/raw-store parity, durable callbacks/continuations, settlement outbox, managed-schema/pruning gates, and physical commit/readback ambiguity remain open. +- Final receipt hardening separates logical transition proof from exact physical fact ownership. A complete persisted owner still requires a nonnegative attempt, but a duplicate's current attempt may differ or be negative; chain physical `JobID` may differ, while batch `JobID` remains its logical member. These nonowners suppress handlers and facts, preserve only a live chain successor, and keep failed chain/batch deliveries on their permanent archive path. SQL aggregate readback now fails closed for stale incarnation, missing completion/member, success-owned cancellation, or owner/outcome disagreement with the member receipt; runtime flags must also match live terminal state. + +## Next Action + +Continue **M1-01/M1-04/M3-09** with managed-schema migration/rollback plus real cross-dialect pruning and physical commit/readback ambiguity gates, and explicit custom/decorated/raw-store fallback contracts. Then expand opaque generation provenance across settlement owners where the backend can support it. In parallel, specify the separate driver-owned settlement outbox and persisted callback/continuation intents needed when no delivery survives. Keep exact successor-enqueue ownership despite conservative at-least-once recovery, partial batch fan-out, Redis claim/enqueue, Core NATS ephemerality, SQS visibility extension, and RabbitMQ reconnect/context-aware lifecycle gaps explicit. diff --git a/public_queue_contract_test.go b/public_queue_contract_test.go new file mode 100644 index 0000000..332c161 --- /dev/null +++ b/public_queue_contract_test.go @@ -0,0 +1,1007 @@ +package queue_test + +import ( + "context" + "errors" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/goforj/queue" +) + +// TestPublicQueueContractNilRegistrationIsNoop verifies nil handlers neither create a delivery target nor replace an established handler. +func TestPublicQueueContractNilRegistrationIsNoop(t *testing.T) { + tests := []struct { + name string + opts []queue.Option + }{ + {name: "direct delivery"}, + {name: "legacy envelope", opts: []queue.Option{queue.WithLegacyDirectEnvelope()}}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + q, err := queue.NewSync(test.opts...) + if err != nil { + t.Fatalf("new sync queue: %v", err) + } + t.Cleanup(func() { + if shutdownErr := q.Shutdown(context.Background()); shutdownErr != nil { + t.Errorf("shutdown: %v", shutdownErr) + } + }) + + const jobType = "contract:nil-registration" + q.Register(jobType, nil) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + if _, err := q.Dispatch(queue.NewJob(jobType)); err == nil { + t.Fatal("nil registration created a delivery target") + } else if message := err.Error(); !strings.Contains(message, "handler") || !strings.Contains(message, "registered") { + t.Fatalf("nil registration dispatch error = %v, want missing handler", err) + } + + var calls atomic.Int32 + q.Register(jobType, func(context.Context, queue.Message) error { + calls.Add(1) + return nil + }) + q.Register(jobType, nil) + if _, err := q.Dispatch(queue.NewJob(jobType)); err != nil { + t.Fatalf("dispatch after nil replacement: %v", err) + } + if calls.Load() != 1 { + t.Fatalf("handler calls = %d, want 1", calls.Load()) + } + }) + } +} + +// TestPublicQueueContractDirectDispatchPreservesMessageIdentity verifies that the normal facade exposes one application job identity rather than its internal workflow envelope. +func TestPublicQueueContractDirectDispatchPreservesMessageIdentity(t *testing.T) { + q, err := queue.NewSync() + if err != nil { + t.Fatalf("new sync queue: %v", err) + } + t.Cleanup(func() { + if shutdownErr := q.Shutdown(context.Background()); shutdownErr != nil { + t.Errorf("shutdown: %v", shutdownErr) + } + }) + + type payload struct { + ID int `json:"id"` + Name string `json:"name"` + } + wantPayload := payload{ID: 42, Name: "facade"} + var ( + seenMessage queue.Message + seenPayload payload + ) + q.Register("contract:direct", func(_ context.Context, message queue.Message) error { + seenMessage = message + return message.Bind(&seenPayload) + }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + + result, err := q.Dispatch( + queue.NewJob("contract:direct"). + Payload(wantPayload). + OnQueue("default"), + ) + if err != nil { + t.Fatalf("dispatch: %v", err) + } + if result.DispatchID == "" { + t.Fatal("dispatch result must contain an ID") + } + if seenPayload != wantPayload { + t.Fatalf("handler payload = %+v, want %+v", seenPayload, wantPayload) + } + if seenMessage.SchemaVersion == 0 { + t.Fatal("message schema version must be populated") + } + if seenMessage.JobType != "contract:direct" { + t.Fatalf("message job type = %q, want %q", seenMessage.JobType, "contract:direct") + } + if seenMessage.DispatchID != result.DispatchID { + t.Fatalf("message dispatch ID = %q, want %q", seenMessage.DispatchID, result.DispatchID) + } + if seenMessage.JobID == "" { + t.Fatal("message job ID must be populated") + } + if seenMessage.ChainID != "" || seenMessage.BatchID != "" { + t.Fatalf("direct message unexpectedly contains workflow identity: %+v", seenMessage) + } +} + +// TestPublicQueueContractChainUsesCanonicalJobs verifies sequential execution, correlation, callbacks, and lookup through Queue alone. +func TestPublicQueueContractChainUsesCanonicalJobs(t *testing.T) { + q, err := queue.NewSync() + if err != nil { + t.Fatalf("new sync queue: %v", err) + } + t.Cleanup(func() { + if shutdownErr := q.Shutdown(context.Background()); shutdownErr != nil { + t.Errorf("shutdown: %v", shutdownErr) + } + }) + + var messages []queue.Message + q.Register("contract:chain:first", func(_ context.Context, message queue.Message) error { + messages = append(messages, message) + return nil + }) + q.Register("contract:chain:second", func(_ context.Context, message queue.Message) error { + messages = append(messages, message) + return nil + }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + + var ( + finallyCalls int + finallyState queue.ChainState + ) + chainID, err := q.Chain( + queue.NewJob("contract:chain:first"), + queue.NewJob("contract:chain:second"), + ). + OnQueue("critical"). + Finally(func(_ context.Context, state queue.ChainState) error { + finallyCalls++ + finallyState = state + return nil + }). + Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch chain: %v", err) + } + + state, err := q.FindChain(context.Background(), chainID) + if err != nil { + t.Fatalf("find chain: %v", err) + } + if state.ChainID != chainID || state.DispatchID == "" { + t.Fatalf("chain identity is incomplete: %+v", state) + } + if state.Queue != "critical" { + t.Fatalf("chain queue = %q, want %q", state.Queue, "critical") + } + if !state.Completed || state.Failed || state.NextIndex != 2 { + t.Fatalf("chain terminal state is inconsistent: %+v", state) + } + if len(messages) != 2 { + t.Fatalf("chain handler calls = %d, want 2", len(messages)) + } + wantTypes := []string{"contract:chain:first", "contract:chain:second"} + for index, message := range messages { + if message.JobType != wantTypes[index] { + t.Fatalf("chain message %d job type = %q, want %q", index, message.JobType, wantTypes[index]) + } + if message.ChainID != chainID || message.BatchID != "" { + t.Fatalf("chain message %d has incorrect workflow identity: %+v", index, message) + } + if message.DispatchID != state.DispatchID || message.JobID == "" { + t.Fatalf("chain message %d has incomplete correlation: %+v", index, message) + } + } + if messages[0].JobID == messages[1].JobID { + t.Fatalf("chain nodes share job ID %q", messages[0].JobID) + } + if finallyCalls != 1 || finallyState.ChainID != chainID || !finallyState.Completed { + t.Fatalf("chain finally callback = (%d, %+v), want one completed callback", finallyCalls, finallyState) + } +} + +// TestPublicQueueContractBatchUsesCanonicalJobs verifies aggregate state, correlation, and callback behavior through Queue alone. +func TestPublicQueueContractBatchUsesCanonicalJobs(t *testing.T) { + q, err := queue.NewSync() + if err != nil { + t.Fatalf("new sync queue: %v", err) + } + t.Cleanup(func() { + if shutdownErr := q.Shutdown(context.Background()); shutdownErr != nil { + t.Errorf("shutdown: %v", shutdownErr) + } + }) + + var messages []queue.Message + q.Register("contract:batch:item", func(_ context.Context, message queue.Message) error { + messages = append(messages, message) + return nil + }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + + var ( + progressCalls int + thenCalls int + finallyCalls int + ) + batchID, err := q.Batch( + queue.NewJob("contract:batch:item").Payload(map[string]int{"id": 1}), + queue.NewJob("contract:batch:item").Payload(map[string]int{"id": 2}), + ). + Name("public contract"). + OnQueue("bulk"). + Progress(func(_ context.Context, _ queue.BatchState) error { + progressCalls++ + return nil + }). + Then(func(_ context.Context, _ queue.BatchState) error { + thenCalls++ + return nil + }). + Finally(func(_ context.Context, _ queue.BatchState) error { + finallyCalls++ + return nil + }). + Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch batch: %v", err) + } + + state, err := q.FindBatch(context.Background(), batchID) + if err != nil { + t.Fatalf("find batch: %v", err) + } + if state.BatchID != batchID || state.DispatchID == "" { + t.Fatalf("batch identity is incomplete: %+v", state) + } + if state.Name != "public contract" || state.Queue != "bulk" { + t.Fatalf("batch metadata is incomplete: %+v", state) + } + if !state.Completed || state.Cancelled || state.Total != 2 || state.Processed != 2 || state.Pending != 0 || state.Failed != 0 { + t.Fatalf("batch terminal state is inconsistent: %+v", state) + } + if len(messages) != 2 { + t.Fatalf("batch handler calls = %d, want 2", len(messages)) + } + for index, message := range messages { + if message.JobType != "contract:batch:item" { + t.Fatalf("batch message %d job type = %q, want %q", index, message.JobType, "contract:batch:item") + } + if message.BatchID != batchID || message.ChainID != "" { + t.Fatalf("batch message %d has incorrect workflow identity: %+v", index, message) + } + if message.DispatchID != state.DispatchID || message.JobID == "" { + t.Fatalf("batch message %d has incomplete correlation: %+v", index, message) + } + } + if messages[0].JobID == messages[1].JobID { + t.Fatalf("batch items share job ID %q", messages[0].JobID) + } + if progressCalls != 2 || thenCalls != 1 || finallyCalls != 1 { + t.Fatalf("batch callback calls = progress:%d then:%d finally:%d, want 2/1/1", progressCalls, thenCalls, finallyCalls) + } +} + +// TestPublicQueueContractAllowedBatchFailureIsOrderIndependent verifies the root facade finishes every accepted item and derives callbacks from aggregate state. +func TestPublicQueueContractAllowedBatchFailureIsOrderIndependent(t *testing.T) { + for _, failureFirst := range []bool{true, false} { + name := "failure_last" + if failureFirst { + name = "failure_first" + } + t.Run(name, func(t *testing.T) { + q, err := queue.NewSync() + if err != nil { + t.Fatalf("new sync queue: %v", err) + } + t.Cleanup(func() { + if shutdownErr := q.Shutdown(context.Background()); shutdownErr != nil { + t.Errorf("shutdown: %v", shutdownErr) + } + }) + + failureErr := errors.New("allowed public batch failure") + var handled int + q.Register("contract:batch:allowed-success", func(context.Context, queue.Message) error { + handled++ + return nil + }) + q.Register("contract:batch:allowed-failure", func(context.Context, queue.Message) error { + handled++ + return failureErr + }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + + jobs := []queue.Job{ + queue.NewJob("contract:batch:allowed-success"), + queue.NewJob("contract:batch:allowed-failure"), + } + if failureFirst { + jobs[0], jobs[1] = jobs[1], jobs[0] + } + + var catchCalls, thenCalls, finallyCalls int + batchID, dispatchErr := q.Batch(jobs...). + AllowFailures(). + Catch(func(_ context.Context, _ queue.BatchState, callbackErr error) error { + if callbackErr == nil || (!errors.Is(callbackErr, failureErr) && callbackErr.Error() != failureErr.Error()) { + t.Fatalf("catch error = %v, want %v", callbackErr, failureErr) + } + catchCalls++ + return nil + }). + Then(func(context.Context, queue.BatchState) error { + thenCalls++ + return nil + }). + Finally(func(context.Context, queue.BatchState) error { + finallyCalls++ + return nil + }). + Dispatch(context.Background()) + if !errors.Is(dispatchErr, failureErr) { + t.Fatalf("dispatch error = %v, want %v", dispatchErr, failureErr) + } + + state, err := q.FindBatch(context.Background(), batchID) + if err != nil { + t.Fatalf("find batch: %v", err) + } + if handled != 2 || state.Processed != 2 || state.Pending != 0 || state.Failed != 1 || !state.Completed || state.Cancelled { + t.Fatalf("handled/state = %d/%+v, want two processed and completed with one allowed failure", handled, state) + } + if catchCalls != 1 || thenCalls != 1 || finallyCalls != 1 { + t.Fatalf("catch/then/finally calls = %d/%d/%d, want 1/1/1", catchCalls, thenCalls, finallyCalls) + } + }) + } +} + +// TestPublicQueueContractRetryEventuallySucceeds verifies retry policy remains effective when dispatched through the workflow-capable facade. +func TestPublicQueueContractRetryEventuallySucceeds(t *testing.T) { + var ( + attempts atomic.Int32 + messages []queue.Message + ) + q, err := queue.NewSync() + if err != nil { + t.Fatalf("new sync queue: %v", err) + } + t.Cleanup(func() { + if shutdownErr := q.Shutdown(context.Background()); shutdownErr != nil { + t.Errorf("shutdown: %v", shutdownErr) + } + }) + q.Register("contract:retry", func(_ context.Context, message queue.Message) error { + messages = append(messages, message) + if attempts.Add(1) < 3 { + return errors.New("transient contract failure") + } + return nil + }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + + result, err := q.Dispatch(queue.NewJob("contract:retry").OnQueue("default").Retry(2)) + if err != nil { + t.Fatalf("dispatch retrying job: %v", err) + } + if attempts.Load() != 3 || len(messages) != 3 { + t.Fatalf("retry handler calls = %d/%d, want 3", attempts.Load(), len(messages)) + } + for index, message := range messages { + if message.DispatchID != result.DispatchID || message.JobID == "" { + t.Fatalf("retry message %d has incomplete identity: %+v", index, message) + } + if message.JobID != messages[0].JobID { + t.Fatalf("retry message %d job ID = %q, want stable ID %q", index, message.JobID, messages[0].JobID) + } + } +} + +// TestPublicQueueContractUniqueValidationReachesFacade verifies invalid deduplication policy cannot be hidden by the internal workflow envelope. +func TestPublicQueueContractUniqueValidationReachesFacade(t *testing.T) { + q, err := queue.NewSync() + if err != nil { + t.Fatalf("new sync queue: %v", err) + } + var calls atomic.Int32 + q.Register("contract:unique", func(context.Context, queue.Message) error { + calls.Add(1) + return nil + }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + t.Cleanup(func() { + if shutdownErr := q.Shutdown(context.Background()); shutdownErr != nil { + t.Errorf("shutdown: %v", shutdownErr) + } + }) + + if _, err := q.Dispatch(queue.NewJob("contract:unique").UniqueFor(-time.Second)); err == nil { + t.Fatal("negative uniqueness TTL must fail public dispatch") + } + if calls.Load() != 0 { + t.Fatalf("invalid unique job executed %d times", calls.Load()) + } +} + +// TestPublicQueueContractUniqueUsesLogicalJob verifies random workflow correlation IDs cannot bypass deduplication. +func TestPublicQueueContractUniqueUsesLogicalJob(t *testing.T) { + q, err := queue.NewSync() + if err != nil { + t.Fatalf("new sync queue: %v", err) + } + q.Register("contract:logical-unique", func(context.Context, queue.Message) error { return nil }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + t.Cleanup(func() { + if shutdownErr := q.Shutdown(context.Background()); shutdownErr != nil { + t.Errorf("shutdown: %v", shutdownErr) + } + }) + + job := queue.NewJob("contract:logical-unique"). + Payload(map[string]int{"account_id": 42}). + OnQueue("critical"). + UniqueFor(time.Minute) + if _, err := q.Dispatch(job); err != nil { + t.Fatalf("first dispatch: %v", err) + } + if _, err := q.Dispatch(job); !errors.Is(err, queue.ErrDuplicate) { + t.Fatalf("second dispatch error = %v, want ErrDuplicate", err) + } +} + +// TestPublicQueueContractObserverSpansEveryLayer verifies one exported observer receives queue, worker, and workflow facts. +func TestPublicQueueContractObserverSpansEveryLayer(t *testing.T) { + var ( + eventsMu sync.Mutex + events []queue.Event + ) + observer := queue.ObserverFunc(func(_ context.Context, event queue.Event) { + eventsMu.Lock() + events = append(events, event) + eventsMu.Unlock() + }) + q, err := queue.NewSync(queue.WithObserver(observer)) + if err != nil { + t.Fatalf("new observed sync queue: %v", err) + } + q.Register("contract:observed", func(context.Context, queue.Message) error { return nil }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + t.Cleanup(func() { + if shutdownErr := q.Shutdown(context.Background()); shutdownErr != nil { + t.Errorf("shutdown: %v", shutdownErr) + } + }) + + result, err := q.Dispatch(queue.NewJob("contract:observed").OnQueue("default")) + if err != nil { + t.Fatalf("dispatch observed job: %v", err) + } + eventsMu.Lock() + snapshot := append([]queue.Event(nil), events...) + eventsMu.Unlock() + + required := map[queue.EventKind]queue.EventLayer{ + queue.EventDispatchStarted: queue.EventLayerQueue, + queue.EventEnqueueAccepted: queue.EventLayerQueue, + queue.EventProcessStarted: queue.EventLayerWorker, + queue.EventJobStarted: queue.EventLayerWorkflow, + queue.EventProcessSucceeded: queue.EventLayerWorker, + queue.EventJobSucceeded: queue.EventLayerWorkflow, + queue.EventDispatchSucceeded: queue.EventLayerQueue, + } + var correlatedJobID string + var correlatedJobKey string + for kind, wantLayer := range required { + var found *queue.Event + for index := range snapshot { + if snapshot[index].Kind == kind { + found = &snapshot[index] + break + } + } + if found == nil { + t.Errorf("observer did not receive %q: %+v", kind, snapshot) + continue + } + if found.Layer != wantLayer { + t.Errorf("%q layer = %q, want %q", kind, found.Layer, wantLayer) + } + if found.SchemaVersion == 0 || found.EventID == "" || found.Time.IsZero() { + t.Errorf("%q event envelope is incomplete: %+v", kind, *found) + } + if found.JobType != "contract:observed" { + t.Errorf("%q job type = %q, want %q", kind, found.JobType, "contract:observed") + } + if found.Queue != "default" { + t.Errorf("%q queue = %q, want default", kind, found.Queue) + } + if found.JobKey == "" { + t.Errorf("%q job key is empty", kind) + } else if correlatedJobKey == "" { + correlatedJobKey = found.JobKey + } else if found.JobKey != correlatedJobKey { + t.Errorf("%q job key = %q, want shared key %q", kind, found.JobKey, correlatedJobKey) + } + if found.DispatchID != result.DispatchID { + t.Errorf("%q dispatch ID = %q, want %q", kind, found.DispatchID, result.DispatchID) + } + if found.JobID == "" { + t.Errorf("%q job ID is empty", kind) + } else if correlatedJobID == "" { + correlatedJobID = found.JobID + } else if found.JobID != correlatedJobID { + t.Errorf("%q job ID = %q, want shared ID %q", kind, found.JobID, correlatedJobID) + } + } +} + +// TestPublicQueueContractObserverUsesEffectiveQueueAcrossDirectAndWorkflows verifies namespace mapping does not split correlated events or collector buckets. +func TestPublicQueueContractObserverUsesEffectiveQueueAcrossDirectAndWorkflows(t *testing.T) { + const ( + logicalQueue = "critical" + whitespaceQueue = " " + physicalQueue = "billing_critical" + physicalWhitespaceQueue = "billing_default" + physicalDefaultQueue = "default" + ) + + var ( + eventsMu sync.Mutex + events []queue.Event + ) + collector := queue.NewStatsCollector() + observer := queue.MultiObserver( + queue.ObserverFunc(func(_ context.Context, event queue.Event) { + eventsMu.Lock() + events = append(events, event) + eventsMu.Unlock() + }), + collector, + ) + q, err := queue.New( + queue.Config{ + Driver: queue.DriverSync, + DefaultQueue: "billing_default", + }, + queue.WithObserver(observer), + ) + if err != nil { + t.Fatalf("new namespaced sync queue: %v", err) + } + t.Cleanup(func() { + if shutdownErr := q.Shutdown(context.Background()); shutdownErr != nil { + t.Errorf("shutdown: %v", shutdownErr) + } + }) + + jobTypes := []string{ + "contract:effective-queue:direct", + "contract:effective-queue:chain:first", + "contract:effective-queue:chain:second", + "contract:effective-queue:batch:first", + "contract:effective-queue:batch:second", + "contract:effective-whitespace:direct", + "contract:effective-whitespace:chain:first", + "contract:effective-whitespace:chain:second", + "contract:effective-whitespace:batch:first", + "contract:effective-whitespace:batch:second", + "contract:effective-default:direct", + "contract:effective-default:chain:first", + "contract:effective-default:chain:second", + "contract:effective-default:batch:first", + "contract:effective-default:batch:second", + } + for _, jobType := range jobTypes { + q.Register(jobType, func(context.Context, queue.Message) error { return nil }) + } + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + + direct, err := q.Dispatch( + queue.NewJob("contract:effective-queue:direct").OnQueue(logicalQueue), + ) + if err != nil { + t.Fatalf("dispatch direct job: %v", err) + } + chainID, err := q.Chain( + queue.NewJob("contract:effective-queue:chain:first"), + queue.NewJob("contract:effective-queue:chain:second"), + ).OnQueue(logicalQueue).Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch chain: %v", err) + } + batchID, err := q.Batch( + queue.NewJob("contract:effective-queue:batch:first"), + queue.NewJob("contract:effective-queue:batch:second"), + ).OnQueue(logicalQueue).Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch batch: %v", err) + } + whitespaceDirect, err := q.Dispatch( + queue.NewJob("contract:effective-whitespace:direct").OnQueue(whitespaceQueue), + ) + if err != nil { + t.Fatalf("dispatch whitespace direct job: %v", err) + } + whitespaceChainID, err := q.Chain( + queue.NewJob("contract:effective-whitespace:chain:first"), + queue.NewJob("contract:effective-whitespace:chain:second"), + ).OnQueue(whitespaceQueue).Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch whitespace chain: %v", err) + } + whitespaceBatchID, err := q.Batch( + queue.NewJob("contract:effective-whitespace:batch:first"), + queue.NewJob("contract:effective-whitespace:batch:second"), + ).OnQueue(whitespaceQueue).Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch whitespace batch: %v", err) + } + defaultDirect, err := q.Dispatch( + queue.NewJob("contract:effective-default:direct"), + ) + if err != nil { + t.Fatalf("dispatch default direct job: %v", err) + } + defaultChainID, err := q.Chain( + queue.NewJob("contract:effective-default:chain:first"), + queue.NewJob("contract:effective-default:chain:second"), + ).Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch default chain: %v", err) + } + defaultBatchID, err := q.Batch( + queue.NewJob("contract:effective-default:batch:first"), + queue.NewJob("contract:effective-default:batch:second"), + ).Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch default batch: %v", err) + } + + eventsMu.Lock() + snapshot := append([]queue.Event(nil), events...) + eventsMu.Unlock() + + scopes := []struct { + name string + expectedQueue string + matches func(queue.Event) bool + }{ + { + name: "direct", + expectedQueue: physicalQueue, + matches: func(event queue.Event) bool { + return event.DispatchID == direct.DispatchID + }, + }, + { + name: "chain", + expectedQueue: physicalQueue, + matches: func(event queue.Event) bool { + return event.ChainID == chainID + }, + }, + { + name: "batch", + expectedQueue: physicalQueue, + matches: func(event queue.Event) bool { + return event.BatchID == batchID + }, + }, + { + name: "whitespace direct", + expectedQueue: physicalWhitespaceQueue, + matches: func(event queue.Event) bool { + return event.DispatchID == whitespaceDirect.DispatchID + }, + }, + { + name: "whitespace chain", + expectedQueue: physicalWhitespaceQueue, + matches: func(event queue.Event) bool { + return event.ChainID == whitespaceChainID + }, + }, + { + name: "whitespace batch", + expectedQueue: physicalWhitespaceQueue, + matches: func(event queue.Event) bool { + return event.BatchID == whitespaceBatchID + }, + }, + { + name: "default direct", + expectedQueue: physicalDefaultQueue, + matches: func(event queue.Event) bool { + return event.DispatchID == defaultDirect.DispatchID + }, + }, + { + name: "default chain", + expectedQueue: physicalDefaultQueue, + matches: func(event queue.Event) bool { + return event.ChainID == defaultChainID + }, + }, + { + name: "default batch", + expectedQueue: physicalDefaultQueue, + matches: func(event queue.Event) bool { + return event.BatchID == defaultBatchID + }, + }, + } + for _, scope := range scopes { + layers := make(map[queue.EventLayer]bool) + matched := 0 + for _, event := range snapshot { + if !scope.matches(event) { + continue + } + matched++ + layers[event.Layer] = true + if event.Queue != scope.expectedQueue { + t.Errorf("%s event %q queue = %q, want %q: %+v", scope.name, event.Kind, event.Queue, scope.expectedQueue, event) + } + } + if matched == 0 { + t.Errorf("observer received no %s events: %+v", scope.name, snapshot) + continue + } + for _, layer := range []queue.EventLayer{ + queue.EventLayerQueue, + queue.EventLayerWorker, + queue.EventLayerWorkflow, + } { + if !layers[layer] { + t.Errorf("%s events did not include %q layer: %+v", scope.name, layer, snapshot) + } + } + } + + stats := collector.Snapshot() + queues := stats.Queues() + if len(queues) != 3 || queues[0] != physicalQueue || queues[1] != physicalWhitespaceQueue || queues[2] != physicalDefaultQueue { + t.Fatalf("collector queues = %v, want [%s %s %s]", queues, physicalQueue, physicalWhitespaceQueue, physicalDefaultQueue) + } + for _, queueName := range []string{physicalQueue, physicalWhitespaceQueue, physicalDefaultQueue} { + if processed := stats.Processed(queueName); processed != 5 { + t.Errorf("collector processed[%q] = %d, want 5", queueName, processed) + } + } + if processed := stats.Processed(logicalQueue); processed != 0 { + t.Errorf("collector created logical queue bucket %q with %d processed jobs", logicalQueue, processed) + } +} + +// TestPublicQueueContractWorkflowAggregatesPreserveQueueAndIdentity verifies aggregate facts retain the triggering logical job's known correlation. +func TestPublicQueueContractWorkflowAggregatesPreserveQueueAndIdentity(t *testing.T) { + var events []queue.Event + q, err := queue.NewSync(queue.WithObserver(queue.ObserverFunc(func(_ context.Context, event queue.Event) { + events = append(events, event) + }))) + if err != nil { + t.Fatalf("new observed sync queue: %v", err) + } + q.Register("contract:aggregate:first", func(context.Context, queue.Message) error { return nil }) + q.Register("contract:aggregate:second", func(context.Context, queue.Message) error { return nil }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + t.Cleanup(func() { + if shutdownErr := q.Shutdown(context.Background()); shutdownErr != nil { + t.Errorf("shutdown: %v", shutdownErr) + } + }) + if _, err := q.Batch().Dispatch(context.Background()); err == nil { + t.Fatal("empty batch dispatch unexpectedly succeeded") + } + chainID, err := q.Chain( + queue.NewJob("contract:aggregate:first").Payload(map[string]string{"value": "first"}).OnQueue("critical"), + queue.NewJob("contract:aggregate:second").Payload(map[string]string{"value": "second"}).OnQueue("critical"), + ).Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch chain: %v", err) + } + batchID, err := q.Batch( + queue.NewJob("contract:aggregate:first").Payload(map[string]string{"value": "batch"}).OnQueue("critical"), + ).Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch batch: %v", err) + } + + tests := []struct { + kind queue.EventKind + chainID string + batchID string + jobType string + }{ + {kind: queue.EventChainStarted, chainID: chainID, jobType: "contract:aggregate:first"}, + {kind: queue.EventChainAdvanced, chainID: chainID, jobType: "contract:aggregate:first"}, + {kind: queue.EventChainCompleted, chainID: chainID, jobType: "contract:aggregate:second"}, + {kind: queue.EventBatchStarted, batchID: batchID, jobType: "contract:aggregate:first"}, + {kind: queue.EventBatchCompleted, batchID: batchID, jobType: "contract:aggregate:first"}, + } + for _, test := range tests { + var aggregate, triggeringJob *queue.Event + for index := range events { + event := &events[index] + if event.ChainID != test.chainID || event.BatchID != test.batchID { + continue + } + if event.Kind == test.kind { + aggregate = event + } + if event.Kind == queue.EventJobStarted && event.JobType == test.jobType { + triggeringJob = event + } + } + if aggregate == nil || triggeringJob == nil { + t.Errorf("missing aggregate or triggering job event %q: aggregate=%+v triggering_job=%+v", test.kind, aggregate, triggeringJob) + continue + } + if aggregate.Queue != "critical" || aggregate.Queue != triggeringJob.Queue || aggregate.JobType != triggeringJob.JobType || aggregate.JobKey == "" || aggregate.JobKey != triggeringJob.JobKey { + t.Errorf("aggregate event %q lost triggering job identity: aggregate=%+v triggering_job=%+v", test.kind, *aggregate, *triggeringJob) + } + } +} + +// TestPublicQueueContractCallbacksPreserveTriggeringJobIdentity verifies callback deliveries keep one logical identity across queue, worker, and workflow facts. +func TestPublicQueueContractCallbacksPreserveTriggeringJobIdentity(t *testing.T) { + var events []queue.Event + q, err := queue.NewSync(queue.WithObserver(queue.ObserverFunc(func(_ context.Context, event queue.Event) { + events = append(events, event) + }))) + if err != nil { + t.Fatalf("new observed sync queue: %v", err) + } + t.Cleanup(func() { + if shutdownErr := q.Shutdown(context.Background()); shutdownErr != nil { + t.Errorf("shutdown: %v", shutdownErr) + } + }) + + sourceErr := errors.New("callback source failed") + q.Register("contract:callback:source", func(context.Context, queue.Message) error { + return sourceErr + }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + _, dispatchErr := q.Chain( + queue.NewJob("contract:callback:source"). + Payload(map[string]string{"value": "callback"}). + OnQueue("critical"). + Retry(0), + ).Catch(func(context.Context, queue.ChainState, error) error { + return nil + }).Dispatch(context.Background()) + if !errors.Is(dispatchErr, sourceErr) { + t.Fatalf("dispatch error = %v, want %v", dispatchErr, sourceErr) + } + + var sourceKey, callbackJobID string + for _, event := range events { + switch event.Kind { + case queue.EventJobStarted: + if event.JobType == "contract:callback:source" { + sourceKey = event.JobKey + } + case queue.EventCallbackStarted: + callbackJobID = event.JobID + } + } + if sourceKey == "" || callbackJobID == "" { + t.Fatalf("source or callback identity missing: source_key=%q callback_job_id=%q events=%+v", sourceKey, callbackJobID, events) + } + + required := map[queue.EventKind]queue.EventLayer{ + queue.EventEnqueueAccepted: queue.EventLayerQueue, + queue.EventProcessStarted: queue.EventLayerWorker, + queue.EventCallbackStarted: queue.EventLayerWorkflow, + queue.EventProcessSucceeded: queue.EventLayerWorker, + queue.EventCallbackSucceeded: queue.EventLayerWorkflow, + } + for kind, wantLayer := range required { + var found *queue.Event + for index := range events { + if events[index].Kind == kind && events[index].JobID == callbackJobID { + found = &events[index] + break + } + } + if found == nil { + t.Errorf("callback observer did not receive %q for job %q: %+v", kind, callbackJobID, events) + continue + } + if found.Layer != wantLayer || found.Queue != "critical" || found.JobType != "contract:callback:source" || found.JobKey != sourceKey { + t.Errorf("callback event %q lost triggering identity: %+v", kind, *found) + } + } +} + +// TestPublicQueueContractLookupUsesOneNotFoundError verifies both workflow shapes share the root lookup error contract. +func TestPublicQueueContractLookupUsesOneNotFoundError(t *testing.T) { + q, err := queue.NewSync() + if err != nil { + t.Fatalf("new sync queue: %v", err) + } + if _, err := q.FindChain(context.Background(), "missing-chain"); !errors.Is(err, queue.ErrWorkflowNotFound) { + t.Fatalf("find missing chain error = %v, want ErrWorkflowNotFound", err) + } + if _, err := q.FindBatch(context.Background(), "missing-batch"); !errors.Is(err, queue.ErrWorkflowNotFound) { + t.Fatalf("find missing batch error = %v, want ErrWorkflowNotFound", err) + } +} + +// TestPublicQueueContractContextHandlesShareLifecycle verifies derived handles do not fork registration, worker, or shutdown state. +func TestPublicQueueContractContextHandlesShareLifecycle(t *testing.T) { + type contextKey string + const markerKey contextKey = "public-contract-marker" + + q, err := queue.NewWorkerpool(queue.WithWorkers(1)) + if err != nil { + t.Fatalf("new workerpool queue: %v", err) + } + derived := q.WithContext(context.WithValue(context.Background(), markerKey, "derived")) + t.Cleanup(func() { + if shutdownErr := derived.Shutdown(context.Background()); shutdownErr != nil { + t.Errorf("derived shutdown: %v", shutdownErr) + } + if shutdownErr := q.Shutdown(context.Background()); shutdownErr != nil { + t.Errorf("base shutdown: %v", shutdownErr) + } + }) + + seen := make(chan string, 1) + q.Register("contract:lifecycle", func(ctx context.Context, _ queue.Message) error { + value, _ := ctx.Value(markerKey).(string) + seen <- value + return nil + }) + if err := derived.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers through derived handle: %v", err) + } + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("idempotent start through base handle: %v", err) + } + if err := q.Ready(context.Background()); err != nil { + t.Fatalf("ready: %v", err) + } + if _, err := derived.Dispatch(queue.NewJob("contract:lifecycle").OnQueue("default")); err != nil { + t.Fatalf("dispatch through derived handle: %v", err) + } + select { + case value := <-seen: + if value != "derived" { + t.Fatalf("handler context marker = %q, want %q", value, "derived") + } + case <-time.After(2 * time.Second): + t.Fatal("workerpool handler did not run") + } + + if err := q.Shutdown(context.Background()); err != nil { + t.Fatalf("shutdown through base handle: %v", err) + } + if err := derived.Shutdown(context.Background()); err != nil { + t.Fatalf("idempotent shutdown through derived handle: %v", err) + } + if _, err := derived.Dispatch(queue.NewJob("contract:lifecycle").OnQueue("default")); !errors.Is(err, queue.ErrQueuerShuttingDown) { + t.Fatalf("dispatch after shared shutdown error = %v, want ErrQueuerShuttingDown", err) + } +} diff --git a/public_workflow_identity_test.go b/public_workflow_identity_test.go new file mode 100644 index 0000000..51cbd99 --- /dev/null +++ b/public_workflow_identity_test.go @@ -0,0 +1,148 @@ +package queue_test + +import ( + "reflect" + "testing" + + "github.com/goforj/queue" + "github.com/goforj/queue/bus" +) + +const ( + queuePackagePath = "github.com/goforj/queue" + busPackagePath = "github.com/goforj/queue/bus" +) + +// TestPublicWorkflowTypesAreOwnedByQueue pins queue as the physical owner of the canonical workflow model. +func TestPublicWorkflowTypesAreOwnedByQueue(t *testing.T) { + t.Parallel() + + types := []struct { + name string + typeOf reflect.Type + }{ + {name: "Message", typeOf: reflectedType[queue.Message]()}, + {name: "DispatchResult", typeOf: reflectedType[queue.DispatchResult]()}, + {name: "StoredJobOptions", typeOf: reflectedType[queue.StoredJobOptions]()}, + {name: "StoredJob", typeOf: reflectedType[queue.StoredJob]()}, + {name: "ChainNode", typeOf: reflectedType[queue.ChainNode]()}, + {name: "ChainRecord", typeOf: reflectedType[queue.ChainRecord]()}, + {name: "ChainState", typeOf: reflectedType[queue.ChainState]()}, + {name: "BatchJob", typeOf: reflectedType[queue.BatchJob]()}, + {name: "BatchJobOutcome", typeOf: reflectedType[queue.BatchJobOutcome]()}, + {name: "BatchRecord", typeOf: reflectedType[queue.BatchRecord]()}, + {name: "BatchState", typeOf: reflectedType[queue.BatchState]()}, + {name: "SQLStoreConfig", typeOf: reflectedType[queue.SQLStoreConfig]()}, + {name: "MiddlewareFunc", typeOf: reflectedType[queue.MiddlewareFunc]()}, + {name: "RetryPolicy", typeOf: reflectedType[queue.RetryPolicy]()}, + {name: "SkipWhen", typeOf: reflectedType[queue.SkipWhen]()}, + {name: "FailOnError", typeOf: reflectedType[queue.FailOnError]()}, + {name: "RateLimit", typeOf: reflectedType[queue.RateLimit]()}, + {name: "WithoutOverlapping", typeOf: reflectedType[queue.WithoutOverlapping]()}, + {name: "WorkflowStore", typeOf: reflectedType[queue.WorkflowStore]()}, + {name: "WorkflowOutcomeStore", typeOf: reflectedType[queue.WorkflowOutcomeStore]()}, + } + + for _, contract := range types { + if got := contract.typeOf.PkgPath(); got != queuePackagePath { + t.Errorf("queue.%s package path = %q, want %q", contract.name, got, queuePackagePath) + } + } +} + +// TestBusCompatibleAliasesResolveToQueue pins the deprecated facade to the canonical queue identities. +func TestBusCompatibleAliasesResolveToQueue(t *testing.T) { + t.Parallel() + + aliases := []struct { + name string + busType reflect.Type + queueType reflect.Type + }{ + {name: "Context", busType: reflectedType[bus.Context](), queueType: reflectedType[queue.Message]()}, + {name: "JobOptions", busType: reflectedType[bus.JobOptions](), queueType: reflectedType[queue.StoredJobOptions]()}, + {name: "DispatchResult", busType: reflectedType[bus.DispatchResult](), queueType: reflectedType[queue.DispatchResult]()}, + {name: "StoredJob", busType: reflectedType[bus.StoredJob](), queueType: reflectedType[queue.StoredJob]()}, + {name: "ChainNode", busType: reflectedType[bus.ChainNode](), queueType: reflectedType[queue.ChainNode]()}, + {name: "ChainRecord", busType: reflectedType[bus.ChainRecord](), queueType: reflectedType[queue.ChainRecord]()}, + {name: "ChainState", busType: reflectedType[bus.ChainState](), queueType: reflectedType[queue.ChainState]()}, + {name: "BatchJob", busType: reflectedType[bus.BatchJob](), queueType: reflectedType[queue.BatchJob]()}, + {name: "BatchJobOutcome", busType: reflectedType[bus.BatchJobOutcome](), queueType: reflectedType[queue.BatchJobOutcome]()}, + {name: "BatchRecord", busType: reflectedType[bus.BatchRecord](), queueType: reflectedType[queue.BatchRecord]()}, + {name: "BatchState", busType: reflectedType[bus.BatchState](), queueType: reflectedType[queue.BatchState]()}, + {name: "Store", busType: reflectedType[bus.Store](), queueType: reflectedType[queue.WorkflowStore]()}, + {name: "WorkflowOutcomeStore", busType: reflectedType[bus.WorkflowOutcomeStore](), queueType: reflectedType[queue.WorkflowOutcomeStore]()}, + {name: "SQLStoreConfig", busType: reflectedType[bus.SQLStoreConfig](), queueType: reflectedType[queue.SQLStoreConfig]()}, + {name: "Next", busType: reflectedType[bus.Next](), queueType: reflectedType[queue.Next]()}, + {name: "Middleware", busType: reflectedType[bus.Middleware](), queueType: reflectedType[queue.Middleware]()}, + {name: "MiddlewareFunc", busType: reflectedType[bus.MiddlewareFunc](), queueType: reflectedType[queue.MiddlewareFunc]()}, + {name: "RetryPolicy", busType: reflectedType[bus.RetryPolicy](), queueType: reflectedType[queue.RetryPolicy]()}, + {name: "SkipWhen", busType: reflectedType[bus.SkipWhen](), queueType: reflectedType[queue.SkipWhen]()}, + {name: "FailOnError", busType: reflectedType[bus.FailOnError](), queueType: reflectedType[queue.FailOnError]()}, + {name: "RateLimiter", busType: reflectedType[bus.RateLimiter](), queueType: reflectedType[queue.RateLimiter]()}, + {name: "RateLimit", busType: reflectedType[bus.RateLimit](), queueType: reflectedType[queue.RateLimit]()}, + {name: "Lock", busType: reflectedType[bus.Lock](), queueType: reflectedType[queue.Lock]()}, + {name: "Locker", busType: reflectedType[bus.Locker](), queueType: reflectedType[queue.Locker]()}, + {name: "WithoutOverlapping", busType: reflectedType[bus.WithoutOverlapping](), queueType: reflectedType[queue.WithoutOverlapping]()}, + } + + for _, contract := range aliases { + if contract.busType != contract.queueType { + t.Errorf("bus.%s type = %v, want queue identity %v", contract.name, contract.busType, contract.queueType) + continue + } + if got := contract.busType.PkgPath(); got != queuePackagePath { + t.Errorf("bus.%s package path = %q, want canonical queue path %q", contract.name, got, queuePackagePath) + } + } +} + +// TestLegacyBusTypesRemainOwnedByBus pins the intentionally distinct compatibility contracts to the bus package. +func TestLegacyBusTypesRemainOwnedByBus(t *testing.T) { + t.Parallel() + + contracts := []struct { + name string + typeOf reflect.Type + }{ + {name: "Bus", typeOf: reflectedType[bus.Bus]()}, + {name: "BatchSpec", typeOf: reflectedType[bus.BatchSpec]()}, + {name: "Fake", typeOf: reflectedType[bus.Fake]()}, + {name: "Handler", typeOf: reflectedType[bus.Handler]()}, + {name: "Observer", typeOf: reflectedType[bus.Observer]()}, + {name: "ObserverFunc", typeOf: reflectedType[bus.ObserverFunc]()}, + {name: "Option", typeOf: reflectedType[bus.Option]()}, + } + + for _, contract := range contracts { + if got := contract.typeOf.PkgPath(); got != busPackagePath { + t.Errorf("bus.%s package path = %q, want %q", contract.name, got, busPackagePath) + } + } + + types := []struct { + name string + busType reflect.Type + queueType reflect.Type + }{ + {name: "Job", busType: reflectedType[bus.Job](), queueType: reflectedType[queue.Job]()}, + {name: "Event", busType: reflectedType[bus.Event](), queueType: reflectedType[queue.Event]()}, + {name: "EventKind", busType: reflectedType[bus.EventKind](), queueType: reflectedType[queue.EventKind]()}, + {name: "ChainBuilder", busType: reflectedType[bus.ChainBuilder](), queueType: reflectedType[queue.ChainBuilder]()}, + {name: "BatchBuilder", busType: reflectedType[bus.BatchBuilder](), queueType: reflectedType[queue.BatchBuilder]()}, + } + + for _, contract := range types { + if got := contract.busType.PkgPath(); got != busPackagePath { + t.Errorf("bus.%s package path = %q, want %q", contract.name, got, busPackagePath) + } + if contract.busType == contract.queueType { + t.Errorf("bus.%s unexpectedly shares queue identity %v", contract.name, contract.queueType) + } + } +} + +// reflectedType returns the reflection identity for T, including interface types. +func reflectedType[T any]() reflect.Type { + return reflect.TypeOf((*T)(nil)).Elem() +} diff --git a/queue.go b/queue.go index 2d29881..b1594c9 100644 --- a/queue.go +++ b/queue.go @@ -45,6 +45,9 @@ type queueRuntime interface { // @group Driver Integration Ready(ctx context.Context) error + // physicalQueueNameOrDefault resolves the effective backend queue name used in canonical events. + physicalQueueNameOrDefault(queueName string) string + // setHandlerContextDecorator decorates handler execution context at registration time. setHandlerContextDecorator(func(context.Context) context.Context) } @@ -68,7 +71,9 @@ func (c WorkerpoolConfig) normalize() WorkerpoolConfig { // Config configures queue creation for New (and advanced driver/runtime interop). // @group Config type Config struct { - Driver Driver + Driver Driver + // Observer is a compatibility attachment path for queue lifecycle events. + // Deprecated: use WithObserver so all event layers share one constructor option. Observer Observer Logger Logger @@ -85,6 +90,7 @@ type runtimeQueueBackend interface { queueBackend Register(jobType string, handler Handler) StartWorkers(ctx context.Context) error + DrainWorkers(ctx context.Context) error } func newSyncQueue() queueBackend { @@ -124,6 +130,7 @@ func New(cfg Config, opts ...Option) (*Queue, error) { func newRuntime(cfg Config) (queueRuntime, error) { cfg = cfg.normalize() + cfg.Observer = ensureObserverSink(cfg.Observer) var q queueBackend var err error @@ -161,14 +168,20 @@ func newRuntime(cfg Config) (queueRuntime, error) { } if runtime != nil { return &nativeQueueRuntime{ - common: common, - runtime: runtime, - registered: make(map[string]Handler), + common: common, + runtime: runtime, + nativeQueueRuntimeState: &nativeQueueRuntimeState{ + registered: make(map[string]Handler), + continuation: busruntime.NewContinuationScope(), + }, }, nil } return &externalQueueRuntime{ - common: common, - registered: make(map[string]Handler), + common: common, + externalQueueRuntimeState: &externalQueueRuntimeState{ + registered: make(map[string]Handler), + continuation: busruntime.NewContinuationScope(), + }, }, nil } @@ -190,22 +203,135 @@ type queueCommon struct { type nativeQueueRuntime struct { common *queueCommon runtime runtimeQueueBackend + *nativeQueueRuntimeState +} - mu sync.Mutex - registered map[string]Handler - started bool - workers int +// nativeQueueRuntimeState stays shared by context-bound handles because worker registration and lifecycle belong to the runtime, not an individual dispatch context. +type nativeQueueRuntimeState struct { + mu sync.Mutex + registered map[string]Handler + handlerSlots map[string]*runtimeHandlerSlot + runtimeRegistrations map[string]struct{} + started bool + draining bool + closed bool + start *runtimeStartAttempt + shutdown *runtimeShutdownAttempt + operations runtimeOperationState + continuation *busruntime.ContinuationScope + workers int } type externalQueueRuntime struct { - common *queueCommon + common *queueCommon + newWorker driverWorkerFactory + *externalQueueRuntimeState +} + +// externalQueueRuntimeState keeps the constructed worker and lifecycle state synchronized across derived queue handles. +type externalQueueRuntimeState struct { + mu sync.Mutex + registered map[string]Handler + handlerSlots map[string]*runtimeHandlerSlot + worker runtimeWorkerBackend + workerRegistrations map[string]struct{} + started bool + draining bool + closed bool + start *runtimeStartAttempt + shutdown *runtimeShutdownAttempt + operations runtimeOperationState + continuation *busruntime.ContinuationScope + workers int +} - mu sync.Mutex - registered map[string]Handler - worker runtimeWorkerBackend - started bool - workers int - newWorker driverWorkerFactory +type runtimeOperationState struct { + active int + idle chan struct{} +} + +// acquire reserves backend resources while the owning lifecycle mutex is held. +func (s *runtimeOperationState) acquire() { + if s.active == 0 { + s.idle = make(chan struct{}) + } + s.active++ +} + +// release returns true when the final operation completed and an idle waiter should be released. +func (s *runtimeOperationState) release() bool { + s.active-- + return s.active == 0 && s.idle != nil +} + +// markIdle closes the current idle generation after release identifies the final operation. +func (s *runtimeOperationState) markIdle() { + close(s.idle) + s.idle = nil +} + +type runtimeShutdownAttempt struct { + done chan struct{} + err error +} + +type runtimeStartAttempt struct { + done chan struct{} + err error +} + +type runtimeHandlerSlot struct { + mu sync.RWMutex + handler Handler +} + +// replace changes the application handler behind one stable backend registration. +func (s *runtimeHandlerSlot) replace(handler Handler) { + s.mu.Lock() + s.handler = handler + s.mu.Unlock() +} + +// invoke resolves the latest handler without holding the slot lock during application execution. +func (s *runtimeHandlerSlot) invoke(ctx context.Context, job Job) error { + s.mu.RLock() + handler := s.handler + s.mu.RUnlock() + return handler(ctx, job) +} + +// updateRuntimeHandlerSlot creates or updates the stable target used for one non-nil job registration. +func updateRuntimeHandlerSlot(slots map[string]*runtimeHandlerSlot, jobType string, handler Handler) (map[string]*runtimeHandlerSlot, *runtimeHandlerSlot) { + if handler == nil { + return slots, nil + } + if slots == nil { + slots = make(map[string]*runtimeHandlerSlot) + } + slot := slots[jobType] + if slot == nil { + slot = &runtimeHandlerSlot{} + slots[jobType] = slot + } + slot.replace(handler) + return slots, slot +} + +// installRuntimeHandler installs one stable trampoline per non-nil job type on a backend. +func installRuntimeHandler(backend interface{ Register(string, Handler) }, common *queueCommon, registrations map[string]struct{}, jobType string, handler Handler, slot *runtimeHandlerSlot) map[string]struct{} { + if handler == nil { + backend.Register(jobType, nil) + return registrations + } + if _, installed := registrations[jobType]; installed { + return registrations + } + backend.Register(jobType, common.wrapRegisteredHandler(jobType, slot.invoke)) + if registrations == nil { + registrations = make(map[string]struct{}) + } + registrations[jobType] = struct{}{} + return registrations } type runtimeWorkerBackend interface { @@ -229,6 +355,27 @@ func (q *queueCommon) context() context.Context { return q.ctx } +// addObserver composes observers at construction time so queue and workflow layers publish to the same application sink. +func (q *queueCommon) addObserver(observer Observer) { + if q == nil || observer == nil { + return + } + q.cfg.Observer = addObserverToSink(q.cfg.Observer, observer) + if observed, ok := q.inner.(*observedQueue); ok { + observed.observer = q.cfg.Observer + return + } + q.inner = newObservedQueue(q.inner, q.driver, q.cfg.Observer) +} + +// observer returns the composed application observer shared by execution and workflow adapters. +func (q *queueCommon) observer() Observer { + if q == nil || !observerHasRecipients(q.cfg.Observer) { + return nil + } + return q.cfg.Observer +} + func (q *queueCommon) WithContext(ctx context.Context) *queueCommon { if q == nil { return nil @@ -251,9 +398,12 @@ func (q *queueCommon) Dispatch(job any) error { return err } dispatchJob = q.physicalJob(dispatchJob) - return q.inner.Dispatch(q.context(), dispatchJob) + ctx, _ := newDispatchAcceptance(q.context()) + return q.inner.Dispatch(ctx, dispatchJob) } +// physicalJob namespaces explicit targets while preserving the current +// backend-specific contract for jobs that omit a queue. func (q *queueCommon) physicalJob(job Job) Job { if job.options.queueName == "" { return job @@ -269,6 +419,7 @@ func (q *queueCommon) physicalQueueName(queueName string) string { return PhysicalQueueName(q.cfg.DefaultQueue, queueName) } +// physicalQueueNameOrDefault resolves the configured default and namespace before a queue name reaches the backend. func (q *queueCommon) physicalQueueNameOrDefault(queueName string) string { queueName = strings.TrimSpace(queueName) if queueName == "" && q != nil { @@ -319,8 +470,26 @@ func queueNamePrefix(defaultQueue string) string { return prefix + "_" } -func (q *nativeQueueRuntime) Driver() Driver { return q.common.Driver() } -func (q *nativeQueueRuntime) Dispatch(job any) error { return q.common.Dispatch(job) } +// Driver returns the native runtime's configured backend identifier. +func (q *nativeQueueRuntime) Driver() Driver { return q.common.Driver() } + +// physicalQueueNameOrDefault keeps canonical event labels aligned with native backend queue names. +func (q *nativeQueueRuntime) physicalQueueNameOrDefault(queueName string) string { + if q == nil || q.common == nil { + return PhysicalQueueName("default", queueName) + } + return q.common.physicalQueueNameOrDefault(queueName) +} + +// Dispatch rejects new application work once native runtime draining begins. +func (q *nativeQueueRuntime) Dispatch(job any) error { + release, err := q.acquireOperation(q.common.context(), true) + if err != nil { + return err + } + defer release() + return q.common.Dispatch(job) +} func (q *nativeQueueRuntime) WithContext(ctx context.Context) queueRuntime { if q == nil { return nil @@ -330,8 +499,26 @@ func (q *nativeQueueRuntime) WithContext(ctx context.Context) queueRuntime { return &clone } -func (q *externalQueueRuntime) Driver() Driver { return q.common.Driver() } -func (q *externalQueueRuntime) Dispatch(job any) error { return q.common.Dispatch(job) } +// Driver returns the external runtime's configured backend identifier. +func (q *externalQueueRuntime) Driver() Driver { return q.common.Driver() } + +// physicalQueueNameOrDefault keeps canonical event labels aligned with external backend queue names. +func (q *externalQueueRuntime) physicalQueueNameOrDefault(queueName string) string { + if q == nil || q.common == nil { + return PhysicalQueueName("default", queueName) + } + return q.common.physicalQueueNameOrDefault(queueName) +} + +// Dispatch rejects new application work once external runtime draining begins. +func (q *externalQueueRuntime) Dispatch(job any) error { + release, err := q.acquireOperation(q.common.context(), true) + if err != nil { + return err + } + defer release() + return q.common.Dispatch(job) +} func (q *externalQueueRuntime) WithContext(ctx context.Context) queueRuntime { if q == nil { return nil @@ -360,8 +547,11 @@ func (q *nativeQueueRuntime) BusRegister(jobType string, handler busruntime.Hand q.Register(jobType, nil) return } + scope := q.continuationScope() q.Register(jobType, func(ctx context.Context, job Job) error { - return handler(ctx, job) + handlerCtx, release := withBusDeliveryContext(ctx, job, scope) + defer release() + return handler(handlerCtx, job) }) } @@ -370,128 +560,303 @@ func (q *externalQueueRuntime) BusRegister(jobType string, handler busruntime.Ha q.Register(jobType, nil) return } + scope := q.continuationScope() q.Register(jobType, func(ctx context.Context, job Job) error { - return handler(ctx, job) + handlerCtx, release := withBusDeliveryContext(ctx, job, scope) + defer release() + return handler(handlerCtx, job) }) } +// withBusDeliveryContext attaches physical attempt and correlation metadata to +// one invocation while keeping both channels out of the application payload. +func withBusDeliveryContext(ctx context.Context, job Job, scope *busruntime.ContinuationScope) (context.Context, func()) { + if ctx == nil { + ctx = context.Background() + } + opts := job.jobOptions() + ctx, release := scope.Permit(ctx) + metadata := DriverMetadata(job) + // Every physical invocation shadows parent metadata so nested legacy or + // low-level jobs cannot inherit correlation from the job that dispatched them. + ctx = busruntime.WithDeliveryMetadata(ctx, metadata) + return busruntime.WithDeliveryAttempt(ctx, busruntime.DeliveryAttempt{ + Number: opts.attempt, + MaxRetry: optionInt(opts.maxRetry), + }), release +} + func (q *nativeQueueRuntime) BusDispatch(ctx context.Context, jobType string, payload []byte, opts busruntime.JobOptions) error { + release, err := q.acquireOperation(ctx, true) + if err != nil { + return err + } + defer release() return q.common.dispatchBusJob(ctx, jobType, payload, opts) } func (q *externalQueueRuntime) BusDispatch(ctx context.Context, jobType string, payload []byte, opts busruntime.JobOptions) error { + release, err := q.acquireOperation(ctx, true) + if err != nil { + return err + } + defer release() return q.common.dispatchBusJob(ctx, jobType, payload, opts) } +// BusDispatchDirect submits an ordinary application job without a workflow envelope. +func (q *nativeQueueRuntime) BusDispatchDirect(ctx context.Context, jobType string, payload []byte, metadata busruntime.DeliveryMetadata, opts busruntime.JobOptions) error { + release, err := q.acquireOperation(ctx, true) + if err != nil { + return err + } + defer release() + return q.common.dispatchDirectJob(ctx, jobType, payload, metadata, opts) +} + +// BusDispatchDirect submits an ordinary application job without a workflow envelope. +func (q *externalQueueRuntime) BusDispatchDirect(ctx context.Context, jobType string, payload []byte, metadata busruntime.DeliveryMetadata, opts busruntime.JobOptions) error { + release, err := q.acquireOperation(ctx, true) + if err != nil { + return err + } + defer release() + return q.common.dispatchDirectJob(ctx, jobType, payload, metadata, opts) +} + +// acquireOperation leases native backend resources through one complete operation. +func (q *nativeQueueRuntime) acquireOperation(ctx context.Context, allowContinuation bool) (func(), error) { + q.mu.Lock() + scope := q.continuationScopeLocked() + if q.closed || (q.draining && (!allowContinuation || !scope.Owns(ctx))) { + q.mu.Unlock() + return nil, ErrQueuerShuttingDown + } + q.operations.acquire() + q.mu.Unlock() + return q.releaseOperation, nil +} + +// releaseOperation ends one native lease and wakes a waiting shutdown when the backend becomes idle. +func (q *nativeQueueRuntime) releaseOperation() { + q.mu.Lock() + if q.operations.release() { + q.operations.markIdle() + } + q.mu.Unlock() +} + +// acquireOperation leases external producer resources through one complete operation. +func (q *externalQueueRuntime) acquireOperation(ctx context.Context, allowContinuation bool) (func(), error) { + q.mu.Lock() + scope := q.continuationScopeLocked() + if q.closed || (q.draining && (!allowContinuation || !scope.Owns(ctx))) { + q.mu.Unlock() + return nil, ErrQueuerShuttingDown + } + q.operations.acquire() + q.mu.Unlock() + return q.releaseOperation, nil +} + +// releaseOperation ends one external lease and wakes a waiting shutdown when the producer becomes idle. +func (q *externalQueueRuntime) releaseOperation() { + q.mu.Lock() + if q.operations.release() { + q.operations.markIdle() + } + q.mu.Unlock() +} + +// continuationScope returns the native runtime's stable permission owner. +func (q *nativeQueueRuntime) continuationScope() *busruntime.ContinuationScope { + q.mu.Lock() + defer q.mu.Unlock() + return q.continuationScopeLocked() +} + +// continuationScopeLocked lazily initializes test-constructed native states while the lifecycle mutex is held. +func (q *nativeQueueRuntime) continuationScopeLocked() *busruntime.ContinuationScope { + if q.continuation == nil { + q.continuation = busruntime.NewContinuationScope() + } + return q.continuation +} + +// continuationScope returns the external runtime's stable permission owner. +func (q *externalQueueRuntime) continuationScope() *busruntime.ContinuationScope { + q.mu.Lock() + defer q.mu.Unlock() + return q.continuationScopeLocked() +} + +// continuationScopeLocked lazily initializes test-constructed external states while the lifecycle mutex is held. +func (q *externalQueueRuntime) continuationScopeLocked() *busruntime.ContinuationScope { + if q.continuation == nil { + q.continuation = busruntime.NewContinuationScope() + } + return q.continuation +} + +// Register linearizes logical and physical state so an activating backend cannot consume a newly registered type without its handler. func (q *nativeQueueRuntime) Register(jobType string, handler Handler) { + if jobType == "" || handler == nil { + return + } q.mu.Lock() + defer q.mu.Unlock() if q.registered == nil { q.registered = make(map[string]Handler) } q.registered[jobType] = handler - started := q.started - q.mu.Unlock() - - if started { - q.runtime.Register(jobType, q.common.wrapRegisteredHandler(jobType, handler)) + var slot *runtimeHandlerSlot + q.handlerSlots, slot = updateRuntimeHandlerSlot(q.handlerSlots, jobType, handler) + if !q.draining && (q.start != nil || q.started) { + q.runtimeRegistrations = installRuntimeHandler(q.runtime, q.common, q.runtimeRegistrations, jobType, handler, slot) } } +// Register linearizes logical and physical state once the external worker generation has been published for activation. func (q *externalQueueRuntime) Register(jobType string, handler Handler) { + if jobType == "" || handler == nil { + return + } q.mu.Lock() + defer q.mu.Unlock() if q.registered == nil { q.registered = make(map[string]Handler) } q.registered[jobType] = handler - w := q.worker - started := q.started - q.mu.Unlock() + var slot *runtimeHandlerSlot + q.handlerSlots, slot = updateRuntimeHandlerSlot(q.handlerSlots, jobType, handler) + if !q.draining && q.worker != nil && (q.start != nil || q.started) { + q.workerRegistrations = installRuntimeHandler(q.worker, q.common, q.workerRegistrationsLocked(), jobType, handler, slot) + } +} - if started && w != nil { - w.Register(jobType, q.common.wrapRegisteredHandler(jobType, handler)) +// workerRegistrationsLocked returns the handler types already installed on the retained external worker. +func (q *externalQueueRuntime) workerRegistrationsLocked() map[string]struct{} { + if q.workerRegistrations == nil { + q.workerRegistrations = make(map[string]struct{}) } + return q.workerRegistrations } +// StartWorkers installs the current handler generation before activating the backend and serializes concurrent lifecycle calls. func (q *nativeQueueRuntime) StartWorkers(ctx context.Context) error { if ctx == nil { ctx = context.Background() } q.mu.Lock() + if q.closed || q.draining { + q.mu.Unlock() + return ErrQueuerShuttingDown + } if q.started { q.mu.Unlock() return nil } - registered := make(map[string]Handler, len(q.registered)) + if q.start != nil { + attempt := q.start + q.mu.Unlock() + return waitForRuntimeStart(ctx, attempt) + } + attempt := &runtimeStartAttempt{done: make(chan struct{})} + q.start = attempt for jobType, handler := range q.registered { - registered[jobType] = handler + var slot *runtimeHandlerSlot + q.handlerSlots, slot = updateRuntimeHandlerSlot(q.handlerSlots, jobType, handler) + q.runtimeRegistrations = installRuntimeHandler(q.runtime, q.common, q.runtimeRegistrations, jobType, handler, slot) } q.mu.Unlock() - for jobType, handler := range registered { - q.runtime.Register(jobType, q.common.wrapRegisteredHandler(jobType, handler)) - } - if err := q.runtime.StartWorkers(ctx); err != nil { - return err - } + err := q.runtime.StartWorkers(ctx) q.mu.Lock() - q.started = true + if err == nil { + q.started = true + } + attempt.err = err + q.start = nil + close(attempt.done) q.mu.Unlock() - return nil + return err } +// StartWorkers publishes and catches up a worker before activation so registrations cannot complete against a stale startup snapshot. func (q *externalQueueRuntime) StartWorkers(ctx context.Context) error { if ctx == nil { ctx = context.Background() } q.mu.Lock() + if q.closed || q.draining { + q.mu.Unlock() + return ErrQueuerShuttingDown + } if q.started { q.mu.Unlock() return nil } - workers := q.workers - registered := make(map[string]Handler, len(q.registered)) - for jobType, handler := range q.registered { - registered[jobType] = handler + if q.start != nil { + attempt := q.start + q.mu.Unlock() + return waitForRuntimeStart(ctx, attempt) } + attempt := &runtimeStartAttempt{done: make(chan struct{})} + q.start = attempt + w := q.worker + workers := q.workers q.mu.Unlock() - var ( - w runtimeWorkerBackend - err error - ) - if q.newWorker != nil { - driverWorker, e := q.newWorker(defaultWorkerCount(workers)) - if e != nil { - return e - } - w = driverWorkerBackendAdapter{driverWorker} - } else { - w, err = newExternalWorker(q.common.cfg, workers) - if err != nil { - return err + var err error + if w == nil { + if q.newWorker != nil { + driverWorker, e := q.newWorker(defaultWorkerCount(workers)) + if e != nil { + err = e + } else { + w = driverWorkerBackendAdapter{driverWorker} + } + } else { + w, err = newExternalWorker(q.common.cfg, workers) } } - if setter, ok := w.(runtimeWorkerContextDecoratorSetter); ok { - setter.SetHandlerContextDecorator(q.common.handlerContextDecorator) + if err == nil { + q.mu.Lock() + q.worker = w + if setter, ok := w.(runtimeWorkerContextDecoratorSetter); ok { + setter.SetHandlerContextDecorator(q.common.handlerContextDecorator) + } + for jobType, handler := range q.registered { + var slot *runtimeHandlerSlot + q.handlerSlots, slot = updateRuntimeHandlerSlot(q.handlerSlots, jobType, handler) + q.workerRegistrations = installRuntimeHandler(w, q.common, q.workerRegistrationsLocked(), jobType, handler, slot) + } + q.mu.Unlock() + err = w.StartWorkers(ctx) } - for jobType, handler := range registered { - w.Register(jobType, q.common.wrapRegisteredHandler(jobType, handler)) + q.mu.Lock() + if w != nil { + // A partially started worker remains owned so Shutdown can finish cleanup instead of leaking factory resources. + q.worker = w } - if err := w.StartWorkers(ctx); err != nil { - return err + if err == nil { + q.started = true } - q.mu.Lock() - q.worker = w - q.started = true + attempt.err = err + q.start = nil + close(attempt.done) q.mu.Unlock() - return nil + return err } func (q *nativeQueueRuntime) Workers(count int) queueRuntime { q.mu.Lock() defer q.mu.Unlock() - if !q.started && count > 0 { + if !q.started && !q.draining && !q.closed && q.start == nil && count > 0 { q.workers = count + if setter, ok := q.runtime.(interface{ setWorkers(int) }); ok { + setter.setWorkers(count) + } } return q } @@ -499,46 +864,177 @@ func (q *nativeQueueRuntime) Workers(count int) queueRuntime { func (q *externalQueueRuntime) Workers(count int) queueRuntime { q.mu.Lock() defer q.mu.Unlock() - if !q.started && count > 0 { + if !q.started && !q.draining && !q.closed && q.start == nil && count > 0 { q.workers = count } return q } +// Shutdown retains native runtime state until cleanup succeeds so timed-out drains remain retryable. func (q *nativeQueueRuntime) Shutdown(ctx context.Context) error { if ctx == nil { ctx = context.Background() } q.mu.Lock() - wasStarted := q.started - q.started = false + if q.start != nil { + q.draining = true + attempt := q.start + q.mu.Unlock() + if err := waitForRuntimeStartCompletion(ctx, attempt); err != nil { + return err + } + return q.Shutdown(ctx) + } + if q.shutdown != nil { + attempt := q.shutdown + q.mu.Unlock() + return waitForRuntimeShutdown(ctx, attempt) + } + if q.closed { + q.mu.Unlock() + return nil + } + q.draining = true + attempt := &runtimeShutdownAttempt{done: make(chan struct{})} + q.shutdown = attempt + idle := q.operations.idle q.mu.Unlock() - if wasStarted { - return q.runtime.Shutdown(ctx) + err := waitForRuntimeOperations(ctx, idle) + if err == nil { + err = q.runtime.DrainWorkers(ctx) } - return nil + if err == nil { + // Worker drain expires every handler-issued continuation permit. A second + // operation snapshot is therefore stable and must finish before resources close. + q.mu.Lock() + idle = q.operations.idle + q.mu.Unlock() + err = waitForRuntimeOperations(ctx, idle) + } + if err == nil { + err = q.common.inner.Shutdown(ctx) + } + q.mu.Lock() + attempt.err = err + q.shutdown = nil + if err == nil { + q.started = false + q.draining = false + q.closed = true + } + close(attempt.done) + q.mu.Unlock() + return err } +// Shutdown drains the worker before producer resources and retains both until every cleanup succeeds. func (q *externalQueueRuntime) Shutdown(ctx context.Context) error { if ctx == nil { ctx = context.Background() } q.mu.Lock() + if q.start != nil { + q.draining = true + attempt := q.start + q.mu.Unlock() + if err := waitForRuntimeStartCompletion(ctx, attempt); err != nil { + return err + } + return q.Shutdown(ctx) + } + if q.shutdown != nil { + attempt := q.shutdown + q.mu.Unlock() + return waitForRuntimeShutdown(ctx, attempt) + } + if q.closed { + q.mu.Unlock() + return nil + } w := q.worker - wasStarted := q.started - q.started = false - q.worker = nil + q.draining = true + attempt := &runtimeShutdownAttempt{done: make(chan struct{})} + q.shutdown = attempt + idle := q.operations.idle q.mu.Unlock() - if wasStarted { - if w != nil { - if err := w.Shutdown(ctx); err != nil { - return err - } + err := waitForRuntimeOperations(ctx, idle) + if w != nil { + if err == nil { + err = w.Shutdown(ctx) } + if err == nil { + q.mu.Lock() + q.worker = nil + q.workerRegistrations = nil + q.started = false + q.mu.Unlock() + } + } + if err == nil { + // A handler may admit a descendant after the initial snapshot. Once worker drain returns, its scoped permit has expired, so this generation is stable. + q.mu.Lock() + idle = q.operations.idle + q.mu.Unlock() + err = waitForRuntimeOperations(ctx, idle) + } + if err == nil { + err = q.common.inner.Shutdown(ctx) + } + q.mu.Lock() + attempt.err = err + q.shutdown = nil + if err == nil { + q.draining = false + q.closed = true + } + close(attempt.done) + q.mu.Unlock() + return err +} + +// waitForRuntimeOperations prevents resource cleanup from overtaking an operation that already passed the lifecycle gate. +func waitForRuntimeOperations(ctx context.Context, idle <-chan struct{}) error { + if idle == nil { + return nil + } + select { + case <-idle: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// waitForRuntimeShutdown lets concurrent callers share one cleanup attempt while honoring their own deadline. +func waitForRuntimeShutdown(ctx context.Context, attempt *runtimeShutdownAttempt) error { + select { + case <-attempt.done: + return attempt.err + case <-ctx.Done(): + return ctx.Err() + } +} + +// waitForRuntimeStart lets concurrent callers share one startup attempt while honoring their own deadline. +func waitForRuntimeStart(ctx context.Context, attempt *runtimeStartAttempt) error { + select { + case <-attempt.done: + return attempt.err + case <-ctx.Done(): + return ctx.Err() + } +} + +// waitForRuntimeStartCompletion lets shutdown wait for ownership of any worker that startup creates. +func waitForRuntimeStartCompletion(ctx context.Context, attempt *runtimeStartAttempt) error { + select { + case <-attempt.done: + return nil + case <-ctx.Done(): + return ctx.Err() } - return q.common.inner.Shutdown(ctx) } func (q *queueCommon) Pause(ctx context.Context, queueName string) error { @@ -625,43 +1121,105 @@ func (q *queueCommon) Ready(ctx context.Context) error { } func (q *nativeQueueRuntime) Pause(ctx context.Context, queueName string) error { + release, err := q.acquireOperation(ctx, false) + if err != nil { + return err + } + defer release() return q.common.Pause(ctx, queueName) } func (q *nativeQueueRuntime) Resume(ctx context.Context, queueName string) error { + release, err := q.acquireOperation(ctx, false) + if err != nil { + return err + } + defer release() return q.common.Resume(ctx, queueName) } func (q *nativeQueueRuntime) Stats(ctx context.Context) (StatsSnapshot, error) { + release, err := q.acquireOperation(ctx, false) + if err != nil { + return StatsSnapshot{}, err + } + defer release() return q.common.Stats(ctx) } func (q *nativeQueueRuntime) Ready(ctx context.Context) error { + release, err := q.acquireOperation(ctx, false) + if err != nil { + return err + } + defer release() return q.common.Ready(ctx) } func (q *externalQueueRuntime) Pause(ctx context.Context, queueName string) error { + release, err := q.acquireOperation(ctx, false) + if err != nil { + return err + } + defer release() return q.common.Pause(ctx, queueName) } func (q *externalQueueRuntime) Resume(ctx context.Context, queueName string) error { + release, err := q.acquireOperation(ctx, false) + if err != nil { + return err + } + defer release() return q.common.Resume(ctx, queueName) } func (q *externalQueueRuntime) Stats(ctx context.Context) (StatsSnapshot, error) { + release, err := q.acquireOperation(ctx, false) + if err != nil { + return StatsSnapshot{}, err + } + defer release() return q.common.Stats(ctx) } func (q *externalQueueRuntime) Ready(ctx context.Context) error { + release, err := q.acquireOperation(ctx, false) + if err != nil { + return err + } + defer release() return q.common.Ready(ctx) } +// wrapRegisteredHandler keeps each backend's context decoration and process +// observation at a single execution boundary. func (q *queueCommon) wrapRegisteredHandler(jobType string, handler Handler) Handler { - if handler == nil || q.cfg.Observer == nil { + if handler == nil { return handler } // Redis worker emits process lifecycle events natively. - // Skip shared handler wrapping to avoid duplicate process_* events. + // Skip shared handler wrapping and decoration to avoid duplicate process_* events + // and context decoration. if q.cfg.Driver == DriverRedis { return handler } + if !observerHasRecipients(q.cfg.Observer) { + return wrapHandlerContext(q.handlerContextDecorator, handler) + } return wrapObservedHandler(q.cfg.Observer, q.cfg.Driver, "", jobType, q.handlerContextDecorator, handler) } +// wrapHandlerContext applies optional execution context decoration while +// preserving the original context when the decorator returns nil. +func wrapHandlerContext(decorator func(context.Context) context.Context, handler Handler) Handler { + if decorator == nil || handler == nil { + return handler + } + return func(ctx context.Context, job Job) error { + if decorated := decorator(ctx); decorated != nil { + ctx = busruntime.PreserveDeliveryContext(ctx, decorated) + } + return handler(ctx, job) + } +} + +// dispatchBusJob preserves workflow policy and logical identity while adapting onto the canonical root job. func (q *queueCommon) dispatchBusJob(ctx context.Context, jobType string, payload []byte, opts busruntime.JobOptions) error { + ctx, acceptance := newDispatchAcceptance(ctx) job := NewJob(jobType).Payload(payload) if opts.Queue != "" { job = job.OnQueue(opts.Queue) @@ -672,16 +1230,58 @@ func (q *queueCommon) dispatchBusJob(ctx context.Context, jobType string, payloa if opts.Timeout > 0 { job = job.Timeout(opts.Timeout) } - if opts.Retry > 0 { - job = job.Retry(opts.Retry) + // Workflow policy always owns the retry budget; omitting zero lets several backends invent a different default. + job = job.Retry(opts.Retry) + if opts.Backoff > 0 { + job = job.Backoff(opts.Backoff) } + if opts.UniqueFor > 0 { + logical := resolveLogicalJob(jobType, payload) + job = job.UniqueFor(opts.UniqueFor).withLogicalIdentity(logical.jobType, logical.payload) + } + err := q.inner.Dispatch(ctx, q.physicalJob(job)) + if err == nil { + acceptance.markAccepted() + return nil + } + if acceptance.isAccepted() { + return acceptedExecutionError{cause: err} + } + return err +} + +// dispatchDirectJob preserves direct application bytes while attaching +// correlation through the driver metadata channel instead of a workflow envelope. +func (q *queueCommon) dispatchDirectJob(ctx context.Context, jobType string, payload []byte, metadata busruntime.DeliveryMetadata, opts busruntime.JobOptions) error { + ctx, acceptance := newDispatchAcceptance(ctx) + job := NewJob(jobType).Payload(payload) + if opts.Queue != "" { + job = job.OnQueue(opts.Queue) + } + if opts.Delay > 0 { + job = job.Delay(opts.Delay) + } + if opts.Timeout > 0 { + job = job.Timeout(opts.Timeout) + } + // Direct workflow policy still owns an explicit zero retry budget. + job = job.Retry(opts.Retry) if opts.Backoff > 0 { job = job.Backoff(opts.Backoff) } if opts.UniqueFor > 0 { job = job.UniqueFor(opts.UniqueFor) } - return q.inner.Dispatch(ctx, q.physicalJob(job)) + job = DriverWithMetadata(job, metadata) + err := q.inner.Dispatch(ctx, q.physicalJob(job)) + if err == nil { + acceptance.markAccepted() + return nil + } + if acceptance.isAccepted() { + return acceptedExecutionError{cause: err} + } + return err } func newExternalWorker(cfg Config, concurrency int) (runtimeWorkerBackend, error) { @@ -699,6 +1299,11 @@ type driverRuntimeQueueBackendAdapter struct { driverRuntimeQueueBackend } +// DrainWorkers forwards the native driver's worker-drain lifecycle phase. +func (a driverRuntimeQueueBackendAdapter) DrainWorkers(ctx context.Context) error { + return a.driverRuntimeQueueBackend.DrainWorkers(ctx) +} + type driverWorkerBackendAdapter struct { driverWorkerBackend } @@ -889,7 +1494,15 @@ func optionalDriverMovedError(driver Driver) error { } } +// jobFromAny applies this runtime's default queue while sharing the canonical +// value-to-job conversion with the public fake. func (q *queueCommon) jobFromAny(job any) (Job, error) { + return normalizeDispatchJob(job, q.cfg.DefaultQueue) +} + +// normalizeDispatchJob keeps typed-value inference and default queue selection +// identical without changing when production backends validate acceptance. +func normalizeDispatchJob(job any, defaultQueue string) (Job, error) { if job, ok := job.(Job); ok { if job.Type == "" { return Job{}, fmt.Errorf("dispatch job type is required") @@ -912,9 +1525,11 @@ func (q *queueCommon) jobFromAny(job any) (Job, error) { if err != nil { return Job{}, fmt.Errorf("marshal dispatch job: %w", err) } - return NewJob(jobType).Payload(payload).OnQueue(q.cfg.DefaultQueue), nil + return NewJob(jobType).Payload(payload).OnQueue(defaultQueue), nil } +// jobTypeFromValue limits implicit names to declared Go types so anonymous +// payload shapes cannot accidentally become unstable queue contracts. func jobTypeFromValue(v any) string { t := reflect.TypeOf(v) if t == nil { diff --git a/queue_local.go b/queue_local.go index 1636792..251081f 100644 --- a/queue_local.go +++ b/queue_local.go @@ -6,6 +6,9 @@ import ( "sync" "sync/atomic" "time" + + "github.com/goforj/queue/busruntime" + "github.com/goforj/queue/internal/uniqueness" ) // localQueue is an in-memory queue implementation supporting sync and workerpool drivers. @@ -16,14 +19,21 @@ type localQueue struct { metricsMu sync.RWMutex queueMu sync.RWMutex handlers map[string]Handler - unique map[string]time.Time + unique uniqueness.MemoryStore metrics map[string]*localQueueMetrics pausedQueues map[string]bool workQueue chan queuedJob - shutdownCh chan struct{} + workPending int + workIdle chan struct{} + continuation *busruntime.ContinuationScope + resizeBuffer bool shutdownOnce sync.Once workerWG sync.WaitGroup - delayedWG sync.WaitGroup + + syncWorkMu sync.Mutex + syncWorkPending int + syncWorkIdle chan struct{} + shuttingDown atomic.Bool enqueued atomic.Int64 started atomic.Int64 @@ -31,14 +41,13 @@ type localQueue struct { delayed atomic.Int64 } -type workerContextKey string - -const workerEnqueueKey workerContextKey = "queue.worker.enqueue.allowed" +const localRedeliveryBackoff = time.Millisecond type queuedJob struct { - ctx context.Context - job Job - opts jobOptions + ctx context.Context + job Job + opts jobOptions + ready <-chan struct{} } type localQueueMetrics struct { @@ -54,14 +63,15 @@ func newLocalQueue(driver Driver) *localQueue { } func newLocalQueueWithConfig(driver Driver, cfg WorkerpoolConfig) *localQueue { + resizeBuffer := cfg.QueueCapacity <= 0 q := &localQueue{ driver: driver, cfg: cfg.normalize(), handlers: make(map[string]Handler), - unique: make(map[string]time.Time), metrics: make(map[string]*localQueueMetrics), pausedQueues: make(map[string]bool), - shutdownCh: make(chan struct{}), + continuation: busruntime.NewContinuationScope(), + resizeBuffer: resizeBuffer, } return q } @@ -140,26 +150,30 @@ func (d *localQueue) StartWorkers(_ context.Context) error { // _ = q.StartWorkers(context.Background()) // _ = q.Shutdown(context.Background()) func (d *localQueue) Shutdown(ctx context.Context) error { - if d.driver != DriverWorkerpool { + return d.DrainWorkers(ctx) +} + +// DrainWorkers stops admission from unrelated callers and waits for the +// accepted local work tree to finish while handler continuations remain valid. +func (d *localQueue) DrainWorkers(ctx context.Context) error { + if d.driver != DriverWorkerpool && d.driver != DriverSync { return nil } d.shutdownOnce.Do(func() { d.shuttingDown.Store(true) - close(d.shutdownCh) }) - if err := waitGroupWithContext(ctx, &d.delayedWG); err != nil { - return fmt.Errorf("workerpool delayed jobs drain failed: %w (%s)", err, d.shutdownStats()) + if d.driver == DriverSync { + if err := d.waitForSyncWork(ctx); err != nil { + return fmt.Errorf("sync jobs drain failed: %w (%s)", err, d.shutdownStats()) + } + return nil } - d.queueMu.Lock() - if d.workQueue != nil { - close(d.workQueue) - d.workQueue = nil + if err := d.closeWorkerQueueWhenIdle(ctx); err != nil { + return fmt.Errorf("workerpool queued jobs drain failed: %w (%s)", err, d.shutdownStats()) } - d.queueMu.Unlock() - if err := waitGroupWithContext(ctx, &d.workerWG); err != nil { return fmt.Errorf("workerpool active jobs drain failed: %w (%s)", err, d.shutdownStats()) } @@ -199,7 +213,8 @@ func (d *localQueue) Ready(ctx context.Context) error { // Delay(10 * time.Millisecond) // _, _ = q.Dispatch(job) func (d *localQueue) Dispatch(ctx context.Context, job Job) error { - if d.shuttingDown.Load() && !allowEnqueueDuringShutdown(ctx) { + ctx, acceptance := ensureDispatchAcceptance(ctx) + if d.shuttingDown.Load() && !d.continuation.Owns(ctx) { return ErrQueuerShuttingDown } if err := job.validate(); err != nil { @@ -207,21 +222,61 @@ func (d *localQueue) Dispatch(ctx context.Context, job Job) error { } parsed := job.jobOptions() queueName := normalizeQueueName(parsed.queueName) + if err := d.validateEnqueue(job, queueName); err != nil { + return err + } + var ( + uniqueKey string + uniqueToken uint64 + ) if parsed.uniqueTTL > 0 { - if !d.claimUnique(job, queueName, parsed.uniqueTTL) { + var acquired bool + uniqueKey, uniqueToken, acquired = d.claimUnique(job, queueName, parsed.uniqueTTL) + if !acquired { return ErrDuplicate } } if parsed.delay <= 0 { - return d.enqueueNow(ctx, job, parsed) + if d.driver == DriverSync { + if err := d.reserveSyncWork(ctx); err != nil { + d.unique.Release(uniqueKey, uniqueToken) + return err + } + defer d.finishSyncWork() + } + err := d.enqueueNow(ctx, job, parsed) + if err != nil && !acceptance.isAccepted() { + d.unique.Release(uniqueKey, uniqueToken) + } + return err + } + var reservedQueue chan queuedJob + switch d.driver { + case DriverWorkerpool: + var reserveErr error + reservedQueue, reserveErr = d.reserveWorkerQueue(ctx) + if reserveErr != nil { + d.unique.Release(uniqueKey, uniqueToken) + return reserveErr + } + case DriverSync: + if err := d.reserveSyncWork(ctx); err != nil { + d.unique.Release(uniqueKey, uniqueToken) + return err + } } - d.delayedWG.Add(1) d.delayed.Add(1) d.updateQueueMetrics(queueName, func(metrics *localQueueMetrics) { metrics.Delayed++ }) + if acceptance := dispatchAcceptanceFromContext(ctx); acceptance != nil { + acceptance.markAccepted() + } + delayedCtx := context.WithoutCancel(ctx) go func() { - defer d.delayedWG.Done() + if d.driver == DriverSync { + defer d.finishSyncWork() + } defer d.delayed.Add(-1) defer d.updateQueueMetrics(queueName, func(metrics *localQueueMetrics) { if metrics.Delayed > 0 { @@ -230,27 +285,29 @@ func (d *localQueue) Dispatch(ctx context.Context, job Job) error { }) timer := time.NewTimer(parsed.delay) defer timer.Stop() - select { - case <-timer.C: - _ = d.enqueueNow(context.Background(), job, parsed) - case <-d.shutdownCh: + <-timer.C + if d.driver == DriverWorkerpool { + if err := d.enqueueReservedAsync(delayedCtx, job, parsed, reservedQueue, false); err != nil { + d.finishQueuedWork() + } return } + _ = d.enqueueNow(delayedCtx, job, parsed) }() return nil } func (d *localQueue) enqueueNow(ctx context.Context, job Job, parsed jobOptions) error { queueName := normalizeQueueName(parsed.queueName) - if d.isPaused(queueName) { - return ErrQueuePaused - } - if _, ok := d.lookup(job.Type); !ok { - return fmt.Errorf("no handler registered for job type %q", job.Type) + if err := d.validateEnqueue(job, queueName); err != nil { + return err } if d.driver == DriverWorkerpool { return d.enqueueAsync(ctx, job, parsed) } + if acceptance := dispatchAcceptanceFromContext(ctx); acceptance != nil { + acceptance.markAccepted() + } d.updateQueueMetrics(queueName, func(metrics *localQueueMetrics) { metrics.Active++ }) @@ -269,53 +326,166 @@ func (d *localQueue) enqueueNow(ctx context.Context, job Job, parsed jobOptions) } func (d *localQueue) enqueueAsync(ctx context.Context, job Job, parsed jobOptions) error { - if d.shuttingDown.Load() && !allowEnqueueDuringShutdown(ctx) { - return ErrQueuerShuttingDown - } if ctx == nil { ctx = context.Background() } - workQueue, err := d.workerQueueForEnqueue() + workQueue, err := d.reserveWorkerQueue(ctx) if err != nil { return err } + return d.enqueueReservedAsync(ctx, job, parsed, workQueue, true) +} + +// enqueueReservedAsync accepts one already-reserved workerpool slot and preserves handler progress when bounded capacity is full. +func (d *localQueue) enqueueReservedAsync(ctx context.Context, job Job, parsed jobOptions, workQueue chan queuedJob, markAcceptance bool) error { + var ready chan struct{} + acceptance := dispatchAcceptanceFromContext(ctx) + if markAcceptance && acceptance != nil { + ready = make(chan struct{}) + } + queued := queuedJob{ctx: ctx, job: job, opts: parsed, ready: ready} + if d.continuation.Owns(ctx) { + select { + case workQueue <- queued: + d.recordQueuedJob(parsed, acceptance, ready) + return nil + default: + if ready != nil { + acceptance.markAccepted() + close(ready) + queued.ready = nil + } + d.recordQueuedJob(parsed, nil, nil) + go func() { workQueue <- queued }() + return nil + } + } select { - case workQueue <- queuedJob{ctx: ctx, job: job, opts: parsed}: - d.enqueued.Add(1) - d.updateQueueMetrics(normalizeQueueName(parsed.queueName), func(metrics *localQueueMetrics) { - metrics.Pending++ - }) + case workQueue <- queued: + d.recordQueuedJob(parsed, acceptance, ready) return nil case <-ctx.Done(): + d.finishQueuedWork() return ctx.Err() } } -func (d *localQueue) workerQueueForEnqueue() (chan queuedJob, error) { - d.queueMu.RLock() - workQueue := d.workQueue - d.queueMu.RUnlock() - if workQueue != nil { - return workQueue, nil +// recordQueuedJob updates acceptance and metrics only after this in-memory backend owns the reserved work. +func (d *localQueue) recordQueuedJob(parsed jobOptions, acceptance *dispatchAcceptance, ready chan struct{}) { + d.enqueued.Add(1) + d.updateQueueMetrics(normalizeQueueName(parsed.queueName), func(metrics *localQueueMetrics) { + metrics.Pending++ + }) + if ready != nil { + defer close(ready) + acceptance.markAccepted() } +} - // Self-heal: if the in-memory worker queue is unexpectedly nil while the - // the queue runtime is active, rebuild workers so dispatch can continue. +// reserveWorkerQueue keeps the channel open until this queued or active job and all descendants finish. +func (d *localQueue) reserveWorkerQueue(ctx context.Context) (chan queuedJob, error) { d.queueMu.Lock() defer d.queueMu.Unlock() - if d.workQueue != nil { - return d.workQueue, nil + if d.shuttingDown.Load() && !d.continuation.Owns(ctx) { + return nil, ErrQueuerShuttingDown } - if d.shuttingDown.Load() { - return nil, ErrWorkerpoolQueueNotInitialized + if d.workQueue == nil { + if d.shuttingDown.Load() { + return nil, ErrWorkerpoolQueueNotInitialized + } + d.startMemoryWorkersLocked() } - d.startMemoryWorkersLocked() if d.workQueue == nil { return nil, ErrWorkerpoolQueueNotInitialized } + if d.workPending == 0 { + d.workIdle = make(chan struct{}) + } + d.workPending++ return d.workQueue, nil } +// reserveSyncWork keeps shutdown attached to the current Sync work generation, including descendants admitted by a live handler. +func (d *localQueue) reserveSyncWork(ctx context.Context) error { + d.syncWorkMu.Lock() + defer d.syncWorkMu.Unlock() + if d.shuttingDown.Load() && !d.continuation.Owns(ctx) { + return ErrQueuerShuttingDown + } + if d.syncWorkPending == 0 { + d.syncWorkIdle = make(chan struct{}) + } + d.syncWorkPending++ + return nil +} + +// finishSyncWork releases one Sync job only after its handler can no longer admit descendants. +func (d *localQueue) finishSyncWork() { + d.syncWorkMu.Lock() + d.syncWorkPending-- + if d.syncWorkPending == 0 && d.syncWorkIdle != nil { + close(d.syncWorkIdle) + } + d.syncWorkMu.Unlock() +} + +// waitForSyncWork waits on the stable channel for the active Sync work generation without creating shutdown waiter goroutines. +func (d *localQueue) waitForSyncWork(ctx context.Context) error { + d.syncWorkMu.Lock() + if d.syncWorkPending == 0 { + d.syncWorkMu.Unlock() + return nil + } + idle := d.syncWorkIdle + d.syncWorkMu.Unlock() + if ctx == nil { + <-idle + return nil + } + select { + case <-idle: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// finishQueuedWork releases one accepted workerpool job after its handler can no longer enqueue descendants. +func (d *localQueue) finishQueuedWork() { + d.queueMu.Lock() + d.workPending-- + if d.workPending == 0 && d.workIdle != nil { + close(d.workIdle) + d.workIdle = nil + } + d.queueMu.Unlock() +} + +// closeWorkerQueueWhenIdle waits for the accepted work tree to quiesce before closing the worker channel. +func (d *localQueue) closeWorkerQueueWhenIdle(ctx context.Context) error { + if ctx == nil { + ctx = context.Background() + } + for { + d.queueMu.Lock() + if d.workPending == 0 { + if d.workQueue != nil { + close(d.workQueue) + d.workQueue = nil + } + d.queueMu.Unlock() + return nil + } + idle := d.workIdle + d.queueMu.Unlock() + select { + case <-idle: + case <-ctx.Done(): + return ctx.Err() + } + } +} + func (d *localQueue) startMemoryWorkers() { d.queueMu.Lock() defer d.queueMu.Unlock() @@ -323,7 +493,7 @@ func (d *localQueue) startMemoryWorkers() { } func (d *localQueue) startMemoryWorkersLocked() { - if d.workQueue != nil { + if d.workQueue != nil || d.shuttingDown.Load() { return } workers := d.cfg.Workers @@ -336,11 +506,31 @@ func (d *localQueue) startMemoryWorkersLocked() { } } +// setWorkers applies high-level worker configuration before the in-memory runtime starts. +func (d *localQueue) setWorkers(count int) { + if count <= 0 { + return + } + d.queueMu.Lock() + defer d.queueMu.Unlock() + if d.workQueue != nil || d.shuttingDown.Load() { + return + } + d.cfg.Workers = count + if d.resizeBuffer { + d.cfg.QueueCapacity = count + } +} + func (d *localQueue) worker(workQueue <-chan queuedJob) { defer d.workerWG.Done() jobTimeout := d.cfg.DefaultJobTimeout for job := range workQueue { func() { + defer d.finishQueuedWork() + if job.ready != nil { + <-job.ready + } d.started.Add(1) defer d.finished.Add(1) queueName := normalizeQueueName(job.opts.queueName) @@ -361,7 +551,7 @@ func (d *localQueue) worker(workQueue <-chan queuedJob) { } metrics.Failed++ }) - workerCtx := context.WithValue(job.ctx, workerEnqueueKey, true) + workerCtx := job.ctx if jobTimeout > 0 { var cancel context.CancelFunc workerCtx, cancel = context.WithTimeout(workerCtx, jobTimeout) @@ -379,6 +569,17 @@ func (d *localQueue) worker(workQueue <-chan queuedJob) { } } +// validateEnqueue rejects work before uniqueness is claimed or an acceptance fact is committed. +func (d *localQueue) validateEnqueue(job Job, queueName string) error { + if d.isPaused(queueName) { + return ErrQueuePaused + } + if _, ok := d.lookup(job.Type); !ok { + return fmt.Errorf("no handler registered for job type %q", job.Type) + } + return nil +} + func (d *localQueue) run(ctx context.Context, job Job) error { handler, ok := d.lookup(job.Type) if !ok { @@ -396,11 +597,10 @@ func (d *localQueue) runWithRetry(ctx context.Context, job Job, parsed jobOption ctx, cancel = context.WithTimeout(ctx, *parsed.timeout) defer cancel() } - attempts := 1 + maxRetry := 0 if parsed.maxRetry != nil && *parsed.maxRetry > 0 { - attempts += *parsed.maxRetry + maxRetry = *parsed.maxRetry } - var lastErr error jobForRun := job if parsed.maxRetry != nil { jobForRun = jobForRun.Retry(*parsed.maxRetry) @@ -408,26 +608,54 @@ func (d *localQueue) runWithRetry(ctx context.Context, job Job, parsed jobOption if parsed.queueName != "" { jobForRun = jobForRun.OnQueue(parsed.queueName) } - for attempt := 1; attempt <= attempts; attempt++ { - lastErr = d.run(ctx, jobForRun.withAttempt(attempt-1)) - if lastErr == nil { + for attempt := 0; ; { + delivery := busruntime.DeliveryAttempt{Number: attempt, MaxRetry: maxRetry} + attemptCtx := busruntime.WithDeliveryAttempt(ctx, delivery) + attemptCtx, release := d.continuation.Permit(attemptCtx) + err := func() error { + defer release() + return d.run(attemptCtx, jobForRun.withAttempt(attempt)) + }() + switch busruntime.ClassifyAttempt(delivery, err) { + case busruntime.AttemptSucceeded: return nil + case busruntime.AttemptFailed: + return err + case busruntime.AttemptRetry: + attempt++ + delay := time.Duration(0) + if parsed.backoff != nil { + delay = *parsed.backoff + } + if waitErr := waitForLocalRetry(ctx, delay); waitErr != nil { + return waitErr + } + case busruntime.AttemptRedeliver: + if waitErr := waitForLocalRetry(ctx, localRedeliveryBackoff); waitErr != nil { + return waitErr + } } - if attempt == attempts { - break - } - if parsed.backoff == nil || *parsed.backoff <= 0 { - continue - } - timer := time.NewTimer(*parsed.backoff) + } +} + +// waitForLocalRetry keeps retry and redelivery waits cancellable while avoiding timers for immediate application retries. +func waitForLocalRetry(ctx context.Context, delay time.Duration) error { + if delay <= 0 { select { - case <-timer.C: case <-ctx.Done(): - timer.Stop() return ctx.Err() + default: + return nil } } - return lastErr + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-timer.C: + return nil + case <-ctx.Done(): + return ctx.Err() + } } func (d *localQueue) lookup(jobType string) (Handler, bool) { @@ -508,23 +736,11 @@ func (d *localQueue) History(ctx context.Context, queueName string, window Queue return SinglePointHistory(snapshot, queueName), nil } -func (d *localQueue) claimUnique(job Job, queueName string, ttl time.Duration) bool { - now := time.Now() - key := queueName + ":" + job.Type + ":" + string(job.PayloadBytes()) - - d.mu.Lock() - defer d.mu.Unlock() - - for candidate, expiresAt := range d.unique { - if expiresAt.Before(now) { - delete(d.unique, candidate) - } - } - if expiresAt, ok := d.unique[key]; ok && expiresAt.After(now) { - return false - } - d.unique[key] = now.Add(ttl) - return true +// claimUnique returns the ownership token needed to compensate a pre-acceptance failure. +func (d *localQueue) claimUnique(job Job, queueName string, ttl time.Duration) (string, uint64, bool) { + key := DriverUniqueKey(job, queueName) + token, ok := d.unique.Acquire(key, ttl) + return key, token, ok } func (d *localQueue) updateQueueMetrics(queueName string, update func(metrics *localQueueMetrics)) { @@ -580,15 +796,6 @@ func waitGroupWithContext(ctx context.Context, waitGroup *sync.WaitGroup) error } } -func allowEnqueueDuringShutdown(ctx context.Context) bool { - if ctx == nil { - return false - } - value := ctx.Value(workerEnqueueKey) - allowed, _ := value.(bool) - return allowed -} - func (d *localQueue) shutdownStats() string { d.queueMu.RLock() queued := 0 diff --git a/queue_local_test.go b/queue_local_test.go index 3858df0..ef7f3cb 100644 --- a/queue_local_test.go +++ b/queue_local_test.go @@ -6,6 +6,8 @@ import ( "sync/atomic" "testing" "time" + + "github.com/goforj/queue/busruntime" ) func TestLocalQueue_Driver(t *testing.T) { @@ -58,6 +60,218 @@ func TestLocalQueue_DispatchDelayed(t *testing.T) { } } +// TestLocalQueueSyncShutdownSucceedsWhenCanceledAndIdle prevents an expired caller budget from failing already-complete cleanup. +func TestLocalQueueSyncShutdownSucceedsWhenCanceledAndIdle(t *testing.T) { + d := newLocalQueue(DriverSync) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + if err := d.Shutdown(ctx); err != nil { + t.Fatalf("idle sync shutdown failed: %v", err) + } + if !d.shuttingDown.Load() { + t.Fatal("idle sync shutdown did not latch shutdown state") + } + + d.Register("job:after-idle-shutdown", func(context.Context, Job) error { return nil }) + err := d.Dispatch(context.Background(), NewJob("job:after-idle-shutdown")) + if !errors.Is(err, ErrQueuerShuttingDown) { + t.Fatalf("dispatch after idle shutdown error = %v, want %v", err, ErrQueuerShuttingDown) + } +} + +// TestLocalQueueSyncShutdownHonorsCancellationWithPendingWork keeps a pending drain generation bounded and retryable. +func TestLocalQueueSyncShutdownHonorsCancellationWithPendingWork(t *testing.T) { + d := newLocalQueue(DriverSync) + if err := d.reserveSyncWork(context.Background()); err != nil { + t.Fatalf("reserve sync work: %v", err) + } + pending := true + t.Cleanup(func() { + if pending { + d.finishSyncWork() + } + }) + d.syncWorkMu.Lock() + sharedDone := d.syncWorkIdle + d.syncWorkMu.Unlock() + if sharedDone == nil { + t.Fatal("pending sync work did not open a drain generation") + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := d.Shutdown(ctx); !errors.Is(err, context.Canceled) { + t.Fatalf("pending sync shutdown error = %v, want %v", err, context.Canceled) + } + for range 32 { + if err := d.Shutdown(ctx); !errors.Is(err, context.Canceled) { + t.Fatalf("repeated pending sync shutdown error = %v, want %v", err, context.Canceled) + } + d.syncWorkMu.Lock() + currentDone := d.syncWorkIdle + d.syncWorkMu.Unlock() + if currentDone != sharedDone { + t.Fatal("pending sync shutdown retry replaced the work drain generation") + } + } + if !d.shuttingDown.Load() { + t.Fatal("canceled sync shutdown did not latch shutdown state") + } + d.Register("job:after-canceled-shutdown", func(context.Context, Job) error { return nil }) + if err := d.Dispatch(context.Background(), NewJob("job:after-canceled-shutdown")); !errors.Is(err, ErrQueuerShuttingDown) { + t.Fatalf("dispatch after canceled shutdown error = %v, want %v", err, ErrQueuerShuttingDown) + } + + d.finishSyncWork() + pending = false + if err := d.Shutdown(context.Background()); err != nil { + t.Fatalf("sync shutdown retry failed after work completed: %v", err) + } + select { + case <-sharedDone: + default: + t.Fatal("completed sync work did not close its drain generation") + } +} + +// continuationGateContext reports when Dispatch has observed its initial live continuation permit. +type continuationGateContext struct { + context.Context + passed chan<- struct{} +} + +// Value preserves the wrapped context while exposing the otherwise internal continuation lookup as a deterministic test seam. +func (c continuationGateContext) Value(key any) any { + value := c.Context.Value(key) + if value != nil { + select { + case c.passed <- struct{}{}: + default: + } + } + return value +} + +// TestLocalQueueRejectsContinuationThatEscapesBeforeReservation verifies shutdown cannot be overtaken by a child that passed the initial permit gate but lost ownership before reserving work. +func TestLocalQueueRejectsContinuationThatEscapesBeforeReservation(t *testing.T) { + tests := []struct { + name string + driver Driver + delayed bool + }{ + {name: "sync immediate", driver: DriverSync}, + {name: "sync delayed", driver: DriverSync, delayed: true}, + {name: "workerpool immediate", driver: DriverWorkerpool}, + {name: "workerpool delayed", driver: DriverWorkerpool, delayed: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + d := newLocalQueueWithConfig(test.driver, WorkerpoolConfig{Workers: 1, QueueCapacity: 1}) + var handlerCalls atomic.Int64 + jobType := "job:escaped:" + test.name + d.Register(jobType, func(context.Context, Job) error { + handlerCalls.Add(1) + return nil + }) + + parentPending := true + switch test.driver { + case DriverSync: + if err := d.reserveSyncWork(context.Background()); err != nil { + t.Fatalf("reserve parent Sync work: %v", err) + } + case DriverWorkerpool: + if _, err := d.reserveWorkerQueue(context.Background()); err != nil { + t.Fatalf("reserve parent Workerpool work: %v", err) + } + } + + permitCtx, releasePermit := d.continuation.Permit(context.Background()) + permitActive := true + muHeld := false + t.Cleanup(func() { + if permitActive { + releasePermit() + } + if muHeld { + d.mu.Unlock() + } + if parentPending { + if test.driver == DriverSync { + d.finishSyncWork() + } else { + d.finishQueuedWork() + } + } + if err := d.Shutdown(context.Background()); err != nil { + t.Errorf("cleanup shutdown: %v", err) + } + }) + + canceledCtx, cancel := context.WithCancel(context.Background()) + cancel() + if err := d.Shutdown(canceledCtx); !errors.Is(err, context.Canceled) { + t.Fatalf("establish draining state error = %v, want %v", err, context.Canceled) + } + + passedInitialGate := make(chan struct{}, 1) + childResult := make(chan error, 1) + job := NewJob(jobType). + Payload([]byte(test.name)). + OnQueue("default"). + UniqueFor(time.Hour) + if test.delayed { + job = job.Delay(time.Millisecond) + } + + d.mu.Lock() + muHeld = true + go func() { + childResult <- d.Dispatch(continuationGateContext{Context: permitCtx, passed: passedInitialGate}, job) + }() + select { + case <-passedInitialGate: + case <-time.After(5 * time.Second): + t.Fatal("child did not pass the initial continuation gate") + } + + releasePermit() + permitActive = false + if test.driver == DriverSync { + d.finishSyncWork() + } else { + d.finishQueuedWork() + } + parentPending = false + if err := d.Shutdown(context.Background()); err != nil { + t.Fatalf("finish parent drain: %v", err) + } + d.mu.Unlock() + muHeld = false + + select { + case err := <-childResult: + if !errors.Is(err, ErrQueuerShuttingDown) { + t.Fatalf("escaped continuation error = %v, want %v", err, ErrQueuerShuttingDown) + } + case <-time.After(5 * time.Second): + t.Fatal("escaped continuation did not finish") + } + if got := handlerCalls.Load(); got != 0 { + t.Fatalf("escaped continuation handler calls = %d, want 0", got) + } + if got := d.delayed.Load(); got != 0 { + t.Fatalf("escaped continuation delayed jobs = %d, want 0", got) + } + key := DriverUniqueKey(job, "default") + if _, ok := d.unique.Acquire(key, time.Hour); !ok { + t.Fatal("escaped continuation retained its uniqueness claim") + } + }) + } +} + func TestLocalQueue_DispatchMissingHandlerFails(t *testing.T) { d := newLocalQueue(DriverSync) err := d.Dispatch(context.Background(), NewJob("missing").OnQueue("default")) @@ -107,6 +321,26 @@ func TestLocalQueue_DispatchWithUnique(t *testing.T) { } } +// TestLocalQueueUniqueClaimCompensatesRejectedEnqueue verifies a failed acceptance cannot poison the TTL window. +func TestLocalQueueUniqueClaimCompensatesRejectedEnqueue(t *testing.T) { + d := newLocalQueueWithConfig(DriverWorkerpool, WorkerpoolConfig{Workers: 1}) + d.Register("job:unique:rejected", func(context.Context, Job) error { return nil }) + d.queueMu.Lock() + d.workQueue = make(chan queuedJob) + d.queueMu.Unlock() + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + job := NewJob("job:unique:rejected").Payload([]byte("same")).OnQueue("default").UniqueFor(time.Minute) + if err := d.Dispatch(ctx, job); !errors.Is(err, context.Canceled) { + t.Fatalf("rejected dispatch error = %v, want context canceled", err) + } + key := DriverUniqueKey(job, "default") + if _, ok := d.unique.Acquire(key, time.Minute); !ok { + t.Fatal("rejected dispatch retained its uniqueness claim") + } +} + func TestLocalQueue_WorkerpoolDispatchRunsOnWorkers(t *testing.T) { t.Setenv("QUEUE_WORKERPOOL_WORKERS", "2") t.Setenv("QUEUE_WORKERPOOL_BUFFER", "4") @@ -276,6 +510,77 @@ func TestLocalQueue_SyncRetriesWithBackoff(t *testing.T) { } } +// TestLocalQueue_PermanentErrorStopsRetries verifies terminal application failures do not consume the remaining retry budget. +func TestLocalQueue_PermanentErrorStopsRetries(t *testing.T) { + d := newLocalQueue(DriverSync) + cause := errors.New("invalid recipient") + var calls atomic.Int64 + d.Register("job:permanent", func(_ context.Context, _ Job) error { + calls.Add(1) + return busruntime.Permanent(cause) + }) + + err := d.Dispatch(context.Background(), NewJob("job:permanent").Retry(5)) + if !busruntime.IsPermanent(err) || !errors.Is(err, cause) { + t.Fatalf("dispatch error = %v, want permanent cause", err) + } + if calls.Load() != 1 { + t.Fatalf("handler calls = %d, want 1", calls.Load()) + } +} + +// TestLocalQueue_UncommittedErrorRedeliversSameAttempt verifies infrastructure failures do not consume application retries. +func TestLocalQueue_UncommittedErrorRedeliversSameAttempt(t *testing.T) { + d := newLocalQueue(DriverSync) + infrastructureErr := errors.New("workflow store unavailable") + transientErr := errors.New("application failed") + attempts := make([]int, 0, 3) + d.Register("job:redeliver", func(_ context.Context, job Job) error { + attempts = append(attempts, job.jobOptions().attempt) + switch len(attempts) { + case 1: + return busruntime.Uncommitted(infrastructureErr) + case 2: + return transientErr + default: + return nil + } + }) + + if err := d.Dispatch(context.Background(), NewJob("job:redeliver").Retry(1)); err != nil { + t.Fatalf("dispatch: %v", err) + } + want := []int{0, 0, 1} + if len(attempts) != len(want) { + t.Fatalf("attempts = %v, want %v", attempts, want) + } + for index := range want { + if attempts[index] != want[index] { + t.Fatalf("attempts = %v, want %v", attempts, want) + } + } +} + +// TestLocalQueue_UncommittedRedeliveryHonorsCancellation verifies local infrastructure redelivery cannot spin after its caller stops waiting. +func TestLocalQueue_UncommittedRedeliveryHonorsCancellation(t *testing.T) { + d := newLocalQueue(DriverSync) + ctx, cancel := context.WithCancel(context.Background()) + var calls atomic.Int64 + d.Register("job:redeliver:cancel", func(_ context.Context, _ Job) error { + calls.Add(1) + cancel() + return busruntime.Uncommitted(errors.New("workflow store unavailable")) + }) + + err := d.Dispatch(ctx, NewJob("job:redeliver:cancel").Retry(3)) + if !errors.Is(err, context.Canceled) { + t.Fatalf("dispatch error = %v, want context canceled", err) + } + if calls.Load() != 1 { + t.Fatalf("handler calls = %d, want 1", calls.Load()) + } +} + func TestLocalQueue_WorkerpoolRetriesWithBackoff(t *testing.T) { d := newLocalQueueWithConfig(DriverWorkerpool, WorkerpoolConfig{Workers: 1, QueueCapacity: 4}) triggered := make(chan struct{}, 1) @@ -397,3 +702,149 @@ func TestLocalQueue_SyncStatsTrackFailuresPerQueue(t *testing.T) { t.Fatalf("unexpected low counters: %+v", low) } } + +// TestWorkerpoolShutdownDrainsWorkflowDescendants verifies channel closure waits until an active chain can enqueue and run its next node. +func TestWorkerpoolShutdownDrainsWorkflowDescendants(t *testing.T) { + testWorkerpoolShutdownDrainsWorkflowDescendants(t, WithWorkers(1)) +} + +// TestWorkerpoolShutdownDrainsWorkflowDescendantsWithReplacementDecorator +// verifies a replacement handler context cannot discard backend continuation authority. +func TestWorkerpoolShutdownDrainsWorkflowDescendantsWithReplacementDecorator(t *testing.T) { + for _, withObserver := range []bool{false, true} { + name := "without observer" + opts := []Option{ + WithWorkers(1), + WithHandlerContextDecorator(func(context.Context) context.Context { + return context.Background() + }), + } + if withObserver { + name = "with observer" + opts = append(opts, WithObserver(ObserverFunc(func(context.Context, Event) {}))) + } + t.Run(name, func(t *testing.T) { + testWorkerpoolShutdownDrainsWorkflowDescendants(t, opts...) + }) + } +} + +// testWorkerpoolShutdownDrainsWorkflowDescendants exercises workflow drain +// through the public Queue while using runtime state only as a deterministic gate. +func testWorkerpoolShutdownDrainsWorkflowDescendants(t *testing.T, opts ...Option) { + t.Helper() + q, err := NewWorkerpool(opts...) + if err != nil { + t.Fatalf("new workerpool: %v", err) + } + firstStarted := make(chan struct{}) + releaseFirst := make(chan struct{}) + secondRan := make(chan struct{}) + q.Register("shutdown:chain:first", func(context.Context, Message) error { + close(firstStarted) + <-releaseFirst + return nil + }) + q.Register("shutdown:chain:second", func(context.Context, Message) error { + close(secondRan) + return nil + }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + chainID, err := q.Chain( + NewJob("shutdown:chain:first"), + NewJob("shutdown:chain:second").Delay(25*time.Millisecond), + ).Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch chain: %v", err) + } + <-firstStarted + shutdownResult := make(chan error, 1) + go func() { shutdownResult <- q.Shutdown(context.Background()) }() + runtime := q.q.(*nativeQueueRuntime) + local := runtime.runtime.(*localQueue) + if local.cfg.Workers != 1 || local.cfg.QueueCapacity != 1 { + t.Fatalf("configured workerpool = workers:%d capacity:%d, want 1/1", local.cfg.Workers, local.cfg.QueueCapacity) + } + drainDeadline := time.Now().Add(2 * time.Second) + for { + runtime.mu.Lock() + draining := runtime.draining + runtime.mu.Unlock() + if draining && local.shuttingDown.Load() { + break + } + if time.Now().After(drainDeadline) { + t.Fatal("timed out waiting for root and workerpool drain gates") + } + time.Sleep(time.Millisecond) + } + close(releaseFirst) + select { + case <-secondRan: + case <-time.After(5 * time.Second): + t.Fatal("shutdown stranded the descendant chain node") + } + select { + case shutdownErr := <-shutdownResult: + if shutdownErr != nil { + t.Fatalf("shutdown: %v", shutdownErr) + } + case <-time.After(5 * time.Second): + t.Fatal("shutdown did not finish after descendant work quiesced") + } + state, err := q.FindChain(context.Background(), chainID) + if err != nil { + t.Fatalf("find chain: %v", err) + } + if !state.Completed || state.Failed || state.NextIndex != 2 { + t.Fatalf("chain state after shutdown = %+v", state) + } +} + +// TestWorkerpoolTerminalCallbacksDoNotDeadlockBoundedQueue verifies one handler can schedule sibling callbacks when its only queue slot is already occupied. +func TestWorkerpoolTerminalCallbacksDoNotDeadlockBoundedQueue(t *testing.T) { + q, err := NewWorkerpool(WithWorkers(1)) + if err != nil { + t.Fatalf("new workerpool: %v", err) + } + q.Register("shutdown:batch:failure", func(context.Context, Message) error { + return errors.New("terminal batch failure") + }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + catchRan := make(chan struct{}, 1) + finallyRan := make(chan struct{}, 1) + batchID, err := q.Batch(NewJob("shutdown:batch:failure").Retry(0)). + Catch(func(context.Context, BatchState, error) error { + catchRan <- struct{}{} + return nil + }). + Finally(func(context.Context, BatchState) error { + finallyRan <- struct{}{} + return nil + }). + Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch batch: %v", err) + } + for name, callback := range map[string]<-chan struct{}{"catch": catchRan, "finally": finallyRan} { + select { + case <-callback: + case <-time.After(5 * time.Second): + t.Fatalf("%s callback deadlocked behind bounded worker queue", name) + } + } + if err := q.Shutdown(context.Background()); err != nil { + t.Fatalf("shutdown: %v", err) + } + state, err := q.FindBatch(context.Background(), batchID) + if err != nil { + t.Fatalf("find batch: %v", err) + } + if !state.Completed || !state.Cancelled || state.Failed != 1 { + t.Fatalf("terminal batch state = %+v", state) + } +} diff --git a/queue_null.go b/queue_null.go index cdc8cb3..b4d9e29 100644 --- a/queue_null.go +++ b/queue_null.go @@ -3,17 +3,17 @@ package queue import ( "context" "fmt" - "sync" "time" + + "github.com/goforj/queue/internal/uniqueness" ) type nullQueue struct { - mu sync.Mutex - unique map[string]time.Time + unique uniqueness.MemoryStore } func newNullQueue() queueBackend { - return &nullQueue{unique: make(map[string]time.Time)} + return &nullQueue{} } func (q *nullQueue) Driver() Driver { @@ -54,6 +54,11 @@ func (q *nullQueue) Shutdown(context.Context) error { return nil } +// DrainWorkers completes immediately because the null backend executes no work. +func (q *nullQueue) DrainWorkers(context.Context) error { + return nil +} + func (q *nullQueue) Ready(ctx context.Context) error { if ctx == nil { return nil @@ -61,19 +66,8 @@ func (q *nullQueue) Ready(ctx context.Context) error { return ctx.Err() } +// claimUnique records the null backend's accepted TTL window. func (q *nullQueue) claimUnique(job Job, queueName string, ttl time.Duration) bool { - now := time.Now() - key := queueName + ":" + jobEventKey(job) - q.mu.Lock() - defer q.mu.Unlock() - for k, expiresAt := range q.unique { - if now.After(expiresAt) { - delete(q.unique, k) - } - } - if expiresAt, ok := q.unique[key]; ok && now.Before(expiresAt) { - return false - } - q.unique[key] = now.Add(ttl) - return true + _, ok := q.unique.Acquire(DriverUniqueKey(job, queueName), ttl) + return ok } diff --git a/queue_runtime_unit_test.go b/queue_runtime_unit_test.go index ac30d51..ca414a8 100644 --- a/queue_runtime_unit_test.go +++ b/queue_runtime_unit_test.go @@ -3,6 +3,7 @@ package queue import ( "context" "errors" + "sync" "testing" "time" @@ -10,15 +11,137 @@ import ( ) type runtimeBackendStub struct { - registered map[string]Handler - startCalls int - stopCalls int - startErr error - stopErr error + registered map[string]Handler + startCalls int + drainCalls int + stopCalls int + startErr error + stopErr error + dispatchEntered chan struct{} + releaseDispatch chan struct{} + dispatchOnce sync.Once +} + +type blockingRuntimeBackendStub struct { + runtimeBackendStub + startEntered chan struct{} + releaseStart chan struct{} + startOnce sync.Once +} + +type blockingStrictRegistrationRuntimeBackendStub struct { + strictRegistrationRuntimeBackendStub + startEntered chan struct{} + releaseStart chan struct{} + startOnce sync.Once +} + +type blockingReadyRuntimeBackendStub struct { + runtimeBackendStub + readyEntered chan struct{} + releaseReady chan struct{} + readyOnce sync.Once + readyCalls int +} + +type blockingShutdownRuntimeBackendStub struct { + runtimeBackendStub + shutdownEntered chan struct{} + releaseShutdown chan struct{} + shutdownOnce sync.Once +} + +type phasedShutdownRuntimeBackendStub struct { + runtimeBackendStub + drainEntered chan struct{} + releaseDrain chan struct{} + drainOnce sync.Once +} + +type strictRegistrationRuntimeBackendStub struct { + runtimeBackendStub + registrations map[string]int +} + +// Register panics when one worker receives the same pattern twice, matching Asynq ServeMux behavior. +func (s *strictRegistrationRuntimeBackendStub) Register(jobType string, handler Handler) { + if s.registrations == nil { + s.registrations = make(map[string]int) + } + s.registrations[jobType]++ + if s.registrations[jobType] > 1 { + panic("duplicate worker registration: " + jobType) + } + s.runtimeBackendStub.Register(jobType, handler) +} + +// StartWorkers rejects a canceled attempt before accepting a later live retry. +func (s *strictRegistrationRuntimeBackendStub) StartWorkers(ctx context.Context) error { + s.startCalls++ + return ctx.Err() +} + +// Shutdown exposes the worker-drained boundary while deliberately ignoring cancellation like a backend cleanup that already committed. +func (s *blockingShutdownRuntimeBackendStub) Shutdown(context.Context) error { + s.stopCalls++ + s.shutdownOnce.Do(func() { close(s.shutdownEntered) }) + <-s.releaseShutdown + return s.stopErr +} + +// DrainWorkers exposes the pre-resource-close boundary of a native shutdown. +func (s *phasedShutdownRuntimeBackendStub) DrainWorkers(context.Context) error { + s.drainOnce.Do(func() { close(s.drainEntered) }) + <-s.releaseDrain + return nil +} + +// Ready exposes a deterministic producer-resource boundary for shutdown lease tests. +func (s *blockingReadyRuntimeBackendStub) Ready(ctx context.Context) error { + s.readyCalls++ + s.readyOnce.Do(func() { close(s.readyEntered) }) + select { + case <-s.releaseReady: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// StartWorkers exposes a deterministic startup boundary for lifecycle race tests. +func (s *blockingRuntimeBackendStub) StartWorkers(context.Context) error { + s.startCalls++ + s.startOnce.Do(func() { close(s.startEntered) }) + <-s.releaseStart + return s.startErr +} + +// StartWorkers exposes the live-start boundary while retaining strict registration counts across retries. +func (s *blockingStrictRegistrationRuntimeBackendStub) StartWorkers(context.Context) error { + s.startCalls++ + s.startOnce.Do(func() { close(s.startEntered) }) + <-s.releaseStart + return s.startErr +} + +// waitForRuntimeDraining waits until a shutdown goroutine has crossed the lifecycle gate. +func waitForRuntimeDraining(t *testing.T, draining func() bool) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for !draining() { + if time.Now().After(deadline) { + t.Fatal("timed out waiting for runtime to begin draining") + } + time.Sleep(time.Millisecond) + } } func (s *runtimeBackendStub) Driver() Driver { return DriverSync } func (s *runtimeBackendStub) Dispatch(context.Context, Job) error { + if s.dispatchEntered != nil { + s.dispatchOnce.Do(func() { close(s.dispatchEntered) }) + <-s.releaseDispatch + } return nil } @@ -34,24 +157,38 @@ func (s *runtimeBackendStub) StartWorkers(context.Context) error { return s.startErr } +// DrainWorkers completes the stub's distinct worker-drain lifecycle phase. +func (s *runtimeBackendStub) DrainWorkers(context.Context) error { + s.drainCalls++ + return nil +} + func (s *runtimeBackendStub) Shutdown(context.Context) error { s.stopCalls++ return s.stopErr } type queueBackendRecorder struct { - dispatched []Job - shutdowns int + dispatched []Job + shutdowns int + shutdownErr error + dispatchEntered chan struct{} + releaseDispatch chan struct{} + dispatchOnce sync.Once } func (q *queueBackendRecorder) Driver() Driver { return DriverNull } func (q *queueBackendRecorder) Dispatch(_ context.Context, job Job) error { q.dispatched = append(q.dispatched, job) + if q.dispatchEntered != nil { + q.dispatchOnce.Do(func() { close(q.dispatchEntered) }) + <-q.releaseDispatch + } return nil } func (q *queueBackendRecorder) Shutdown(context.Context) error { q.shutdowns++ - return nil + return q.shutdownErr } type driverQueueBackendStub struct { @@ -104,6 +241,11 @@ func (s *driverRuntimeBackendStub) StartWorkers(context.Context) error { return s.startErr } +// DrainWorkers completes the driver stub's distinct worker-drain phase. +func (s *driverRuntimeBackendStub) DrainWorkers(context.Context) error { + return nil +} + func TestQueueCommon_JobFromAnyAndHelpers(t *testing.T) { common := &queueCommon{cfg: Config{DefaultQueue: "default"}} @@ -113,6 +255,9 @@ func TestQueueCommon_JobFromAnyAndHelpers(t *testing.T) { if _, err := common.jobFromAny(NewJob("")); err == nil { t.Fatal("expected empty job type error") } + if _, err := common.jobFromAny(NewJob("deferred:validation").Retry(-1)); err != nil { + t.Fatalf("jobFromAny changed backend validation timing: %v", err) + } if _, err := common.jobFromAny(struct{ F func() }{}); err == nil { t.Fatal("expected marshal error for func field") } @@ -132,7 +277,13 @@ func TestQueueCommonDispatchAndNativeRuntimeWrappers(t *testing.T) { inner := &queueBackendRecorder{} worker := &runtimeBackendStub{} common := &queueCommon{inner: inner, cfg: Config{DefaultQueue: "default"}, driver: DriverSync} - q := &nativeQueueRuntime{common: common, runtime: worker, registered: map[string]Handler{}} + q := &nativeQueueRuntime{ + common: common, + runtime: worker, + nativeQueueRuntimeState: &nativeQueueRuntimeState{ + registered: map[string]Handler{}, + }, + } if q.Driver() != DriverSync { t.Fatalf("expected driver sync, got %q", q.Driver()) @@ -163,8 +314,85 @@ func TestQueueCommonDispatchAndNativeRuntimeWrappers(t *testing.T) { if err := q.Shutdown(nil); err != nil { t.Fatalf("shutdown failed: %v", err) } - if worker.stopCalls != 1 { - t.Fatalf("expected shutdown called once, got %d", worker.stopCalls) + if worker.drainCalls != 1 || worker.stopCalls != 0 || inner.shutdowns != 1 { + t.Fatalf("native drain/runtime close/inner close calls = %d/%d/%d, want 1/0/1", worker.drainCalls, worker.stopCalls, inner.shutdowns) + } +} + +func TestRuntimeWithContextSharesLifecycleState(t *testing.T) { + native := &nativeQueueRuntime{ + common: &queueCommon{cfg: Config{DefaultQueue: "default"}}, + runtime: &runtimeBackendStub{}, + nativeQueueRuntimeState: &nativeQueueRuntimeState{ + registered: map[string]Handler{}, + }, + } + nativeDerived, ok := native.WithContext(context.Background()).(*nativeQueueRuntime) + if !ok { + t.Fatal("expected a derived native runtime") + } + if nativeDerived.nativeQueueRuntimeState != native.nativeQueueRuntimeState { + t.Fatal("derived native runtime does not share lifecycle state") + } + nativeDerived.Workers(3) + if native.workers != 3 { + t.Fatalf("native worker count = %d, want shared value 3", native.workers) + } + + external := &externalQueueRuntime{ + common: &queueCommon{cfg: Config{DefaultQueue: "default"}}, + externalQueueRuntimeState: &externalQueueRuntimeState{ + registered: map[string]Handler{}, + }, + } + externalDerived, ok := external.WithContext(context.Background()).(*externalQueueRuntime) + if !ok { + t.Fatal("expected a derived external runtime") + } + if externalDerived.externalQueueRuntimeState != external.externalQueueRuntimeState { + t.Fatal("derived external runtime does not share lifecycle state") + } + externalDerived.Workers(5) + if external.workers != 5 { + t.Fatalf("external worker count = %d, want shared value 5", external.workers) + } +} + +// TestRuntimeEventQueueResolvers verifies every runtime shape exposes the same +// namespace mapping without requiring a live backend. +func TestRuntimeEventQueueResolvers(t *testing.T) { + common := &queueCommon{cfg: Config{DefaultQueue: "billing_default"}} + if got := common.physicalQueueNameOrDefault(""); got != "billing_default" { + t.Fatalf("common default queue = %q, want billing_default", got) + } + if got := common.physicalQueueNameOrDefault("critical"); got != "billing_critical" { + t.Fatalf("common explicit queue = %q, want billing_critical", got) + } + + native := &nativeQueueRuntime{common: common} + if got := native.physicalQueueNameOrDefault("critical"); got != "billing_critical" { + t.Fatalf("native explicit queue = %q, want billing_critical", got) + } + if got := (*nativeQueueRuntime)(nil).physicalQueueNameOrDefault(""); got != "default" { + t.Fatalf("nil native default queue = %q, want default", got) + } + external := &externalQueueRuntime{common: common} + if got := external.physicalQueueNameOrDefault("critical"); got != "billing_critical" { + t.Fatalf("external explicit queue = %q, want billing_critical", got) + } + if got := (*externalQueueRuntime)(nil).physicalQueueNameOrDefault(""); got != "default" { + t.Fatalf("nil external default queue = %q, want default", got) + } + + fake := NewFake() + if got := fake.physicalQueueNameOrDefault(""); got != "default" { + t.Fatalf("fake default queue = %q, want default", got) + } + if got := fake.physicalQueueNameOrDefault("critical"); got != "critical" { + t.Fatalf("fake explicit queue = %q, want critical", got) + } + if got := (*FakeQueue)(nil).physicalQueueNameOrDefault(""); got != "default" { + t.Fatalf("nil fake default queue = %q, want default", got) } } @@ -194,6 +422,9 @@ func TestQueueCommonDispatchPhysicalizesTargetQueues(t *testing.T) { q := &nativeQueueRuntime{ common: &queueCommon{inner: inner, cfg: Config{DefaultQueue: "billing_default"}, driver: DriverSync}, runtime: &runtimeBackendStub{}, + nativeQueueRuntimeState: &nativeQueueRuntimeState{ + registered: map[string]Handler{}, + }, } if err := q.Dispatch(NewJob("job:explicit").OnQueue("reports")); err != nil { @@ -220,10 +451,12 @@ func TestExternalQueueRuntimeRegisterShutdownAndWorkers(t *testing.T) { worker := &runtimeBackendStub{} common := &queueCommon{inner: inner, cfg: Config{DefaultQueue: "default"}, driver: DriverNATS} q := &externalQueueRuntime{ - common: common, - registered: map[string]Handler{}, - worker: worker, - started: true, + common: common, + externalQueueRuntimeState: &externalQueueRuntimeState{ + registered: map[string]Handler{}, + worker: worker, + started: true, + }, } q.Workers(3) @@ -261,6 +494,900 @@ func TestExternalQueueRuntimeRegisterShutdownAndWorkers(t *testing.T) { } } +// TestRuntimeSameKeyReplacementDuringBlockedStart verifies a completed registration remains current while startup is in flight. +func TestRuntimeSameKeyReplacementDuringBlockedStart(t *testing.T) { + for _, external := range []bool{false, true} { + name := "native" + if external { + name = "external" + } + t.Run(name, func(t *testing.T) { + worker := &blockingStrictRegistrationRuntimeBackendStub{ + startEntered: make(chan struct{}), + releaseStart: make(chan struct{}), + } + var runtime queueRuntime + if external { + runtime = &externalQueueRuntime{ + common: &queueCommon{inner: &queueBackendRecorder{}, cfg: Config{DefaultQueue: "default"}, driver: DriverSQS}, + newWorker: func(int) (driverWorkerBackend, error) { + return worker, nil + }, + externalQueueRuntimeState: &externalQueueRuntimeState{registered: map[string]Handler{}}, + } + } else { + runtime = &nativeQueueRuntime{ + common: &queueCommon{inner: worker, cfg: Config{DefaultQueue: "default"}, driver: DriverSync}, + runtime: worker, + nativeQueueRuntimeState: &nativeQueueRuntimeState{ + registered: map[string]Handler{}, + }, + } + } + + var firstCalls, secondCalls int + runtime.Register("job:replace", func(context.Context, Job) error { + firstCalls++ + return nil + }) + startResult := make(chan error, 1) + go func() { startResult <- runtime.StartWorkers(context.Background()) }() + <-worker.startEntered + runtime.Register("job:replace", func(context.Context, Job) error { + secondCalls++ + return nil + }) + close(worker.releaseStart) + if err := <-startResult; err != nil { + t.Fatalf("start workers: %v", err) + } + handler := worker.registered["job:replace"] + if handler == nil { + t.Fatal("worker did not receive replacement slot") + } + if err := handler(context.Background(), NewJob("job:replace")); err != nil { + t.Fatalf("invoke replacement: %v", err) + } + if firstCalls != 0 || secondCalls != 1 { + t.Fatalf("replacement calls = first:%d second:%d, want 0/1", firstCalls, secondCalls) + } + if err := runtime.Shutdown(context.Background()); err != nil { + t.Fatalf("shutdown: %v", err) + } + }) + } +} + +// TestRuntimeNewRegistrationIsLiveDuringBlockedStart verifies Register cannot complete while a consuming backend still lacks the new type. +func TestRuntimeNewRegistrationIsLiveDuringBlockedStart(t *testing.T) { + startErr := errors.New("worker start failed") + tests := []struct { + name string + external bool + startFails bool + }{ + {name: "native_success"}, + {name: "native_failed_start_retry", startFails: true}, + {name: "external_success", external: true}, + {name: "external_failed_start_retry", external: true, startFails: true}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + worker := &blockingStrictRegistrationRuntimeBackendStub{ + startEntered: make(chan struct{}), + releaseStart: make(chan struct{}), + } + if test.startFails { + worker.startErr = startErr + } + var runtime queueRuntime + if test.external { + runtime = &externalQueueRuntime{ + common: &queueCommon{inner: &queueBackendRecorder{}, cfg: Config{DefaultQueue: "default"}, driver: DriverSQS}, + newWorker: func(int) (driverWorkerBackend, error) { + return worker, nil + }, + externalQueueRuntimeState: &externalQueueRuntimeState{registered: map[string]Handler{}}, + } + } else { + runtime = &nativeQueueRuntime{ + common: &queueCommon{inner: worker, cfg: Config{DefaultQueue: "default"}, driver: DriverSync}, + runtime: worker, + nativeQueueRuntimeState: &nativeQueueRuntimeState{ + registered: map[string]Handler{}, + }, + } + } + + startResult := make(chan error, 1) + go func() { startResult <- runtime.StartWorkers(context.Background()) }() + <-worker.startEntered + + var firstCalls, replacementCalls int + registrationRuntime := runtime.WithContext(context.Background()) + registrationRuntime.Register("job:late", func(context.Context, Job) error { + firstCalls++ + return nil + }) + handler := worker.registered["job:late"] + if handler == nil { + t.Fatal("Register returned while the live backend still lacked the new handler") + } + if err := handler(context.Background(), NewJob("job:late")); err != nil { + t.Fatalf("invoke late handler: %v", err) + } + registrationRuntime.Register("job:late", func(context.Context, Job) error { + replacementCalls++ + return nil + }) + if err := handler(context.Background(), NewJob("job:late")); err != nil { + t.Fatalf("invoke replacement handler: %v", err) + } + if firstCalls != 1 || replacementCalls != 1 { + t.Fatalf("late handler calls = first:%d replacement:%d, want 1/1", firstCalls, replacementCalls) + } + if registrations := worker.registrations["job:late"]; registrations != 1 { + t.Fatalf("late backend registrations = %d, want 1", registrations) + } + for _, jobType := range []string{"job:late:second", "job:late:third"} { + registrationRuntime.Register(jobType, func(context.Context, Job) error { return nil }) + if registrations := worker.registrations[jobType]; registrations != 1 { + t.Fatalf("backend registrations for %q = %d, want 1", jobType, registrations) + } + } + + close(worker.releaseStart) + err := <-startResult + if test.startFails { + if !errors.Is(err, startErr) { + t.Fatalf("first start error = %v, want %v", err, startErr) + } + worker.startErr = nil + if err := runtime.StartWorkers(context.Background()); err != nil { + t.Fatalf("retry start: %v", err) + } + } else if err != nil { + t.Fatalf("start workers: %v", err) + } + if registrations := worker.registrations["job:late"]; registrations != 1 { + t.Fatalf("late backend registrations after start retry = %d, want 1", registrations) + } + if err := runtime.Shutdown(context.Background()); err != nil { + t.Fatalf("shutdown: %v", err) + } + }) + } +} + +// TestRuntimeConcurrentStartWaiterPreservesLateRegistration verifies a canceled waiter cannot create or disturb the active startup generation. +func TestRuntimeConcurrentStartWaiterPreservesLateRegistration(t *testing.T) { + for _, external := range []bool{false, true} { + name := "native" + if external { + name = "external" + } + t.Run(name, func(t *testing.T) { + worker := &blockingStrictRegistrationRuntimeBackendStub{ + startEntered: make(chan struct{}), + releaseStart: make(chan struct{}), + } + var runtime queueRuntime + if external { + runtime = &externalQueueRuntime{ + common: &queueCommon{inner: &queueBackendRecorder{}, cfg: Config{DefaultQueue: "default"}, driver: DriverSQS}, + newWorker: func(int) (driverWorkerBackend, error) { + return worker, nil + }, + externalQueueRuntimeState: &externalQueueRuntimeState{registered: map[string]Handler{}}, + } + } else { + runtime = &nativeQueueRuntime{ + common: &queueCommon{inner: worker, cfg: Config{DefaultQueue: "default"}, driver: DriverSync}, + runtime: worker, + nativeQueueRuntimeState: &nativeQueueRuntimeState{ + registered: map[string]Handler{}, + }, + } + } + + firstResult := make(chan error, 1) + go func() { firstResult <- runtime.StartWorkers(context.Background()) }() + <-worker.startEntered + secondCtx, cancelSecond := context.WithCancel(context.Background()) + secondResult := make(chan error, 1) + go func() { + secondResult <- runtime.StartWorkers(secondCtx) + }() + cancelSecond() + if err := <-secondResult; !errors.Is(err, context.Canceled) { + t.Fatalf("concurrent start waiter error = %v, want context canceled", err) + } + runtime.Register("job:shared-start", func(context.Context, Job) error { return nil }) + if worker.registered["job:shared-start"] == nil { + t.Fatal("late registration was absent from the shared startup generation") + } + close(worker.releaseStart) + if err := <-firstResult; err != nil { + t.Fatalf("first start: %v", err) + } + if worker.startCalls != 1 { + t.Fatalf("backend start calls = %d, want 1", worker.startCalls) + } + if registrations := worker.registrations["job:shared-start"]; registrations != 1 { + t.Fatalf("shared-start backend registrations = %d, want 1", registrations) + } + if err := runtime.Shutdown(context.Background()); err != nil { + t.Fatalf("shutdown: %v", err) + } + }) + } +} + +// TestExternalRuntimeRegistrationDuringWorkerConstructionIsInstalledBeforeStart verifies factory latency cannot exclude a completed registration from startup. +func TestExternalRuntimeRegistrationDuringWorkerConstructionIsInstalledBeforeStart(t *testing.T) { + worker := &blockingStrictRegistrationRuntimeBackendStub{ + startEntered: make(chan struct{}), + releaseStart: make(chan struct{}), + } + factoryEntered := make(chan struct{}) + releaseFactory := make(chan struct{}) + runtime := &externalQueueRuntime{ + common: &queueCommon{inner: &queueBackendRecorder{}, cfg: Config{DefaultQueue: "default"}, driver: DriverSQS}, + newWorker: func(int) (driverWorkerBackend, error) { + close(factoryEntered) + <-releaseFactory + return worker, nil + }, + externalQueueRuntimeState: &externalQueueRuntimeState{registered: map[string]Handler{}}, + } + + startResult := make(chan error, 1) + go func() { startResult <- runtime.StartWorkers(context.Background()) }() + <-factoryEntered + runtime.Register("job:during-factory", func(context.Context, Job) error { return nil }) + runtime.Register("job:during-factory", nil) + close(releaseFactory) + <-worker.startEntered + if worker.registered["job:during-factory"] == nil { + t.Fatal("registration completed during worker construction but was absent when the worker started") + } + if registrations := worker.registrations["job:during-factory"]; registrations != 1 { + t.Fatalf("factory-window backend registrations = %d, want 1", registrations) + } + close(worker.releaseStart) + if err := <-startResult; err != nil { + t.Fatalf("start workers: %v", err) + } + if err := runtime.Shutdown(context.Background()); err != nil { + t.Fatalf("shutdown: %v", err) + } +} + +// TestNativeRuntimeShutdownRetainsStateForPublicRetry verifies a failed drain cannot make later cleanup a no-op. +func TestNativeRuntimeShutdownRetainsStateForPublicRetry(t *testing.T) { + shutdownErr := errors.New("native shutdown timed out") + backend := &runtimeBackendStub{stopErr: shutdownErr} + runtime := &nativeQueueRuntime{ + common: &queueCommon{inner: backend, cfg: Config{DefaultQueue: "default"}, driver: DriverSync}, + runtime: backend, + nativeQueueRuntimeState: &nativeQueueRuntimeState{ + registered: map[string]Handler{}, + started: true, + }, + } + publicQueue, err := newQueueFromRuntime(runtime) + if err != nil { + t.Fatalf("new public queue: %v", err) + } + + if err := publicQueue.Shutdown(context.Background()); !errors.Is(err, shutdownErr) { + t.Fatalf("first shutdown error = %v, want %v", err, shutdownErr) + } + if !runtime.started || !runtime.draining { + t.Fatalf("native runtime lost retryable state: started=%t draining=%t", runtime.started, runtime.draining) + } + if err := publicQueue.StartWorkers(context.Background()); !errors.Is(err, ErrQueuerShuttingDown) { + t.Fatalf("start during drain error = %v, want ErrQueuerShuttingDown", err) + } + + backend.stopErr = nil + if err := publicQueue.Shutdown(context.Background()); err != nil { + t.Fatalf("retry shutdown: %v", err) + } + if backend.stopCalls != 2 { + t.Fatalf("native shutdown calls = %d, want 2", backend.stopCalls) + } + if runtime.started || runtime.draining { + t.Fatalf("native runtime remained active: started=%t draining=%t", runtime.started, runtime.draining) + } +} + +// TestExternalRuntimeShutdownRetainsWorkerForPublicRetry verifies worker and producer cleanup preserve their ordering after timeout. +func TestExternalRuntimeShutdownRetainsWorkerForPublicRetry(t *testing.T) { + shutdownErr := errors.New("worker shutdown timed out") + inner := &queueBackendRecorder{} + worker := &runtimeBackendStub{stopErr: shutdownErr} + runtime := &externalQueueRuntime{ + common: &queueCommon{inner: inner, cfg: Config{DefaultQueue: "default"}, driver: DriverSQS}, + externalQueueRuntimeState: &externalQueueRuntimeState{ + registered: map[string]Handler{}, + worker: worker, + started: true, + }, + } + publicQueue, err := newQueueFromRuntime(runtime) + if err != nil { + t.Fatalf("new public queue: %v", err) + } + + if err := publicQueue.Shutdown(context.Background()); !errors.Is(err, shutdownErr) { + t.Fatalf("first shutdown error = %v, want %v", err, shutdownErr) + } + if runtime.worker != worker || !runtime.started || !runtime.draining { + t.Fatalf("external runtime lost retryable state: worker=%T started=%t draining=%t", runtime.worker, runtime.started, runtime.draining) + } + if inner.shutdowns != 0 { + t.Fatalf("producer shutdowns = %d before worker drain, want 0", inner.shutdowns) + } + if err := publicQueue.StartWorkers(context.Background()); !errors.Is(err, ErrQueuerShuttingDown) { + t.Fatalf("start during drain error = %v, want ErrQueuerShuttingDown", err) + } + + worker.stopErr = nil + if err := publicQueue.Shutdown(context.Background()); err != nil { + t.Fatalf("retry shutdown: %v", err) + } + if worker.stopCalls != 2 || inner.shutdowns != 1 { + t.Fatalf("worker/producer shutdown calls = %d/%d, want 2/1", worker.stopCalls, inner.shutdowns) + } + if runtime.worker != nil || runtime.started || runtime.draining { + t.Fatalf("external runtime retained completed state: worker=%T started=%t draining=%t", runtime.worker, runtime.started, runtime.draining) + } +} + +// TestNativeRuntimeShutdownClosesNeverStartedBackend verifies producer-owned resources do not depend on worker startup. +func TestNativeRuntimeShutdownClosesNeverStartedBackend(t *testing.T) { + backend := &runtimeBackendStub{} + runtime := &nativeQueueRuntime{ + common: &queueCommon{inner: backend, cfg: Config{DefaultQueue: "default"}, driver: DriverDatabase}, + runtime: backend, + nativeQueueRuntimeState: &nativeQueueRuntimeState{ + registered: map[string]Handler{}, + }, + } + if err := runtime.Shutdown(context.Background()); err != nil { + t.Fatalf("shutdown never-started runtime: %v", err) + } + if backend.stopCalls != 1 || !runtime.closed { + t.Fatalf("backend stops/closed = %d/%t, want 1/true", backend.stopCalls, runtime.closed) + } + if err := runtime.Shutdown(context.Background()); err != nil { + t.Fatalf("idempotent shutdown: %v", err) + } + if backend.stopCalls != 1 { + t.Fatalf("idempotent backend stops = %d, want 1", backend.stopCalls) + } + if err := runtime.StartWorkers(context.Background()); !errors.Is(err, ErrQueuerShuttingDown) { + t.Fatalf("start after shutdown error = %v, want ErrQueuerShuttingDown", err) + } + if err := runtime.Dispatch(NewJob("job:closed")); !errors.Is(err, ErrQueuerShuttingDown) { + t.Fatalf("dispatch after shutdown error = %v, want ErrQueuerShuttingDown", err) + } +} + +// TestExternalRuntimeShutdownLatchesIntentDuringStart verifies a blocked startup cannot admit work after shutdown begins. +func TestExternalRuntimeShutdownLatchesIntentDuringStart(t *testing.T) { + inner := &queueBackendRecorder{} + worker := &blockingRuntimeBackendStub{ + startEntered: make(chan struct{}), + releaseStart: make(chan struct{}), + } + var factoryCalls int + runtime := &externalQueueRuntime{ + common: &queueCommon{inner: inner, cfg: Config{DefaultQueue: "default"}, driver: DriverSQS}, + newWorker: func(int) (driverWorkerBackend, error) { + factoryCalls++ + return worker, nil + }, + externalQueueRuntimeState: &externalQueueRuntimeState{registered: map[string]Handler{}}, + } + + startResult := make(chan error, 1) + go func() { startResult <- runtime.StartWorkers(context.Background()) }() + <-worker.startEntered + runtime.Register("job:before-startup-drain", func(context.Context, Job) error { return nil }) + if worker.registered["job:before-startup-drain"] == nil { + t.Fatal("pre-drain registration was absent during external startup") + } + shutdownResult := make(chan error, 1) + go func() { shutdownResult <- runtime.Shutdown(context.Background()) }() + waitForRuntimeDraining(t, func() bool { + runtime.mu.Lock() + defer runtime.mu.Unlock() + return runtime.draining + }) + runtime.Register("job:after-startup-drain", func(context.Context, Job) error { return nil }) + if worker.registered["job:after-startup-drain"] != nil { + t.Fatal("post-drain registration reached the external worker") + } + if err := runtime.Dispatch(NewJob("job:rejected").OnQueue("default")); !errors.Is(err, ErrQueuerShuttingDown) { + t.Fatalf("dispatch during startup drain = %v, want ErrQueuerShuttingDown", err) + } + if err := runtime.StartWorkers(context.Background()); !errors.Is(err, ErrQueuerShuttingDown) { + t.Fatalf("fresh start during startup drain = %v, want ErrQueuerShuttingDown", err) + } + close(worker.releaseStart) + if err := <-startResult; err != nil { + t.Fatalf("original start: %v", err) + } + if err := <-shutdownResult; err != nil { + t.Fatalf("shutdown racing start: %v", err) + } + if factoryCalls != 1 || worker.startCalls != 1 || worker.stopCalls != 1 || inner.shutdowns != 1 { + t.Fatalf("factory/start/stop/producer calls = %d/%d/%d/%d, want 1/1/1/1", factoryCalls, worker.startCalls, worker.stopCalls, inner.shutdowns) + } + if runtime.worker != nil || runtime.started || runtime.draining || !runtime.closed { + t.Fatalf("runtime lifecycle after shutdown = worker:%T started:%t draining:%t closed:%t", runtime.worker, runtime.started, runtime.draining, runtime.closed) + } +} + +// TestNativeRuntimeShutdownLatchesIntentDuringStart verifies native startup uses the same shutdown gate. +func TestNativeRuntimeShutdownLatchesIntentDuringStart(t *testing.T) { + worker := &blockingRuntimeBackendStub{ + startEntered: make(chan struct{}), + releaseStart: make(chan struct{}), + } + runtime := &nativeQueueRuntime{ + common: &queueCommon{inner: worker, cfg: Config{DefaultQueue: "default"}, driver: DriverSync}, + runtime: worker, + nativeQueueRuntimeState: &nativeQueueRuntimeState{ + registered: map[string]Handler{}, + }, + } + startResult := make(chan error, 1) + go func() { startResult <- runtime.StartWorkers(context.Background()) }() + <-worker.startEntered + runtime.Register("job:before-startup-drain", func(context.Context, Job) error { return nil }) + if worker.registered["job:before-startup-drain"] == nil { + t.Fatal("pre-drain registration was absent during native startup") + } + shutdownResult := make(chan error, 1) + go func() { shutdownResult <- runtime.Shutdown(context.Background()) }() + waitForRuntimeDraining(t, func() bool { + runtime.mu.Lock() + defer runtime.mu.Unlock() + return runtime.draining + }) + runtime.Register("job:after-startup-drain", func(context.Context, Job) error { return nil }) + if worker.registered["job:after-startup-drain"] != nil { + t.Fatal("post-drain registration reached the native backend") + } + if err := runtime.Dispatch(NewJob("job:rejected")); !errors.Is(err, ErrQueuerShuttingDown) { + t.Fatalf("dispatch during startup drain = %v, want ErrQueuerShuttingDown", err) + } + if err := runtime.StartWorkers(context.Background()); !errors.Is(err, ErrQueuerShuttingDown) { + t.Fatalf("fresh start during startup drain = %v, want ErrQueuerShuttingDown", err) + } + close(worker.releaseStart) + if err := <-startResult; err != nil { + t.Fatalf("original start: %v", err) + } + if err := <-shutdownResult; err != nil { + t.Fatalf("shutdown racing start: %v", err) + } + if worker.startCalls != 1 || worker.stopCalls != 1 { + t.Fatalf("native start/stop calls = %d/%d, want 1/1", worker.startCalls, worker.stopCalls) + } + if runtime.started || runtime.draining || !runtime.closed { + t.Fatalf("native lifecycle after shutdown = started:%t draining:%t closed:%t", runtime.started, runtime.draining, runtime.closed) + } +} + +// TestExternalRuntimeRetainsFailedStartWorkerForCleanup verifies partial factory resources remain reachable by Shutdown. +func TestExternalRuntimeRetainsFailedStartWorkerForCleanup(t *testing.T) { + startErr := errors.New("worker start failed") + worker := &runtimeBackendStub{startErr: startErr} + inner := &queueBackendRecorder{} + runtime := &externalQueueRuntime{ + common: &queueCommon{inner: inner, cfg: Config{DefaultQueue: "default"}, driver: DriverSQS}, + newWorker: func(int) (driverWorkerBackend, error) { + return worker, nil + }, + externalQueueRuntimeState: &externalQueueRuntimeState{registered: map[string]Handler{}}, + } + if err := runtime.StartWorkers(context.Background()); !errors.Is(err, startErr) { + t.Fatalf("start error = %v, want %v", err, startErr) + } + if runtime.worker == nil || runtime.started { + t.Fatalf("failed-start ownership = worker:%T started:%t", runtime.worker, runtime.started) + } + if err := runtime.Shutdown(context.Background()); err != nil { + t.Fatalf("shutdown failed-start worker: %v", err) + } + if worker.stopCalls != 1 || inner.shutdowns != 1 || runtime.worker != nil || !runtime.closed { + t.Fatalf("cleanup = worker stops:%d producer stops:%d retained:%T closed:%t", worker.stopCalls, inner.shutdowns, runtime.worker, runtime.closed) + } +} + +// TestExternalRuntimeRetryPreservesSameKeyReplacement verifies a retained worker exposes the latest handler without duplicate registration. +func TestExternalRuntimeRetryPreservesSameKeyReplacement(t *testing.T) { + worker := &strictRegistrationRuntimeBackendStub{} + var factoryCalls int + runtime := &externalQueueRuntime{ + common: &queueCommon{inner: &queueBackendRecorder{}, cfg: Config{DefaultQueue: "default"}, driver: DriverRedis}, + newWorker: func(int) (driverWorkerBackend, error) { + factoryCalls++ + return worker, nil + }, + externalQueueRuntimeState: &externalQueueRuntimeState{registered: map[string]Handler{}}, + } + t.Cleanup(func() { + if err := runtime.Shutdown(context.Background()); err != nil { + t.Errorf("shutdown retried runtime: %v", err) + } + }) + var firstCalls, secondCalls int + runtime.Register("job:replace", func(context.Context, Job) error { + firstCalls++ + return nil + }) + canceled, cancel := context.WithCancel(context.Background()) + cancel() + if err := runtime.StartWorkers(canceled); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled start error = %v, want context.Canceled", err) + } + runtime.Register("job:replace", func(context.Context, Job) error { + secondCalls++ + return nil + }) + if err := runtime.StartWorkers(context.Background()); err != nil { + t.Fatalf("retry start: %v", err) + } + if factoryCalls != 1 { + t.Fatalf("worker factory calls = %d, want retained worker reused once", factoryCalls) + } + if worker.registrations["job:replace"] != 1 { + t.Fatalf("worker registrations = %d, want 1", worker.registrations["job:replace"]) + } + if err := worker.registered["job:replace"](context.Background(), NewJob("job:replace")); err != nil { + t.Fatalf("invoke replacement: %v", err) + } + if firstCalls != 0 || secondCalls != 1 { + t.Fatalf("replacement calls = first:%d second:%d, want 0/1", firstCalls, secondCalls) + } +} + +// TestExternalRuntimeStartedReplacementUsesOneRegistration verifies strict workers never receive a duplicate pattern after startup. +func TestExternalRuntimeStartedReplacementUsesOneRegistration(t *testing.T) { + worker := &strictRegistrationRuntimeBackendStub{} + runtime := &externalQueueRuntime{ + common: &queueCommon{inner: &queueBackendRecorder{}, cfg: Config{DefaultQueue: "default"}, driver: DriverRedis}, + newWorker: func(int) (driverWorkerBackend, error) { + return worker, nil + }, + externalQueueRuntimeState: &externalQueueRuntimeState{registered: map[string]Handler{}}, + } + t.Cleanup(func() { + if err := runtime.Shutdown(context.Background()); err != nil { + t.Errorf("shutdown started runtime: %v", err) + } + }) + var firstCalls, secondCalls int + runtime.Register("job:replace", func(context.Context, Job) error { + firstCalls++ + return nil + }) + if err := runtime.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + runtime.Register("job:replace", func(context.Context, Job) error { + secondCalls++ + return nil + }) + if worker.registrations["job:replace"] != 1 { + t.Fatalf("worker registrations = %d, want 1", worker.registrations["job:replace"]) + } + if err := worker.registered["job:replace"](context.Background(), NewJob("job:replace")); err != nil { + t.Fatalf("invoke replacement: %v", err) + } + if firstCalls != 0 || secondCalls != 1 { + t.Fatalf("replacement calls = first:%d second:%d, want 0/1", firstCalls, secondCalls) + } +} + +// TestExternalRuntimeDoesNotRedrainWorkerAfterProducerFailure verifies retry resumes at the incomplete cleanup phase. +func TestExternalRuntimeDoesNotRedrainWorkerAfterProducerFailure(t *testing.T) { + producerErr := errors.New("producer shutdown failed") + worker := &runtimeBackendStub{} + inner := &queueBackendRecorder{shutdownErr: producerErr} + runtime := &externalQueueRuntime{ + common: &queueCommon{inner: inner, cfg: Config{DefaultQueue: "default"}, driver: DriverSQS}, + externalQueueRuntimeState: &externalQueueRuntimeState{ + registered: map[string]Handler{}, + worker: worker, + started: true, + }, + } + if err := runtime.Shutdown(context.Background()); !errors.Is(err, producerErr) { + t.Fatalf("first shutdown error = %v, want %v", err, producerErr) + } + if worker.stopCalls != 1 || runtime.worker != nil || runtime.started || !runtime.draining { + t.Fatalf("partial cleanup = worker stops:%d retained:%T started:%t draining:%t", worker.stopCalls, runtime.worker, runtime.started, runtime.draining) + } + inner.shutdownErr = nil + if err := runtime.Shutdown(context.Background()); err != nil { + t.Fatalf("retry producer shutdown: %v", err) + } + if worker.stopCalls != 1 || inner.shutdowns != 2 || !runtime.closed { + t.Fatalf("retry cleanup = worker stops:%d producer stops:%d closed:%t", worker.stopCalls, inner.shutdowns, runtime.closed) + } +} + +// TestRuntimeShutdownWaitsForDispatchLease verifies cleanup honors its deadline without overtaking an accepted producer operation. +func TestRuntimeShutdownWaitsForDispatchLease(t *testing.T) { + tests := []struct { + name string + construct func(backend *runtimeBackendStub, producer *queueBackendRecorder) queueRuntime + shutdowns func(backend *runtimeBackendStub, producer *queueBackendRecorder) int + }{ + { + name: "native", + construct: func(backend *runtimeBackendStub, _ *queueBackendRecorder) queueRuntime { + return &nativeQueueRuntime{ + common: &queueCommon{inner: backend, cfg: Config{DefaultQueue: "default"}, driver: DriverSync}, + runtime: backend, + nativeQueueRuntimeState: &nativeQueueRuntimeState{ + registered: map[string]Handler{}, + }, + } + }, + shutdowns: func(backend *runtimeBackendStub, _ *queueBackendRecorder) int { return backend.stopCalls }, + }, + { + name: "external", + construct: func(_ *runtimeBackendStub, producer *queueBackendRecorder) queueRuntime { + return &externalQueueRuntime{ + common: &queueCommon{inner: producer, cfg: Config{DefaultQueue: "default"}, driver: DriverSQS}, + externalQueueRuntimeState: &externalQueueRuntimeState{ + registered: map[string]Handler{}, + }, + } + }, + shutdowns: func(_ *runtimeBackendStub, producer *queueBackendRecorder) int { return producer.shutdowns }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + entered := make(chan struct{}) + release := make(chan struct{}) + backend := &runtimeBackendStub{dispatchEntered: entered, releaseDispatch: release} + producer := &queueBackendRecorder{dispatchEntered: entered, releaseDispatch: release} + runtime := test.construct(backend, producer) + dispatchResult := make(chan error, 1) + go func() { dispatchResult <- runtime.Dispatch(NewJob("job:leased")) }() + <-entered + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + defer cancel() + if err := runtime.Shutdown(ctx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("shutdown error = %v, want deadline exceeded", err) + } + if calls := test.shutdowns(backend, producer); calls != 0 { + t.Fatalf("backend shutdown overtook dispatch: calls=%d", calls) + } + close(release) + if err := <-dispatchResult; err != nil { + t.Fatalf("dispatch: %v", err) + } + if err := runtime.Shutdown(context.Background()); err != nil { + t.Fatalf("retry shutdown: %v", err) + } + if calls := test.shutdowns(backend, producer); calls != 1 { + t.Fatalf("backend shutdown calls = %d, want 1", calls) + } + }) + } +} + +// TestExternalRuntimeShutdownWaitsForLateContinuationLease verifies a descendant admitted during worker drain finishes before producer cleanup. +func TestExternalRuntimeShutdownWaitsForLateContinuationLease(t *testing.T) { + worker := &blockingShutdownRuntimeBackendStub{ + shutdownEntered: make(chan struct{}), + releaseShutdown: make(chan struct{}), + } + producer := &queueBackendRecorder{ + dispatchEntered: make(chan struct{}), + releaseDispatch: make(chan struct{}), + } + runtime := &externalQueueRuntime{ + common: &queueCommon{inner: producer, cfg: Config{DefaultQueue: "default"}, driver: DriverSQS}, + externalQueueRuntimeState: &externalQueueRuntimeState{ + registered: map[string]Handler{}, + worker: worker, + started: true, + continuation: busruntime.NewContinuationScope(), + }, + } + + shutdownCtx, cancelShutdown := context.WithCancel(context.Background()) + shutdownResult := make(chan error, 1) + go func() { shutdownResult <- runtime.Shutdown(shutdownCtx) }() + <-worker.shutdownEntered + + continuationCtx, releaseContinuation := runtime.continuationScope().Permit(context.Background()) + dispatchResult := make(chan error, 1) + go func() { + dispatchResult <- runtime.WithContext(continuationCtx).Dispatch(NewJob("job:late-continuation")) + }() + <-producer.dispatchEntered + // Handler return expires its permit, but the operation it admitted still owns the producer until Dispatch returns. + releaseContinuation() + cancelShutdown() + close(worker.releaseShutdown) + + if err := <-shutdownResult; !errors.Is(err, context.Canceled) { + t.Fatalf("shutdown error = %v, want context canceled while continuation is active", err) + } + if producer.shutdowns != 0 { + t.Fatalf("producer shutdown overtook late continuation: calls=%d", producer.shutdowns) + } + + close(producer.releaseDispatch) + if err := <-dispatchResult; err != nil { + t.Fatalf("late continuation dispatch: %v", err) + } + if err := runtime.Shutdown(context.Background()); err != nil { + t.Fatalf("retry shutdown: %v", err) + } + if worker.stopCalls != 1 || producer.shutdowns != 1 { + t.Fatalf("worker/producer shutdown calls = %d/%d, want 1/1", worker.stopCalls, producer.shutdowns) + } +} + +// TestNativeRuntimeShutdownWaitsForLateContinuationBeforeResourceClose verifies +// native cleanup takes a stable post-drain lease snapshot before closing resources. +func TestNativeRuntimeShutdownWaitsForLateContinuationBeforeResourceClose(t *testing.T) { + backend := &phasedShutdownRuntimeBackendStub{ + runtimeBackendStub: runtimeBackendStub{ + dispatchEntered: make(chan struct{}), + releaseDispatch: make(chan struct{}), + }, + drainEntered: make(chan struct{}), + releaseDrain: make(chan struct{}), + } + runtime := &nativeQueueRuntime{ + common: &queueCommon{inner: backend, cfg: Config{DefaultQueue: "default"}, driver: DriverSync}, + runtime: backend, + nativeQueueRuntimeState: &nativeQueueRuntimeState{ + registered: map[string]Handler{}, + started: true, + continuation: busruntime.NewContinuationScope(), + }, + } + + shutdownCtx, cancelShutdown := context.WithCancel(context.Background()) + shutdownResult := make(chan error, 1) + go func() { shutdownResult <- runtime.Shutdown(shutdownCtx) }() + <-backend.drainEntered + + continuationCtx, releaseContinuation := runtime.continuationScope().Permit(context.Background()) + dispatchResult := make(chan error, 1) + go func() { + dispatchResult <- runtime.WithContext(continuationCtx).Dispatch(NewJob("job:late-native-continuation")) + }() + <-backend.dispatchEntered + // The originating handler can return after admission while the dispatch + // lease continues to protect the backend resource on its behalf. + releaseContinuation() + close(backend.releaseDrain) + cancelShutdown() + + if err := <-shutdownResult; !errors.Is(err, context.Canceled) { + t.Fatalf("shutdown error = %v, want context canceled while continuation is active", err) + } + if backend.stopCalls != 0 { + t.Fatalf("resource close overtook late continuation: calls=%d", backend.stopCalls) + } + + close(backend.releaseDispatch) + if err := <-dispatchResult; err != nil { + t.Fatalf("late continuation dispatch: %v", err) + } + if err := runtime.Shutdown(context.Background()); err != nil { + t.Fatalf("retry shutdown: %v", err) + } + if backend.stopCalls != 1 { + t.Fatalf("resource close calls = %d, want 1", backend.stopCalls) + } +} + +// TestRuntimeShutdownWaitsForReadinessLease verifies readiness cannot reopen or outlive producer cleanup. +func TestRuntimeShutdownWaitsForReadinessLease(t *testing.T) { + for _, external := range []bool{false, true} { + name := "native" + if external { + name = "external" + } + t.Run(name, func(t *testing.T) { + backend := &blockingReadyRuntimeBackendStub{ + readyEntered: make(chan struct{}), + releaseReady: make(chan struct{}), + } + var runtime queueRuntime + if external { + runtime = &externalQueueRuntime{ + common: &queueCommon{inner: backend, cfg: Config{DefaultQueue: "default"}, driver: DriverNATS}, + externalQueueRuntimeState: &externalQueueRuntimeState{ + registered: map[string]Handler{}, + }, + } + } else { + runtime = &nativeQueueRuntime{ + common: &queueCommon{inner: backend, cfg: Config{DefaultQueue: "default"}, driver: DriverSync}, + runtime: backend, + nativeQueueRuntimeState: &nativeQueueRuntimeState{ + registered: map[string]Handler{}, + }, + } + } + readyResult := make(chan error, 1) + go func() { readyResult <- runtime.Ready(context.Background()) }() + <-backend.readyEntered + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond) + if err := runtime.Shutdown(ctx); !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("shutdown error = %v, want deadline exceeded", err) + } + cancel() + if backend.stopCalls != 0 { + t.Fatalf("backend shutdown overtook readiness: calls=%d", backend.stopCalls) + } + if err := runtime.Ready(context.Background()); !errors.Is(err, ErrQueuerShuttingDown) { + t.Fatalf("ready during drain = %v, want ErrQueuerShuttingDown", err) + } + close(backend.releaseReady) + if err := <-readyResult; err != nil { + t.Fatalf("readiness operation: %v", err) + } + if err := runtime.Shutdown(context.Background()); err != nil { + t.Fatalf("retry shutdown: %v", err) + } + if err := runtime.Ready(context.Background()); !errors.Is(err, ErrQueuerShuttingDown) { + t.Fatalf("ready after close = %v, want ErrQueuerShuttingDown", err) + } + if backend.readyCalls != 1 { + t.Fatalf("backend readiness calls = %d, want 1", backend.readyCalls) + } + }) + } +} + +// TestRuntimeContinuationPermissionIsScopedAndEphemeral verifies foreign or escaped handler contexts cannot bypass a drain. +func TestRuntimeContinuationPermissionIsScopedAndEphemeral(t *testing.T) { + backend := &runtimeBackendStub{} + runtime := &nativeQueueRuntime{ + common: &queueCommon{inner: backend, cfg: Config{DefaultQueue: "default"}, driver: DriverSync}, + runtime: backend, + nativeQueueRuntimeState: &nativeQueueRuntimeState{ + registered: map[string]Handler{}, + draining: true, + }, + } + foreign := busruntime.NewContinuationScope() + foreignCtx, releaseForeign := foreign.Permit(context.Background()) + defer releaseForeign() + if err := runtime.WithContext(foreignCtx).Dispatch(NewJob("job:foreign")); !errors.Is(err, ErrQueuerShuttingDown) { + t.Fatalf("foreign continuation dispatch = %v, want ErrQueuerShuttingDown", err) + } + + ownCtx, releaseOwn := runtime.continuationScope().Permit(context.Background()) + if err := runtime.WithContext(ownCtx).Dispatch(NewJob("job:owned")); err != nil { + t.Fatalf("owned continuation dispatch: %v", err) + } + releaseOwn() + if err := runtime.WithContext(ownCtx).Dispatch(NewJob("job:escaped")); !errors.Is(err, ErrQueuerShuttingDown) { + t.Fatalf("escaped continuation dispatch = %v, want ErrQueuerShuttingDown", err) + } +} + func TestQueueCommon_PauseResumeStatsUnsupported(t *testing.T) { common := &queueCommon{ inner: &queueBackendRecorder{}, @@ -281,25 +1408,37 @@ func TestRuntimeBusWrappers_NilRegisterAndDispatch(t *testing.T) { inner := &queueBackendRecorder{} nativeBackend := &runtimeBackendStub{} native := &nativeQueueRuntime{ - common: &queueCommon{inner: inner, cfg: Config{DefaultQueue: "default"}, driver: DriverSync}, - runtime: nativeBackend, - registered: map[string]Handler{}, + common: &queueCommon{inner: inner, cfg: Config{DefaultQueue: "default"}, driver: DriverSync}, + runtime: nativeBackend, + nativeQueueRuntimeState: &nativeQueueRuntimeState{ + registered: map[string]Handler{}, + }, } externalWorker := &runtimeBackendStub{} external := &externalQueueRuntime{ - common: &queueCommon{inner: inner, cfg: Config{DefaultQueue: "default"}, driver: DriverNATS}, - registered: map[string]Handler{}, - worker: externalWorker, - started: true, + common: &queueCommon{inner: inner, cfg: Config{DefaultQueue: "default"}, driver: DriverNATS}, + externalQueueRuntimeState: &externalQueueRuntimeState{ + registered: map[string]Handler{}, + worker: externalWorker, + started: true, + }, } native.BusRegister("job:nil:native", nil) external.BusRegister("job:nil:external", nil) - if _, ok := native.registered["job:nil:native"]; !ok { - t.Fatal("expected native BusRegister(nil) to store registration") + native.Register("", func(context.Context, Job) error { return nil }) + external.Register("", func(context.Context, Job) error { return nil }) + if _, ok := native.registered["job:nil:native"]; ok { + t.Fatal("native BusRegister(nil) mutated logical registrations") + } + if _, ok := externalWorker.registered["job:nil:external"]; ok { + t.Fatal("external BusRegister(nil) reached the worker") + } + if _, ok := native.registered[""]; ok { + t.Fatal("native empty registration mutated logical registrations") } - if h, ok := externalWorker.registered["job:nil:external"]; !ok || h != nil { - t.Fatal("expected external BusRegister(nil) to forward nil handler") + if _, ok := externalWorker.registered[""]; ok { + t.Fatal("external empty registration reached the worker") } opts := busruntime.JobOptions{ @@ -327,12 +1466,73 @@ func TestRuntimeBusWrappers_NilRegisterAndDispatch(t *testing.T) { } } +// TestRuntimeBusRegisterPropagatesDeliveryAttempt verifies native and external adapters expose physical retry metadata to orchestration. +func TestRuntimeBusRegisterPropagatesDeliveryAttempt(t *testing.T) { + tests := []struct { + name string + register func(string, busruntime.Handler) Handler + }{ + { + name: "native", + register: func(jobType string, handler busruntime.Handler) Handler { + native := &nativeQueueRuntime{ + common: &queueCommon{driver: DriverSync}, + runtime: &runtimeBackendStub{}, + nativeQueueRuntimeState: &nativeQueueRuntimeState{ + registered: map[string]Handler{}, + }, + } + native.BusRegister(jobType, handler) + return native.registered[jobType] + }, + }, + { + name: "external", + register: func(jobType string, handler busruntime.Handler) Handler { + external := &externalQueueRuntime{ + common: &queueCommon{driver: DriverNATS}, + externalQueueRuntimeState: &externalQueueRuntimeState{ + registered: map[string]Handler{}, + }, + } + external.BusRegister(jobType, handler) + return external.registered[jobType] + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + const jobType = "job:attempt" + var got busruntime.DeliveryAttempt + var ok bool + handler := test.register(jobType, func(ctx context.Context, _ busruntime.InboundJob) error { + got, ok = busruntime.DeliveryAttemptFromContext(ctx) + return nil + }) + if handler == nil { + t.Fatal("bus handler was not registered") + } + job := DriverWithAttempt(NewJob(jobType).Retry(4), 2) + if err := handler(context.Background(), job); err != nil { + t.Fatalf("invoke bus handler: %v", err) + } + want := busruntime.DeliveryAttempt{Number: 2, MaxRetry: 4} + if !ok || got != want { + t.Fatalf("delivery attempt = %+v, %t; want %+v, true", got, ok, want) + } + }) + } +} + func TestRuntimeBusDispatchPhysicalizesTargetQueue(t *testing.T) { inner := &queueBackendRecorder{} native := &nativeQueueRuntime{ - common: &queueCommon{inner: inner, cfg: Config{DefaultQueue: "billing_default"}, driver: DriverSync}, - runtime: &runtimeBackendStub{}, - registered: map[string]Handler{}, + common: &queueCommon{inner: inner, cfg: Config{DefaultQueue: "billing_default"}, driver: DriverSync}, + runtime: &runtimeBackendStub{}, + nativeQueueRuntimeState: &nativeQueueRuntimeState{ + registered: map[string]Handler{}, + }, } if err := native.BusDispatch(context.Background(), "job:native", []byte(`{"n":1}`), busruntime.JobOptions{Queue: "reports"}); err != nil { @@ -346,6 +1546,28 @@ func TestRuntimeBusDispatchPhysicalizesTargetQueue(t *testing.T) { } } +// TestRuntimeBusDispatchPreservesZeroRetry verifies backend defaults cannot replace workflow policy. +func TestRuntimeBusDispatchPreservesZeroRetry(t *testing.T) { + inner := &queueBackendRecorder{} + native := &nativeQueueRuntime{ + common: &queueCommon{inner: inner, cfg: Config{DefaultQueue: "default"}, driver: DriverSync}, + runtime: &runtimeBackendStub{}, + nativeQueueRuntimeState: &nativeQueueRuntimeState{ + registered: map[string]Handler{}, + }, + } + if err := native.BusDispatch(context.Background(), "bus:job", []byte(`{"schema_version":1}`), busruntime.JobOptions{}); err != nil { + t.Fatalf("BusDispatch failed: %v", err) + } + if len(inner.dispatched) != 1 { + t.Fatalf("dispatched jobs = %d, want 1", len(inner.dispatched)) + } + maxRetry := inner.dispatched[0].jobOptions().maxRetry + if maxRetry == nil || *maxRetry != 0 { + t.Fatalf("max retry = %v, want explicit zero", maxRetry) + } +} + func TestDriverAdapters_PauseResumeStats_Branches(t *testing.T) { a := driverQueueBackendAdapter{&queueBackendRecorder{}} if err := a.Pause(context.Background(), "q"); !errors.Is(err, ErrPauseUnsupported) { @@ -401,7 +1623,12 @@ func TestExternalQueueRuntimePauseResumeStatsWrappers(t *testing.T) { }, } common := &queueCommon{inner: inner, cfg: Config{DefaultQueue: "default"}, driver: DriverNull} - q := &externalQueueRuntime{common: common, registered: map[string]Handler{}} + q := &externalQueueRuntime{ + common: common, + externalQueueRuntimeState: &externalQueueRuntimeState{ + registered: map[string]Handler{}, + }, + } if err := q.Pause(context.Background(), "default"); err != nil { t.Fatalf("pause failed: %v", err) @@ -443,9 +1670,11 @@ func TestNativeRuntimeStartWorkersErrorPath(t *testing.T) { inner := &queueBackendRecorder{} worker := &runtimeBackendStub{startErr: errors.New("start failed")} q := &nativeQueueRuntime{ - common: &queueCommon{inner: inner, cfg: Config{DefaultQueue: "default"}, driver: DriverSync}, - runtime: worker, - registered: map[string]Handler{"job:one": func(context.Context, Job) error { return nil }}, + common: &queueCommon{inner: inner, cfg: Config{DefaultQueue: "default"}, driver: DriverSync}, + runtime: worker, + nativeQueueRuntimeState: &nativeQueueRuntimeState{ + registered: map[string]Handler{"job:one": func(context.Context, Job) error { return nil }}, + }, } if err := q.StartWorkers(context.Background()); err == nil { t.Fatal("expected start workers error") @@ -455,23 +1684,212 @@ func TestNativeRuntimeStartWorkersErrorPath(t *testing.T) { } } -func TestQueueCommonWrapRegisteredHandlerWithoutObserver(t *testing.T) { +// TestQueueCommonWrapRegisteredHandlerPreservesContextOnNilDecoration verifies a decorator can decline replacement without erasing the original context. +func TestQueueCommonWrapRegisteredHandlerPreservesContextOnNilDecoration(t *testing.T) { + type contextKey struct{} + key := contextKey{} + const want = "original" + + for _, withObserver := range []bool{false, true} { + name := "without observer" + if withObserver { + name = "with observer" + } + t.Run(name, func(t *testing.T) { + original := context.WithValue(context.Background(), key, want) + var observed int + observer := ensureObserverSink(nil) + if withObserver { + observer = ensureObserverSink(ObserverFunc(func(ctx context.Context, event Event) { + if event.Kind != EventProcessStarted && event.Kind != EventProcessSucceeded { + return + } + observed++ + if ctx != original { + t.Errorf("observer context changed after nil decorator result") + } + })) + } + decoratorCalls := 0 + common := &queueCommon{ + cfg: Config{Driver: DriverSync, Observer: observer}, + handlerContextDecorator: func(context.Context) context.Context { + decoratorCalls++ + return nil + }, + } + handlerCalls := 0 + wrapped := common.wrapRegisteredHandler("job:nil-decoration", func(ctx context.Context, _ Job) error { + handlerCalls++ + if ctx != original { + t.Error("handler context changed after nil decorator result") + } + if got, _ := ctx.Value(key).(string); got != want { + t.Errorf("handler context value = %q, want %q", got, want) + } + return nil + }) + if err := wrapped(original, NewJob("job:nil-decoration")); err != nil { + t.Fatalf("wrapped handler: %v", err) + } + if decoratorCalls != 1 || handlerCalls != 1 { + t.Fatalf("decorator/handler calls = %d/%d, want 1/1", decoratorCalls, handlerCalls) + } + wantObserved := 0 + if withObserver { + wantObserved = 2 + } + if observed != wantObserved { + t.Fatalf("observed process events = %d, want %d", observed, wantObserved) + } + }) + } + common := &queueCommon{cfg: Config{Driver: DriverSync}} - h := func(context.Context, Job) error { return nil } - if got := common.wrapRegisteredHandler("job:x", h); got == nil { - t.Fatal("expected non-nil passthrough handler") + if got := common.wrapRegisteredHandler("job:nil-handler", nil); got != nil { + t.Fatal("expected nil handler to remain nil") } - if got := common.wrapRegisteredHandler("job:x", nil); got != nil { - t.Fatal("expected nil passthrough handler") +} + +// TestRuntimeHandlerContextDecoratorNativeExternalParity verifies both runtime registration paths decorate handlers regardless of observer recipients. +func TestRuntimeHandlerContextDecoratorNativeExternalParity(t *testing.T) { + type contextKey struct{} + key := contextKey{} + const want = "decorated" + + for _, runtimeShape := range []string{"native", "external"} { + for _, withObserver := range []bool{false, true} { + name := runtimeShape + "/without observer" + if withObserver { + name = runtimeShape + "/with observer" + } + t.Run(name, func(t *testing.T) { + backend := &runtimeBackendStub{} + var observed int + observer := ensureObserverSink(nil) + if withObserver { + observer = ensureObserverSink(ObserverFunc(func(ctx context.Context, event Event) { + if event.Kind != EventProcessStarted && event.Kind != EventProcessSucceeded { + return + } + observed++ + if got, _ := ctx.Value(key).(string); got != want { + t.Errorf("observer context value = %q, want %q", got, want) + } + })) + } + + driver := DriverSync + common := &queueCommon{ + inner: backend, + cfg: Config{Driver: driver, DefaultQueue: "default", Observer: observer}, + driver: driver, + } + var runtime queueRuntime = &nativeQueueRuntime{ + common: common, + runtime: backend, + nativeQueueRuntimeState: &nativeQueueRuntimeState{ + registered: map[string]Handler{}, + }, + } + if runtimeShape == "external" { + driver = DriverSQS + common.inner = &queueBackendRecorder{} + common.cfg.Driver = driver + common.driver = driver + runtime = &externalQueueRuntime{ + common: common, + newWorker: func(int) (driverWorkerBackend, error) { + return backend, nil + }, + externalQueueRuntimeState: &externalQueueRuntimeState{ + registered: map[string]Handler{}, + }, + } + } + + decoratorCalls := 0 + runtime.setHandlerContextDecorator(func(ctx context.Context) context.Context { + decoratorCalls++ + return context.WithValue(ctx, key, want) + }) + handlerCalls := 0 + runtime.Register("job:parity", func(ctx context.Context, _ Job) error { + handlerCalls++ + if got, _ := ctx.Value(key).(string); got != want { + t.Errorf("handler context value = %q, want %q", got, want) + } + return nil + }) + if err := runtime.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + registered := backend.registered["job:parity"] + if registered == nil { + t.Fatal("backend did not receive registered handler") + } + if err := registered(context.Background(), NewJob("job:parity")); err != nil { + t.Fatalf("registered handler: %v", err) + } + if err := runtime.Shutdown(context.Background()); err != nil { + t.Fatalf("shutdown runtime: %v", err) + } + + if decoratorCalls != 1 || handlerCalls != 1 { + t.Fatalf("decorator/handler calls = %d/%d, want 1/1", decoratorCalls, handlerCalls) + } + wantObserved := 0 + if withObserver { + wantObserved = 2 + } + if observed != wantObserved { + t.Fatalf("observed process events = %d, want %d", observed, wantObserved) + } + }) + } } - common.cfg.Observer = ObserverFunc(func(context.Context, Event) {}) - if wrapped := common.wrapRegisteredHandler("job:x", h); wrapped == nil { - t.Fatal("expected wrapped handler") +} + +// TestQueueCommonWrapRegisteredHandlerDefersRedisDecoration verifies the shared wrapper leaves Redis's native handler boundary untouched. +func TestQueueCommonWrapRegisteredHandlerDefersRedisDecoration(t *testing.T) { + decoratorCalls := 0 + observerCalls := 0 + common := &queueCommon{ + cfg: Config{ + Driver: DriverRedis, + Observer: ObserverFunc(func(context.Context, Event) { + observerCalls++ + }), + }, + handlerContextDecorator: func(ctx context.Context) context.Context { + decoratorCalls++ + return context.WithValue(ctx, "decorated", true) + }, + } + handlerCalls := 0 + wrapped := common.wrapRegisteredHandler("job:redis", func(ctx context.Context, _ Job) error { + handlerCalls++ + if ctx.Value("decorated") != nil { + t.Fatal("shared wrapper decorated Redis handler context") + } + return nil + }) + if err := wrapped(context.Background(), NewJob("job:redis")); err != nil { + t.Fatalf("wrapped Redis handler: %v", err) + } + if decoratorCalls != 0 || observerCalls != 0 || handlerCalls != 1 { + t.Fatalf("decorator/observer/handler calls = %d/%d/%d, want 0/0/1", decoratorCalls, observerCalls, handlerCalls) } } func TestWorkersSetOnlyBeforeStartNative(t *testing.T) { - q := &nativeQueueRuntime{common: &queueCommon{cfg: Config{}}, runtime: &runtimeBackendStub{}} + q := &nativeQueueRuntime{ + common: &queueCommon{cfg: Config{}}, + runtime: &runtimeBackendStub{}, + nativeQueueRuntimeState: &nativeQueueRuntimeState{ + registered: map[string]Handler{}, + }, + } q.Workers(0) if q.workers != 0 { t.Fatalf("expected workers unchanged for non-positive, got %d", q.workers) @@ -503,8 +1921,10 @@ func TestQueueCommonPauseResumeStatsUnsupported(t *testing.T) { func TestExternalQueueRuntimeStartWorkersErrorBranches(t *testing.T) { t.Run("factory error for unsupported driver", func(t *testing.T) { q := &externalQueueRuntime{ - common: &queueCommon{inner: &queueBackendRecorder{}, cfg: Config{Driver: Driver("unknown")}, driver: Driver("unknown")}, - registered: map[string]Handler{}, + common: &queueCommon{inner: &queueBackendRecorder{}, cfg: Config{Driver: Driver("unknown")}, driver: Driver("unknown")}, + externalQueueRuntimeState: &externalQueueRuntimeState{ + registered: map[string]Handler{}, + }, } if err := q.StartWorkers(context.Background()); err == nil { t.Fatal("expected start workers error for unsupported driver") @@ -521,12 +1941,14 @@ func TestExternalQueueRuntimeStartWorkersErrorBranches(t *testing.T) { cfg: Config{Driver: DriverNATS}, driver: DriverNATS, }, - registered: map[string]Handler{ - "job:nats": func(context.Context, Job) error { return nil }, - }, newWorker: func(int) (driverWorkerBackend, error) { return nil, errors.New("dial failed") }, + externalQueueRuntimeState: &externalQueueRuntimeState{ + registered: map[string]Handler{ + "job:nats": func(context.Context, Job) error { return nil }, + }, + }, } if err := q.StartWorkers(context.Background()); err == nil { t.Fatal("expected start workers error for unreachable nats") diff --git a/queuecore/queuecore.go b/queuecore/queuecore.go index 88471f3..897ba19 100644 --- a/queuecore/queuecore.go +++ b/queuecore/queuecore.go @@ -21,6 +21,11 @@ func ValidateDriverJob(job queue.Job) error { return queue.ValidateDriverJob(job // DriverOptions returns normalized driver-facing job options for a queue.Job. func DriverOptions(job queue.Job) queue.DriverJobOptions { return queue.DriverOptions(job) } +// UniqueKey returns the queue-scoped logical identity used for driver deduplication. +func UniqueKey(job queue.Job, queueName string) string { + return queue.DriverUniqueKey(job, queueName) +} + // DriverWithAttempt annotates a queue.Job with an attempt count for worker // handler execution paths. func DriverWithAttempt(job queue.Job, attempt int) queue.Job { diff --git a/queuecore/unique_test.go b/queuecore/unique_test.go new file mode 100644 index 0000000..bb9db5c --- /dev/null +++ b/queuecore/unique_test.go @@ -0,0 +1,18 @@ +package queuecore + +import ( + "testing" + + "github.com/goforj/queue" +) + +// TestUniqueKeyDelegatesCanonicalIdentity verifies the driver bridge keeps queue scope intact. +func TestUniqueKeyDelegatesCanonicalIdentity(t *testing.T) { + directKey := UniqueKey(queue.NewJob("reports:build").Payload([]byte(`{"id":1}`)), "critical") + if directKey == "" || directKey != queue.DriverUniqueKey(queue.NewJob("reports:build").Payload([]byte(`{"id":1}`)), "critical") { + t.Fatalf("bridge key does not match root key: %q", directKey) + } + if directKey == UniqueKey(queue.NewJob("reports:build").Payload([]byte(`{"id":1}`)), "default") { + t.Fatalf("direct key must be stable and queue scoped: %q", directKey) + } +} diff --git a/queuefake/doc.go b/queuefake/doc.go index 306a27e..96ceb01 100644 --- a/queuefake/doc.go +++ b/queuefake/doc.go @@ -1,7 +1,5 @@ -// Package queuefake provides a queue-first test harness for queue and workflow assertions. +// Package queuefake preserves the historical queue-first testing harness. // -// It wraps queue.NewFake() for dispatch assertions and bus.NewFake() for workflow -// orchestration assertions so tests can stay aligned with the flattened queue API -// surface. +// Its queue and bus compatibility views now share one concurrency-safe +// queue.FakeQueue. New code should use queue.NewFake directly. package queuefake - diff --git a/queuefake/fake.go b/queuefake/fake.go index 74aca26..a061bd1 100644 --- a/queuefake/fake.go +++ b/queuefake/fake.go @@ -7,15 +7,19 @@ import ( "github.com/goforj/queue/bus" ) -// Fake exposes a queue test harness with assertion helpers for dispatched jobs. -// It wraps queue.NewFake() so tests can inject a queue fake without external services. +// Fake preserves the historical queuefake harness as two typed views over one +// canonical queue.FakeQueue state. +// +// Deprecated: use queue.NewFake directly. // @group Testing type Fake struct { q *queue.FakeQueue b *bus.Fake } -// New creates a fake queue harness backed by queue.NewFake(). +// New creates compatibility views backed by one canonical root fake. +// +// Deprecated: use queue.NewFake directly. // @group Testing // // Example: queuefake harness @@ -26,9 +30,10 @@ type Fake struct { // f.AssertDispatched(t, "emails:send") // f.AssertCount(t, 1) func New() *Fake { + workflow := bus.NewFake() return &Fake{ - q: queue.NewFake(), - b: bus.NewFake(), + q: workflow.Queue(), + b: workflow, } } @@ -42,7 +47,9 @@ func New() *Fake { // _ = q.Dispatch(queue.NewJob("emails:send").OnQueue("default")) func (f *Fake) Queue() *queue.FakeQueue { return f.q } -// Workflow returns the workflow/orchestration fake for chain/batch assertions. +// Workflow returns the deprecated bus view over the same state as Queue. +// +// Deprecated: call Queue().Chain or Queue().Batch. // @group Testing // // Example: workflow fake @@ -56,7 +63,8 @@ func (f *Fake) Queue() *queue.FakeQueue { return f.q } // f.AssertChained(t, []string{"a", "b"}) func (f *Fake) Workflow() *bus.Fake { return f.b } -// Reset clears recorded dispatches. +// Reset clears direct, chain, batch, and workflow-store state atomically from +// every compatibility view. // @group Testing // // Example: reset recorded dispatches @@ -130,7 +138,8 @@ func (f *Fake) CountOn(queueName, jobType string) int { return count } -// Workflow assertion wrappers (forwarded to bus.Fake) keep tests queuefake-first. +// Workflow assertion wrappers retain source compatibility while reading the +// same direct records as the root queue assertions. // AssertNothingWorkflowDispatched fails when any workflow dispatch was recorded. // @group Testing @@ -139,7 +148,10 @@ func (f *Fake) CountOn(queueName, jobType string) int { // // f := queuefake.New() // f.AssertNothingWorkflowDispatched(t) -func (f *Fake) AssertNothingWorkflowDispatched(t testing.TB) { f.b.AssertNothingDispatched(t) } +func (f *Fake) AssertNothingWorkflowDispatched(t testing.TB) { + t.Helper() + f.b.AssertNothingDispatched(t) +} // AssertWorkflowDispatched fails when jobType was not workflow-dispatched. // @group Testing @@ -147,9 +159,12 @@ func (f *Fake) AssertNothingWorkflowDispatched(t testing.TB) { f.b.AssertNothing // Example: assert workflow dispatch by type // // f := queuefake.New() -// _, _ = f.Workflow().Chain(bus.NewJob("a", nil)).Dispatch(nil) +// _, _ = f.Workflow().Dispatch(nil, bus.NewJob("a", nil)) // f.AssertWorkflowDispatched(t, "a") -func (f *Fake) AssertWorkflowDispatched(t testing.TB, jobType string) { f.b.AssertDispatched(t, jobType) } +func (f *Fake) AssertWorkflowDispatched(t testing.TB, jobType string) { + t.Helper() + f.b.AssertDispatched(t, jobType) +} // AssertWorkflowDispatchedOn fails when jobType was not workflow-dispatched on queueName. // @group Testing @@ -157,9 +172,10 @@ func (f *Fake) AssertWorkflowDispatched(t testing.TB, jobType string) { f.b.Asse // Example: assert workflow dispatch on queue // // f := queuefake.New() -// _, _ = f.Workflow().Chain(bus.NewJob("a", nil)).OnQueue("critical").Dispatch(nil) +// _, _ = f.Workflow().Dispatch(nil, bus.NewJob("a", nil).OnQueue("critical")) // f.AssertWorkflowDispatchedOn(t, "critical", "a") func (f *Fake) AssertWorkflowDispatchedOn(t testing.TB, queueName, jobType string) { + t.Helper() f.b.AssertDispatchedOn(t, queueName, jobType) } @@ -170,10 +186,11 @@ func (f *Fake) AssertWorkflowDispatchedOn(t testing.TB, queueName, jobType strin // // f := queuefake.New() // wf := f.Workflow() -// _, _ = wf.Chain(bus.NewJob("a", nil)).Dispatch(nil) -// _, _ = wf.Chain(bus.NewJob("a", nil)).Dispatch(nil) +// _, _ = wf.Dispatch(nil, bus.NewJob("a", nil)) +// _, _ = wf.Dispatch(nil, bus.NewJob("a", nil)) // f.AssertWorkflowDispatchedTimes(t, "a", 2) func (f *Fake) AssertWorkflowDispatchedTimes(t testing.TB, jobType string, expected int) { + t.Helper() f.b.AssertDispatchedTimes(t, jobType, expected) } @@ -184,7 +201,10 @@ func (f *Fake) AssertWorkflowDispatchedTimes(t testing.TB, jobType string, expec // // f := queuefake.New() // f.AssertWorkflowNotDispatched(t, "emails:send") -func (f *Fake) AssertWorkflowNotDispatched(t testing.TB, jobType string) { f.b.AssertNotDispatched(t, jobType) } +func (f *Fake) AssertWorkflowNotDispatched(t testing.TB, jobType string) { + t.Helper() + f.b.AssertNotDispatched(t, jobType) +} // AssertChained fails if no recorded workflow chain matches expected job type order. // @group Testing @@ -194,7 +214,10 @@ func (f *Fake) AssertWorkflowNotDispatched(t testing.TB, jobType string) { f.b.A // f := queuefake.New() // _, _ = f.Workflow().Chain(bus.NewJob("a", nil), bus.NewJob("b", nil)).Dispatch(nil) // f.AssertChained(t, []string{"a", "b"}) -func (f *Fake) AssertChained(t testing.TB, expected []string) { f.b.AssertChained(t, expected) } +func (f *Fake) AssertChained(t testing.TB, expected []string) { + t.Helper() + f.b.AssertChained(t, expected) +} // AssertBatchCount fails if total recorded workflow batch count does not match n. // @group Testing @@ -204,7 +227,10 @@ func (f *Fake) AssertChained(t testing.TB, expected []string) { f.b.AssertChaine // f := queuefake.New() // _, _ = f.Workflow().Batch(bus.NewJob("a", nil)).Dispatch(nil) // f.AssertBatchCount(t, 1) -func (f *Fake) AssertBatchCount(t testing.TB, n int) { f.b.AssertBatchCount(t, n) } +func (f *Fake) AssertBatchCount(t testing.TB, n int) { + t.Helper() + f.b.AssertBatchCount(t, n) +} // AssertNothingBatched fails if any workflow batch was recorded. // @group Testing @@ -213,7 +239,10 @@ func (f *Fake) AssertBatchCount(t testing.TB, n int) { f.b.AssertBatchCount(t, n // // f := queuefake.New() // f.AssertNothingBatched(t) -func (f *Fake) AssertNothingBatched(t testing.TB) { f.b.AssertNothingBatched(t) } +func (f *Fake) AssertNothingBatched(t testing.TB) { + t.Helper() + f.b.AssertNothingBatched(t) +} // AssertBatched fails unless at least one recorded workflow batch matches predicate. // @group Testing @@ -224,6 +253,7 @@ func (f *Fake) AssertNothingBatched(t testing.TB) { f.b.AssertNothingBatched(t) // _, _ = f.Workflow().Batch(bus.NewJob("a", nil), bus.NewJob("b", nil)).Dispatch(nil) // f.AssertBatched(t, func(spec bus.BatchSpec) bool { return len(spec.JobTypes) == 2 }) func (f *Fake) AssertBatched(t testing.TB, predicate func(spec bus.BatchSpec) bool) { + t.Helper() f.b.AssertBatched(t, predicate) } @@ -234,7 +264,10 @@ func (f *Fake) AssertBatched(t testing.TB, predicate func(spec bus.BatchSpec) bo // // f := queuefake.New() // f.AssertNothingDispatched(t) -func (f *Fake) AssertNothingDispatched(t testing.TB) { f.q.AssertNothingDispatched(t) } +func (f *Fake) AssertNothingDispatched(t testing.TB) { + t.Helper() + f.q.AssertNothingDispatched(t) +} // AssertCount fails when total dispatch count is not expected. // @group Testing @@ -246,7 +279,10 @@ func (f *Fake) AssertNothingDispatched(t testing.TB) { f.q.AssertNothingDispatch // _ = q.Dispatch(queue.NewJob("a")) // _ = q.Dispatch(queue.NewJob("b")) // f.AssertCount(t, 2) -func (f *Fake) AssertCount(t testing.TB, expected int) { f.q.AssertCount(t, expected) } +func (f *Fake) AssertCount(t testing.TB, expected int) { + t.Helper() + f.q.AssertCount(t, expected) +} // AssertDispatched fails when jobType was not dispatched. // @group Testing @@ -256,7 +292,10 @@ func (f *Fake) AssertCount(t testing.TB, expected int) { f.q.AssertCount(t, expe // f := queuefake.New() // _ = f.Queue().Dispatch(queue.NewJob("emails:send")) // f.AssertDispatched(t, "emails:send") -func (f *Fake) AssertDispatched(t testing.TB, jobType string) { f.q.AssertDispatched(t, jobType) } +func (f *Fake) AssertDispatched(t testing.TB, jobType string) { + t.Helper() + f.q.AssertDispatched(t, jobType) +} // AssertDispatchedOn fails when jobType was not dispatched on queueName. // @group Testing @@ -267,6 +306,7 @@ func (f *Fake) AssertDispatched(t testing.TB, jobType string) { f.q.AssertDispat // _ = f.Queue().Dispatch(queue.NewJob("emails:send").OnQueue("critical")) // f.AssertDispatchedOn(t, "critical", "emails:send") func (f *Fake) AssertDispatchedOn(t testing.TB, queueName, jobType string) { + t.Helper() f.q.AssertDispatchedOn(t, queueName, jobType) } @@ -281,6 +321,7 @@ func (f *Fake) AssertDispatchedOn(t testing.TB, queueName, jobType string) { // _ = q.Dispatch(queue.NewJob("emails:send")) // f.AssertDispatchedTimes(t, "emails:send", 2) func (f *Fake) AssertDispatchedTimes(t testing.TB, jobType string, expected int) { + t.Helper() f.q.AssertDispatchedTimes(t, jobType, expected) } @@ -291,4 +332,7 @@ func (f *Fake) AssertDispatchedTimes(t testing.TB, jobType string, expected int) // // f := queuefake.New() // f.AssertNotDispatched(t, "emails:send") -func (f *Fake) AssertNotDispatched(t testing.TB, jobType string) { f.q.AssertNotDispatched(t, jobType) } +func (f *Fake) AssertNotDispatched(t testing.TB, jobType string) { + t.Helper() + f.q.AssertNotDispatched(t, jobType) +} diff --git a/queuefake/fake_test.go b/queuefake/fake_test.go index d7e9710..2e3cdff 100644 --- a/queuefake/fake_test.go +++ b/queuefake/fake_test.go @@ -2,6 +2,7 @@ package queuefake_test import ( "context" + "errors" "testing" "github.com/goforj/queue" @@ -77,3 +78,45 @@ func TestFakeHarness_WorkflowAssertions(t *testing.T) { return len(spec.JobTypes) == 2 && spec.JobTypes[0] == "x" }) } + +// TestFakeHarness_CompatibilityViewsShareState verifies queuefake no longer +// owns independent queue and workflow recorder models. +func TestFakeHarness_CompatibilityViewsShareState(t *testing.T) { + fake := queuefake.New() + if fake.Queue() != fake.Workflow().Queue() { + t.Fatal("Queue and Workflow returned different canonical fakes") + } + if err := fake.Queue().Dispatch(queue.NewJob("queue:direct").OnQueue("root")); err != nil { + t.Fatalf("queue dispatch: %v", err) + } + fake.AssertWorkflowDispatched(t, "queue:direct") + if _, err := fake.Workflow().Dispatch(context.Background(), bus.NewJob("workflow:direct", nil).OnQueue("legacy")); err != nil { + t.Fatalf("workflow dispatch: %v", err) + } + fake.AssertDispatched(t, "workflow:direct") + + chainID, err := fake.Queue().Chain(queue.NewJob("chain:shared")).Dispatch(context.Background()) + if err != nil { + t.Fatalf("chain dispatch: %v", err) + } + batchID, err := fake.Workflow().Batch(bus.NewJob("batch:shared", nil)).Dispatch(context.Background()) + if err != nil { + t.Fatalf("batch dispatch: %v", err) + } + fake.AssertChained(t, []string{"chain:shared"}) + fake.AssertBatchCount(t, 1) + + fake.Reset() + fake.AssertNothingDispatched(t) + fake.AssertNothingWorkflowDispatched(t) + fake.AssertNothingBatched(t) + if len(fake.Queue().ChainRecords()) != 0 { + t.Fatal("Reset retained chain records") + } + if _, err := fake.Queue().FindChain(context.Background(), chainID); !errors.Is(err, queue.ErrWorkflowNotFound) { + t.Fatalf("FindChain after Reset error = %v", err) + } + if _, err := fake.Workflow().FindBatch(context.Background(), batchID); !errors.Is(err, bus.ErrNotFound) { + t.Fatalf("FindBatch after Reset error = %v", err) + } +} diff --git a/runtime.go b/runtime.go index 2837d2b..f7e5cfb 100644 --- a/runtime.go +++ b/runtime.go @@ -6,104 +6,59 @@ import ( "fmt" "time" - "github.com/goforj/queue/bus" + "github.com/goforj/queue/busruntime" + "github.com/goforj/queue/internal/workflow" ) -// Message is the handler message passed to the high-level queue runtime. -// It exposes workflow/job metadata and payload binding helpers. -// @group Queue -type Message = bus.Context - -// DispatchResult describes a high-level dispatch operation. -// @group Queue -type DispatchResult = bus.DispatchResult - -// ChainState is the persisted view of a chain workflow. -// @group Queue -type ChainState = bus.ChainState - -// BatchState is the persisted view of a batch workflow. -// @group Queue -type BatchState = bus.BatchState - // WorkflowEventKind identifies high-level workflow runtime lifecycle events. +// +// Deprecated: use EventKind. Delivery and workflow facts now share one event model. // @group Queue -type WorkflowEventKind = bus.EventKind +type WorkflowEventKind = EventKind // WorkflowEvent is emitted by the high-level workflow runtime observer hooks. +// +// Deprecated: use Event. Delivery and workflow facts now share one event model. // @group Queue -type WorkflowEvent = bus.Event +type WorkflowEvent = Event // WorkflowObserver receives high-level workflow runtime events. +// +// Deprecated: use Observer. A single observer now receives every event layer. // @group Queue -type WorkflowObserver = bus.Observer +type WorkflowObserver = Observer // WorkflowObserverFunc adapts a function to a workflow observer. +// +// Deprecated: use ObserverFunc. A single observer now receives every event layer. // @group Queue -type WorkflowObserverFunc = bus.ObserverFunc - -// Next invokes the next middleware/handler in the queue middleware chain. -// @group Queue -type Next = bus.Next - -// Middleware applies behavior around high-level workflow/job execution. -// @group Queue -type Middleware = bus.Middleware - -// MiddlewareFunc adapts a function to queue middleware. -// @group Queue -type MiddlewareFunc = bus.MiddlewareFunc - -// RetryPolicy is a pass-through middleware policy helper. -// @group Queue -type RetryPolicy = bus.RetryPolicy - -// SkipWhen skips execution when the predicate matches. -// @group Queue -type SkipWhen = bus.SkipWhen - -// FailOnError converts matched errors into fatal (non-retryable) failures. -// @group Queue -type FailOnError = bus.FailOnError - -// RateLimiter is used by RateLimit middleware. -// @group Queue -type RateLimiter = bus.RateLimiter - -// RateLimit applies rate limiting before job execution. -// @group Queue -type RateLimit = bus.RateLimit - -// Lock is used by overlap prevention middleware. -// @group Queue -type Lock = bus.Lock - -// Locker acquires locks for overlap prevention middleware. -// @group Queue -type Locker = bus.Locker - -// WithoutOverlapping prevents concurrent execution for the same key. -// @group Queue -type WithoutOverlapping = bus.WithoutOverlapping +type WorkflowObserverFunc = ObserverFunc -// WorkflowStore is the orchestration state store used for chains/batches/callbacks. +// Permanent marks an error as terminal so workers do not spend the remaining application retry budget on it. // @group Queue -type WorkflowStore = bus.Store +func Permanent(err error) error { + return busruntime.Permanent(err) +} -// ErrWorkflowNotFound indicates a workflow state record is not present. +// IsPermanent reports whether an error requests terminal application settlement. // @group Queue -var ErrWorkflowNotFound = bus.ErrNotFound +func IsPermanent(err error) bool { + return busruntime.IsPermanent(err) +} -// Option configures the high-level workflow runtime. +// Option configures the high-level queue and workflow runtime. // @group Queue type Option func(*runtimeOptions) type runtimeOptions struct { - busOpts []bus.Option - workers int + workflowOpts []workflow.Option + workers int + observer Observer handlerContextDecorator func(context.Context) context.Context + legacyDirectEnvelope bool } +// apply ignores nil options so optional configuration slices compose safely. func (o *runtimeOptions) apply(opts []Option) { for _, opt := range opts { if opt != nil { @@ -112,12 +67,12 @@ func (o *runtimeOptions) apply(opts []Option) { } } -// WithObserver installs a workflow lifecycle observer. +// WithObserver installs one observer for queue, worker, and workflow lifecycle events. // @group Queue // -// Example: workflow observer +// Example: observe all queue activity // -// observer := queue.WorkflowObserverFunc(func(_ context.Context, event queue.WorkflowEvent) { +// observer := queue.ObserverFunc(func(_ context.Context, event queue.Event) { // _ = event.Kind // }) // q, err := queue.New(queue.Config{Driver: queue.DriverSync}, queue.WithObserver(observer)) @@ -125,9 +80,16 @@ func (o *runtimeOptions) apply(opts []Option) { // return // } // _ = q -func WithObserver(observer WorkflowObserver) Option { +func WithObserver(observer Observer) Option { return func(o *runtimeOptions) { - o.busOpts = append(o.busOpts, bus.WithObserver(observer)) + if observer == nil { + return + } + if o.observer == nil { + o.observer = observer + return + } + o.observer = MultiObserver(o.observer, observer) } } @@ -144,7 +106,7 @@ func WithObserver(observer WorkflowObserver) Option { // _ = q func WithStore(store WorkflowStore) Option { return func(o *runtimeOptions) { - o.busOpts = append(o.busOpts, bus.WithStore(store)) + o.workflowOpts = append(o.workflowOpts, workflow.WithStore(workflowStoreFromRoot(store))) } } @@ -163,7 +125,7 @@ func WithStore(store WorkflowStore) Option { // _ = q func WithClock(clock func() time.Time) Option { return func(o *runtimeOptions) { - o.busOpts = append(o.busOpts, bus.WithClock(clock)) + o.workflowOpts = append(o.workflowOpts, workflow.WithClock(clock)) } } @@ -182,7 +144,7 @@ func WithClock(clock func() time.Time) Option { // _ = q func WithMiddleware(middlewares ...Middleware) Option { return func(o *runtimeOptions) { - o.busOpts = append(o.busOpts, bus.WithMiddleware(middlewares...)) + o.workflowOpts = append(o.workflowOpts, workflow.WithMiddleware(middlewaresToWorkflow(middlewares)...)) } } @@ -230,15 +192,30 @@ func WithHandlerContextDecorator(fn func(context.Context) context.Context) Optio } } +// WithLegacyDirectEnvelope keeps ordinary dispatches on the version-one +// `bus:job` wire route during a workers-first migration. Remove this option only +// after every consumer can process canonical direct deliveries. See the +// [direct delivery migration guide] for backend-specific rollout and rollback. +// +// [direct delivery migration guide]: https://github.com/goforj/queue/blob/main/docs/direct-delivery-migration.md +// @group Queue +func WithLegacyDirectEnvelope() Option { + return func(o *runtimeOptions) { + o.legacyDirectEnvelope = true + } +} + // Queue is the high-level user-facing queue API. // It composes the queue runtime with the internal orchestration engine. // @group Queue type Queue struct { - q queueRuntime - b bus.Bus - ctx context.Context + q queueRuntime + b workflow.Engine + ctx context.Context + legacyDirectEnvelope bool } +// newHighLevelQueue constructs the selected physical runtime before attaching the canonical workflow engine. func newHighLevelQueue(cfg Config, opts ...Option) (*Queue, error) { q, err := newRuntime(cfg) if err != nil { @@ -247,20 +224,88 @@ func newHighLevelQueue(cfg Config, opts ...Option) (*Queue, error) { return newQueueFromRuntime(q, opts...) } +// newQueueFromRuntime applies root configuration once before registering the single internal workflow engine. func newQueueFromRuntime(q queueRuntime, opts ...Option) (*Queue, error) { var ro runtimeOptions ro.apply(opts) + observer := attachRuntimeObserver(q, ro.observer) if ro.workers > 0 && q != nil { q = q.Workers(ro.workers) } if ro.handlerContextDecorator != nil && q != nil { q.setHandlerContextDecorator(ro.handlerContextDecorator) } - b, err := bus.New(q, ro.busOpts...) + if observer != nil { + driver := Driver("") + resolveQueueName := func(queueName string) string { + return PhysicalQueueName("default", queueName) + } + if q != nil { + driver = q.Driver() + resolveQueueName = func(queueName string) string { + if queueName == "" { + return "default" + } + return q.physicalQueueNameOrDefault(queueName) + } + } + ro.workflowOpts = append(ro.workflowOpts, workflow.WithObserver(workflowObserverAdapter{ + driver: driver, + resolveQueueName: resolveQueueName, + observer: observer, + })) + } + b, err := workflow.New(q, ro.workflowOpts...) if err != nil { return nil, err } - return &Queue{q: q, b: b}, nil + return &Queue{q: q, b: b, legacyDirectEnvelope: ro.legacyDirectEnvelope}, nil +} + +// attachRuntimeObserver composes constructor and option observers before the workflow runtime is built so every layer shares one sink. +func attachRuntimeObserver(q queueRuntime, observer Observer) Observer { + switch runtime := q.(type) { + case *nativeQueueRuntime: + runtime.common.addObserver(observer) + return runtime.common.observer() + case *externalQueueRuntime: + runtime.common.addObserver(observer) + return runtime.common.observer() + default: + return observer + } +} + +type workflowObserverAdapter struct { + driver Driver + resolveQueueName func(string) string + observer Observer +} + +// Observe converts workflow facts into the canonical event envelope without exposing the internal engine model to applications. +func (a workflowObserverAdapter) Observe(ctx context.Context, event workflow.Event) { + queueName := PhysicalQueueName("default", event.Queue) + if a.resolveQueueName != nil { + queueName = a.resolveQueueName(event.Queue) + } + safeObserve(ctx, a.observer, Event{ + SchemaVersion: event.SchemaVersion, + EventID: event.EventID, + Layer: eventLayerForKind(EventKind(event.Kind)), + Kind: EventKind(event.Kind), + Driver: a.driver, + Queue: queueName, + JobType: event.JobType, + JobKey: event.JobKey, + DispatchID: event.DispatchID, + JobID: event.JobID, + ChainID: event.ChainID, + BatchID: event.BatchID, + Attempt: event.Attempt, + Duration: event.Duration, + Err: event.Err, + Time: event.Time, + }) } // NewNull creates a Queue on the null backend. @@ -326,10 +371,12 @@ func NewWorkerpool(opts ...Option) (*Queue, error) { // return nil // }) func (r *Queue) Register(jobType string, handler func(context.Context, Message) error) { - if r == nil { + if r == nil || handler == nil { return } - r.b.Register(jobType, handler) + r.b.Register(jobType, func(ctx context.Context, message workflow.Context) error { + return handler(ctx, messageFromWorkflow(message)) + }) } // Driver reports the configured backend driver for the underlying queue runtime. @@ -379,7 +426,8 @@ func (r *Queue) WithContext(ctx context.Context) *Queue { return &clone } -// Dispatch enqueues a high-level job using the queue's bound context. +// Dispatch enqueues a high-level job using its application type and exact +// payload bytes together with the queue's bound context. // @group Queue // // Example: dispatch @@ -389,21 +437,62 @@ func (r *Queue) WithContext(ctx context.Context) *Queue { // return // } // q.Register("emails:send", func(ctx context.Context, m queue.Message) error { return nil }) +// if err := q.StartWorkers(context.Background()); err != nil { +// return +// } +// defer q.Shutdown(context.Background()) // job := queue.NewJob("emails:send").Payload(map[string]any{"id": 1}).OnQueue("default") // _, _ = q.Dispatch(job) func (r *Queue) Dispatch(job Job) (DispatchResult, error) { if r == nil { return DispatchResult{}, fmt.Errorf("runtime is nil") } - bj, err := toBusJob(job) - if err != nil { - return DispatchResult{}, err - } ctx := r.ctx if ctx == nil { ctx = context.Background() } - return r.b.Dispatch(ctx, bj) + if r.legacyDirectEnvelope { + legacy, legacyErr := toWorkflowJob(job) + if legacyErr != nil { + return DispatchResult{}, legacyErr + } + result, dispatchErr := r.b.Dispatch(ctx, legacy) + return dispatchResultFromWorkflow(result), dispatchErr + } + bj, err := toDirectWorkflowJob(job) + if err != nil { + return DispatchResult{}, err + } + result, err := r.b.DispatchDirect(ctx, bj) + return dispatchResultFromWorkflow(result), err +} + +// toDirectWorkflowJob freezes the canonical root job as exact application +// bytes, avoiding the legacy workflow payload marshaling boundary. +func toDirectWorkflowJob(job Job) (workflow.StoredJob, error) { + if err := job.validate(); err != nil { + return workflow.StoredJob{}, err + } + var timeout time.Duration + if job.options.timeout != nil { + timeout = *job.options.timeout + } + var backoff time.Duration + if job.options.backoff != nil { + backoff = *job.options.backoff + } + return workflow.StoredJob{ + Type: job.Type, + Payload: job.PayloadBytes(), + Options: workflow.JobOptions{ + Queue: job.options.queueName, + Delay: job.options.delay, + Timeout: timeout, + Retry: optionInt(job.options.maxRetry), + Backoff: backoff, + UniqueFor: job.options.uniqueTTL, + }, + }, nil } // Chain creates a chain builder for sequential workflow execution. @@ -417,6 +506,10 @@ func (r *Queue) Dispatch(job Job) (DispatchResult, error) { // } // q.Register("first", func(ctx context.Context, m queue.Message) error { return nil }) // q.Register("second", func(ctx context.Context, m queue.Message) error { return nil }) +// if err := q.StartWorkers(context.Background()); err != nil { +// return +// } +// defer q.Shutdown(context.Background()) // _, _ = q.Chain( // queue.NewJob("first"), // queue.NewJob("second"), @@ -425,15 +518,11 @@ func (r *Queue) Chain(jobs ...Job) ChainBuilder { if r == nil { return &chainBuilderAdapter{} } - busJobs := make([]bus.Job, 0, len(jobs)) - for _, job := range jobs { - bj, err := toBusJob(job) - if err != nil { - return &chainBuilderAdapter{err: err} - } - busJobs = append(busJobs, bj) + workflowJobs, err := toWorkflowJobs(jobs) + if err != nil { + return &chainBuilderAdapter{err: err} } - return &chainBuilderAdapter{inner: r.b.Chain(busJobs...)} + return &chainBuilderAdapter{inner: r.b.Chain(workflowJobs...)} } // Batch creates a batch builder for fan-out workflow execution. @@ -446,6 +535,10 @@ func (r *Queue) Chain(jobs ...Job) ChainBuilder { // return // } // q.Register("emails:send", func(ctx context.Context, m queue.Message) error { return nil }) +// if err := q.StartWorkers(context.Background()); err != nil { +// return +// } +// defer q.Shutdown(context.Background()) // _, _ = q.Batch( // queue.NewJob("emails:send").Payload(map[string]any{"id": 1}), // queue.NewJob("emails:send").Payload(map[string]any{"id": 2}), @@ -454,15 +547,11 @@ func (r *Queue) Batch(jobs ...Job) BatchBuilder { if r == nil { return &batchBuilderAdapter{} } - busJobs := make([]bus.Job, 0, len(jobs)) - for _, job := range jobs { - bj, err := toBusJob(job) - if err != nil { - return &batchBuilderAdapter{err: err} - } - busJobs = append(busJobs, bj) + workflowJobs, err := toWorkflowJobs(jobs) + if err != nil { + return &batchBuilderAdapter{err: err} } - return &batchBuilderAdapter{inner: r.b.Batch(busJobs...)} + return &batchBuilderAdapter{inner: r.b.Batch(workflowJobs...)} } // StartWorkers starts worker processing. @@ -551,7 +640,8 @@ func (r *Queue) FindChain(ctx context.Context, chainID string) (ChainState, erro if r == nil { return ChainState{}, fmt.Errorf("runtime is nil") } - return r.b.FindChain(ctx, chainID) + state, err := r.b.FindChain(ctx, chainID) + return chainStateFromWorkflow(state), err } // FindBatch returns current batch state by ID. @@ -573,7 +663,8 @@ func (r *Queue) FindBatch(ctx context.Context, batchID string) (BatchState, erro if r == nil { return BatchState{}, fmt.Errorf("runtime is nil") } - return r.b.FindBatch(ctx, batchID) + state, err := r.b.FindBatch(ctx, batchID) + return batchStateFromWorkflow(state), err } // Prune deletes old workflow state records. @@ -689,17 +780,26 @@ func (r *Queue) Ready(ctx context.Context) error { // ChainBuilder is the high-level chain workflow builder. // @group Queue type ChainBuilder interface { + // OnQueue applies a default queue to chain jobs without an explicit target. OnQueue(queue string) ChainBuilder + // Catch registers the explicitly ephemeral chain failure callback. Catch(fn func(ctx context.Context, st ChainState, err error) error) ChainBuilder + // Finally registers the explicitly ephemeral chain terminal callback. Finally(fn func(ctx context.Context, st ChainState) error) ChainBuilder + // Dispatch persists and starts the chain workflow. Dispatch(ctx context.Context) (string, error) } type chainBuilderAdapter struct { - inner bus.ChainBuilder - err error + inner workflow.ChainBuilder + err error + dispatchGuard func() func() + dispatchContext func(context.Context) context.Context + onAccepted func(string) + onRejected func(string) } +// OnQueue forwards fluent queue selection while preserving any earlier conversion failure. func (b *chainBuilderAdapter) OnQueue(queue string) ChainBuilder { if b.inner != nil { b.inner = b.inner.OnQueue(queue) @@ -707,20 +807,23 @@ func (b *chainBuilderAdapter) OnQueue(queue string) ChainBuilder { return b } +// Catch forwards the explicitly ephemeral chain failure callback. func (b *chainBuilderAdapter) Catch(fn func(ctx context.Context, st ChainState, err error) error) ChainBuilder { if b.inner != nil { - b.inner = b.inner.Catch(fn) + b.inner = b.inner.Catch(chainCatchToWorkflow(fn)) } return b } +// Finally forwards the explicitly ephemeral chain terminal callback. func (b *chainBuilderAdapter) Finally(fn func(ctx context.Context, st ChainState) error) ChainBuilder { if b.inner != nil { - b.inner = b.inner.Finally(fn) + b.inner = b.inner.Finally(chainFinallyToWorkflow(fn)) } return b } +// Dispatch returns deferred builder errors before asking the internal engine to create state. func (b *chainBuilderAdapter) Dispatch(ctx context.Context) (string, error) { if b.err != nil { return "", b.err @@ -728,27 +831,53 @@ func (b *chainBuilderAdapter) Dispatch(ctx context.Context) (string, error) { if b.inner == nil { return "", fmt.Errorf("chain builder is nil") } - return b.inner.Dispatch(ctx) + if b.dispatchGuard != nil { + release := b.dispatchGuard() + defer release() + } + if b.dispatchContext != nil { + ctx = b.dispatchContext(ctx) + } + chainID, err := b.inner.Dispatch(ctx) + if err == nil && b.onAccepted != nil { + b.onAccepted(chainID) + } else if err != nil && chainID != "" && b.onRejected != nil { + b.onRejected(chainID) + } + return chainID, err } // BatchBuilder is the high-level batch workflow builder. // @group Queue type BatchBuilder interface { + // Name assigns an application-facing label to the batch. Name(name string) BatchBuilder + // OnQueue applies a default queue to batch jobs without an explicit target. OnQueue(queue string) BatchBuilder + // AllowFailures keeps remaining members active after a terminal member failure. AllowFailures() BatchBuilder + // Progress registers the explicitly ephemeral batch progress callback. Progress(fn func(ctx context.Context, st BatchState) error) BatchBuilder + // Then registers the explicitly ephemeral batch success callback. Then(fn func(ctx context.Context, st BatchState) error) BatchBuilder + // Catch registers the explicitly ephemeral batch failure callback. Catch(fn func(ctx context.Context, st BatchState, err error) error) BatchBuilder + // Finally registers the explicitly ephemeral batch terminal callback. Finally(fn func(ctx context.Context, st BatchState) error) BatchBuilder + // Dispatch persists and starts the batch workflow. Dispatch(ctx context.Context) (string, error) } type batchBuilderAdapter struct { - inner bus.BatchBuilder - err error + inner workflow.BatchBuilder + err error + dispatchGuard func() func() + dispatchContext func(context.Context) context.Context + onAccepted func(string) + onRejected func(string) } +// Name forwards the application-facing batch label while preserving any earlier conversion failure. func (b *batchBuilderAdapter) Name(name string) BatchBuilder { if b.inner != nil { b.inner = b.inner.Name(name) @@ -756,6 +885,7 @@ func (b *batchBuilderAdapter) Name(name string) BatchBuilder { return b } +// OnQueue forwards fluent queue selection while preserving any earlier conversion failure. func (b *batchBuilderAdapter) OnQueue(queue string) BatchBuilder { if b.inner != nil { b.inner = b.inner.OnQueue(queue) @@ -763,6 +893,7 @@ func (b *batchBuilderAdapter) OnQueue(queue string) BatchBuilder { return b } +// AllowFailures forwards the aggregate failure policy to the internal engine. func (b *batchBuilderAdapter) AllowFailures() BatchBuilder { if b.inner != nil { b.inner = b.inner.AllowFailures() @@ -770,34 +901,39 @@ func (b *batchBuilderAdapter) AllowFailures() BatchBuilder { return b } +// Progress forwards the explicitly ephemeral batch progress callback. func (b *batchBuilderAdapter) Progress(fn func(ctx context.Context, st BatchState) error) BatchBuilder { if b.inner != nil { - b.inner = b.inner.Progress(fn) + b.inner = b.inner.Progress(batchStateCallbackToWorkflow(fn)) } return b } +// Then forwards the explicitly ephemeral batch success callback. func (b *batchBuilderAdapter) Then(fn func(ctx context.Context, st BatchState) error) BatchBuilder { if b.inner != nil { - b.inner = b.inner.Then(fn) + b.inner = b.inner.Then(batchStateCallbackToWorkflow(fn)) } return b } +// Catch forwards the explicitly ephemeral batch failure callback. func (b *batchBuilderAdapter) Catch(fn func(ctx context.Context, st BatchState, err error) error) BatchBuilder { if b.inner != nil { - b.inner = b.inner.Catch(fn) + b.inner = b.inner.Catch(batchCatchToWorkflow(fn)) } return b } +// Finally forwards the explicitly ephemeral batch terminal callback. func (b *batchBuilderAdapter) Finally(fn func(ctx context.Context, st BatchState) error) BatchBuilder { if b.inner != nil { - b.inner = b.inner.Finally(fn) + b.inner = b.inner.Finally(batchStateCallbackToWorkflow(fn)) } return b } +// Dispatch returns deferred builder errors before asking the internal engine to create state. func (b *batchBuilderAdapter) Dispatch(ctx context.Context) (string, error) { if b.err != nil { return "", b.err @@ -805,22 +941,50 @@ func (b *batchBuilderAdapter) Dispatch(ctx context.Context) (string, error) { if b.inner == nil { return "", fmt.Errorf("batch builder is nil") } - return b.inner.Dispatch(ctx) + if b.dispatchGuard != nil { + release := b.dispatchGuard() + defer release() + } + if b.dispatchContext != nil { + ctx = b.dispatchContext(ctx) + } + batchID, err := b.inner.Dispatch(ctx) + if err == nil && b.onAccepted != nil { + b.onAccepted(batchID) + } else if err != nil && batchID != "" && b.onRejected != nil { + b.onRejected(batchID) + } + return batchID, err +} + +// toWorkflowJobs converts a canonical job slice once so production and fake +// workflow builders share validation and payload ownership rules. +func toWorkflowJobs(jobs []Job) ([]workflow.Job, error) { + converted := make([]workflow.Job, 0, len(jobs)) + for _, job := range jobs { + workflowJob, err := toWorkflowJob(job) + if err != nil { + return nil, err + } + converted = append(converted, workflowJob) + } + return converted, nil } -func toBusJob(job Job) (bus.Job, error) { +// toWorkflowJob converts the canonical root job into the engine's private compatibility model without changing payload bytes. +func toWorkflowJob(job Job) (workflow.Job, error) { if err := job.validate(); err != nil { - return bus.Job{}, err + return workflow.Job{}, err } if job.Type == "" { - return bus.Job{}, fmt.Errorf("job type is required") + return workflow.Job{}, fmt.Errorf("job type is required") } payload := job.PayloadBytes() var busPayload any if payload != nil { busPayload = json.RawMessage(payload) } - j := bus.NewJob(job.Type, busPayload) + j := workflow.NewJob(job.Type, busPayload) if job.options.queueName != "" { j = j.OnQueue(job.options.queueName) } diff --git a/runtime_conversion_test.go b/runtime_conversion_test.go index 39a4ae1..1e0c714 100644 --- a/runtime_conversion_test.go +++ b/runtime_conversion_test.go @@ -6,7 +6,7 @@ import ( "time" ) -func TestToBusJob_MapsPayloadAndOptions(t *testing.T) { +func TestToWorkflowJob_MapsPayloadAndOptions(t *testing.T) { type payload struct { ID int `json:"id"` } @@ -20,9 +20,9 @@ func TestToBusJob_MapsPayloadAndOptions(t *testing.T) { Backoff(500 * time.Millisecond). UniqueFor(30 * time.Second) - got, err := toBusJob(in) + got, err := toWorkflowJob(in) if err != nil { - t.Fatalf("toBusJob error: %v", err) + t.Fatalf("toWorkflowJob error: %v", err) } if got.Type != "emails:send" { @@ -60,18 +60,18 @@ func TestToBusJob_MapsPayloadAndOptions(t *testing.T) { } } -func TestToBusJob_PreservesNilPayload(t *testing.T) { - got, err := toBusJob(NewJob("job:nil")) +func TestToWorkflowJob_PreservesNilPayload(t *testing.T) { + got, err := toWorkflowJob(NewJob("job:nil")) if err != nil { - t.Fatalf("toBusJob error: %v", err) + t.Fatalf("toWorkflowJob error: %v", err) } if got.Payload != nil { t.Fatalf("payload=%T expected nil", got.Payload) } } -func TestToBusJob_ValidationError(t *testing.T) { - if _, err := toBusJob(NewJob("bad").Retry(-1)); err == nil { +func TestToWorkflowJob_ValidationError(t *testing.T) { + if _, err := toWorkflowJob(NewJob("bad").Retry(-1)); err == nil { t.Fatal("expected validation error") } } diff --git a/runtime_test.go b/runtime_test.go index 5ce7045..5fc7b26 100644 --- a/runtime_test.go +++ b/runtime_test.go @@ -3,11 +3,10 @@ package queue import ( "context" "errors" + "runtime" "sync/atomic" "testing" "time" - - "github.com/goforj/queue/bus" ) func TestRuntime_DispatchChainBatch_Sync(t *testing.T) { @@ -167,6 +166,170 @@ func TestNew_WithObserver(t *testing.T) { } } +// TestQueueHandlerContextDecoratorWithAndWithoutObserver verifies context decoration does not depend on observation being enabled. +func TestQueueHandlerContextDecoratorWithAndWithoutObserver(t *testing.T) { + type contextKey struct{} + key := contextKey{} + const want = "jobs" + + for _, withObserver := range []bool{false, true} { + name := "without observer" + if withObserver { + name = "with observer" + } + t.Run(name, func(t *testing.T) { + var decoratorCalls atomic.Int32 + var observedCalls atomic.Int32 + var observerSawWrongContext atomic.Bool + opts := []Option{ + WithHandlerContextDecorator(func(ctx context.Context) context.Context { + decoratorCalls.Add(1) + return context.WithValue(ctx, key, want) + }), + } + if withObserver { + opts = append(opts, WithObserver(ObserverFunc(func(ctx context.Context, event Event) { + if event.Kind != EventProcessStarted && event.Kind != EventProcessSucceeded { + return + } + observedCalls.Add(1) + if got, _ := ctx.Value(key).(string); got != want { + observerSawWrongContext.Store(true) + } + }))) + } + + q, err := NewSync(opts...) + if err != nil { + t.Fatalf("new sync queue: %v", err) + } + t.Cleanup(func() { + if err := q.Shutdown(context.Background()); err != nil { + t.Errorf("shutdown sync queue: %v", err) + } + }) + q.Register("job:decorated", func(ctx context.Context, _ Message) error { + if got, _ := ctx.Value(key).(string); got != want { + return errors.New("handler context was not decorated") + } + return nil + }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + if _, err := q.Dispatch(NewJob("job:decorated")); err != nil { + t.Fatalf("dispatch decorated job: %v", err) + } + + if got := decoratorCalls.Load(); got != 1 { + t.Fatalf("decorator calls = %d, want 1", got) + } + wantObserved := int32(0) + if withObserver { + wantObserved = 2 + } + if got := observedCalls.Load(); got != wantObserved { + t.Fatalf("observed process events = %d, want %d", got, wantObserved) + } + if observerSawWrongContext.Load() { + t.Fatal("observer received a process event without the decorated context") + } + }) + } +} + +// TestQueueSyncShutdownRetriesReuseWorkDrainGeneration verifies public retries +// cannot accumulate goroutines while one delayed Sync handler is still active. +func TestQueueSyncShutdownRetriesReuseWorkDrainGeneration(t *testing.T) { + q, err := NewSync() + if err != nil { + t.Fatalf("new sync queue: %v", err) + } + handlerEntered := make(chan struct{}, 1) + releaseHandler := make(chan struct{}) + release := func() { + select { + case <-releaseHandler: + default: + close(releaseHandler) + } + } + t.Cleanup(func() { + release() + if err := q.Shutdown(context.Background()); err != nil { + t.Errorf("cleanup sync queue: %v", err) + } + }) + + q.Register("job:delayed-shutdown", func(context.Context, Message) error { + handlerEntered <- struct{}{} + <-releaseHandler + return nil + }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + if _, err := q.Dispatch(NewJob("job:delayed-shutdown").Delay(time.Nanosecond)); err != nil { + t.Fatalf("dispatch delayed job: %v", err) + } + select { + case <-handlerEntered: + case <-time.After(5 * time.Second): + t.Fatal("delayed handler did not start") + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := q.Shutdown(ctx); !errors.Is(err, context.Canceled) { + t.Fatalf("first canceled shutdown error = %v, want %v", err, context.Canceled) + } + native, ok := q.q.(*nativeQueueRuntime) + if !ok { + t.Fatalf("sync queue runtime = %T, want *nativeQueueRuntime", q.q) + } + local, ok := native.runtime.(*localQueue) + if !ok { + t.Fatalf("sync backend = %T, want *localQueue", native.runtime) + } + local.syncWorkMu.Lock() + sharedDone := local.syncWorkIdle + local.syncWorkMu.Unlock() + if sharedDone == nil { + t.Fatal("first canceled shutdown did not retain a Sync work generation") + } + goroutinesAfterFirstCancel := runtime.NumGoroutine() + for range 64 { + if err := q.Shutdown(ctx); !errors.Is(err, context.Canceled) { + t.Fatalf("repeated canceled shutdown error = %v, want %v", err, context.Canceled) + } + local.syncWorkMu.Lock() + currentDone := local.syncWorkIdle + local.syncWorkMu.Unlock() + if currentDone != sharedDone { + t.Fatal("public shutdown retry replaced the Sync work generation") + } + } + runtime.Gosched() + if got := runtime.NumGoroutine(); got > goroutinesAfterFirstCancel+2 { + t.Fatalf("canceled shutdown retries grew goroutines from %d to %d", goroutinesAfterFirstCancel, got) + } + select { + case <-sharedDone: + t.Fatal("Sync work generation completed while its handler was blocked") + default: + } + + release() + if err := q.Shutdown(context.Background()); err != nil { + t.Fatalf("retry shutdown after handler completion: %v", err) + } + select { + case <-sharedDone: + default: + t.Fatal("completed public shutdown did not close the shared Sync work generation") + } +} + func TestQueue_Run_WorkerpoolStartsAndShutsDownOnCancel(t *testing.T) { q, err := NewWorkerpool(WithWorkers(2)) if err != nil { @@ -226,7 +389,7 @@ func TestNew_WithStoreClockMiddlewareAndPrune(t *testing.T) { q, err := New( Config{Driver: DriverSync}, - WithStore(bus.NewMemoryStore()), + WithStore(NewMemoryStore()), WithClock(func() time.Time { return fixedNow }), WithObserver(WorkflowObserverFunc(func(context.Context, WorkflowEvent) { observed.Add(1) })), WithMiddleware(MiddlewareFunc(func(ctx context.Context, m Message, next Next) error { diff --git a/scripts/check-generated-docs.sh b/scripts/check-generated-docs.sh new file mode 100755 index 0000000..d0d311d --- /dev/null +++ b/scripts/check-generated-docs.sh @@ -0,0 +1,97 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +GOCACHE_DIR="${GOCACHE:-/tmp/queue-gocache}" +GOMODCACHE_DIR="${GOMODCACHE:-/tmp/queue-gomodcache}" +TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/queue-generated-docs.XXXXXX")" + +cleanup() { + rm -rf "$TMP_DIR" +} +trap cleanup EXIT + +run_generators() { + ( + cd "$ROOT_DIR/docs" + GOWORK=off GOCACHE="$GOCACHE_DIR" GOMODCACHE="$GOMODCACHE_DIR" go run ./readme/main.go + GOWORK=off GOCACHE="$GOCACHE_DIR" GOMODCACHE="$GOMODCACHE_DIR" go run ./examplegen/main.go + TESTCOUNT_USE_INTEGRATION_MANIFEST=1 GOWORK=off GOCACHE="$GOCACHE_DIR" GOMODCACHE="$GOMODCACHE_DIR" go run ./readme/testcounts/main.go + BENCH_RENDER_ONLY=1 GOWORK=off GOCACHE="$GOCACHE_DIR" GOMODCACHE="$GOMODCACHE_DIR" go test -tags=benchrender ./bench -run '^TestRenderBenchmarks$' -count=1 + ) +} + +snapshot_outputs() { + local destination="$1" + mkdir -p "$destination" + cp "$ROOT_DIR/README.md" "$destination/README.md" + cp -R "$ROOT_DIR/examples" "$destination/examples" + mkdir -p "$destination/bench" + cp "$ROOT_DIR/docs/bench/benchmarks_rows.json" "$destination/bench/benchmarks_rows.json" + cp "$ROOT_DIR/docs/bench/benchmarks_ns.svg" "$destination/bench/benchmarks_ns.svg" + cp "$ROOT_DIR/docs/bench/benchmarks_ops.svg" "$destination/bench/benchmarks_ops.svg" + cp "$ROOT_DIR/docs/bench/benchmarks_bytes.svg" "$destination/bench/benchmarks_bytes.svg" + cp "$ROOT_DIR/docs/bench/benchmarks_allocs.svg" "$destination/bench/benchmarks_allocs.svg" + mkdir -p "$destination/testcounts" + cp "$ROOT_DIR/docs/readme/testcounts/integration_count.json" "$destination/testcounts/integration_count.json" +} + +echo "==> test test-count evidence validation" +( + cd "$ROOT_DIR/docs" + GOWORK=off GOCACHE="$GOCACHE_DIR" GOMODCACHE="$GOMODCACHE_DIR" go test -tags=testcounts ./readme/testcounts -count=1 +) + +snapshot_outputs "$TMP_DIR/before" + +echo "==> generate README, examples, test counts, and benchmark dashboard" +run_generators + +first_generation_clean=1 +if ! diff -u "$TMP_DIR/before/README.md" "$ROOT_DIR/README.md"; then + first_generation_clean=0 +fi +if ! diff -ruN "$TMP_DIR/before/examples" "$ROOT_DIR/examples"; then + first_generation_clean=0 +fi +for output in benchmarks_rows.json benchmarks_ns.svg benchmarks_ops.svg benchmarks_bytes.svg benchmarks_allocs.svg; do + if ! diff -u "$TMP_DIR/before/bench/$output" "$ROOT_DIR/docs/bench/$output"; then + first_generation_clean=0 + fi +done +if ! diff -u "$TMP_DIR/before/testcounts/integration_count.json" "$ROOT_DIR/docs/readme/testcounts/integration_count.json"; then + first_generation_clean=0 +fi +if [[ "$first_generation_clean" != "1" ]]; then + echo "generated documentation differs from the checked-in output" +fi + +snapshot_outputs "$TMP_DIR/first" + +echo "==> regenerate generated documentation to verify idempotency" +run_generators + +idempotent=1 +if ! diff -ruN "$TMP_DIR/first/README.md" "$ROOT_DIR/README.md"; then + idempotent=0 +fi +if ! diff -ruN "$TMP_DIR/first/examples" "$ROOT_DIR/examples"; then + idempotent=0 +fi +for output in benchmarks_rows.json benchmarks_ns.svg benchmarks_ops.svg benchmarks_bytes.svg benchmarks_allocs.svg; do + if ! diff -u "$TMP_DIR/first/bench/$output" "$ROOT_DIR/docs/bench/$output"; then + idempotent=0 + fi +done +if ! diff -u "$TMP_DIR/first/testcounts/integration_count.json" "$ROOT_DIR/docs/readme/testcounts/integration_count.json"; then + idempotent=0 +fi + +if [[ "$first_generation_clean" != "1" || "$idempotent" != "1" ]]; then + if [[ "$idempotent" != "1" ]]; then + echo "documentation generators are not idempotent" + fi + exit 1 +fi + +echo "==> generated documentation is current and idempotent" diff --git a/scripts/check-module-inventory.sh b/scripts/check-module-inventory.sh new file mode 100755 index 0000000..3de9e29 --- /dev/null +++ b/scripts/check-module-inventory.sh @@ -0,0 +1,695 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Guards the repository-wide module contract that ordinary Go commands cannot +# see because they stop at nested module boundaries. + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" +RELEASE_VERSION="" +TAG_VERSION="" +EXCLUDES=() + +usage() { + cat <<'USAGE' +Usage: + scripts/check-module-inventory.sh [--release-version ] [--exclude ]... [--tag-version ] + +Checks: + - every go.mod has the expected repository module path and exact policy Go version + - go.work contains every discovered module exactly once and uses the highest module Go version + - policy-protected dependencies remain at or above their safe minimum versions + - minimum-toolchain CI covers every declared module Go version + - the CI race matrix contains root and every discovered driver module exactly once + - sibling requirements use one version and resolve through local replacements + - the release tag planner discovers every module and computes the documented tag + - --release-version rejects sibling dependency pins that would not resolve after release + - --exclude omits matching module owners but rejects an incomplete dependency tag set + - --tag-version verifies an existing synchronized tag family at one commit + +Examples: + scripts/check-module-inventory.sh + scripts/check-module-inventory.sh --release-version v0.3.0 + scripts/check-module-inventory.sh --release-version v0.3.0 --exclude examples + scripts/check-module-inventory.sh --tag-version v0.2.1 +USAGE +} + +fail() { + echo "module inventory guard: $*" >&2 + exit 1 +} + +require_version() { + local version="$1" + if ! validate_release_version "$version"; then + fail "$RELEASE_VERSION_ERROR" + fi +} + +normalize_dir() { + local dir="$1" + dir="${dir#./}" + dir="${dir%/}" + if [[ -z "$dir" ]]; then + dir="." + fi + printf '%s\n' "$dir" +} + +semver_at_least() { + LC_ALL=C awk -v current="$1" -v minimum="$2" ' + function valid_identifiers(value, allow_numeric_leading_zero, parts, count, i, item) { + if (value == "") return 0 + count = split(value, parts, ".") + for (i = 1; i <= count; i++) { + item = parts[i] + if (item == "" || item !~ /^[0-9A-Za-z-]+$/) return 0 + if (!allow_numeric_leading_zero && item ~ /^[0-9]+$/ && length(item) > 1 && substr(item, 1, 1) == "0") return 0 + } + return 1 + } + + function parse(value, result, plus, dash, build, prerelease, core, parts, count, i) { + if (substr(value, 1, 1) != "v") return 0 + value = substr(value, 2) + + plus = index(value, "+") + if (plus > 0) { + build = substr(value, plus + 1) + if (!valid_identifiers(build, 1)) return 0 + value = substr(value, 1, plus - 1) + } + + dash = index(value, "-") + if (dash > 0) { + prerelease = substr(value, dash + 1) + if (!valid_identifiers(prerelease, 0)) return 0 + core = substr(value, 1, dash - 1) + } else { + prerelease = "" + core = value + } + + count = split(core, parts, ".") + if (count != 3) return 0 + for (i = 1; i <= count; i++) { + if (parts[i] !~ /^(0|[1-9][0-9]*)$/) return 0 + } + result["major"] = parts[1] + result["minor"] = parts[2] + result["patch"] = parts[3] + result["prerelease"] = prerelease + return 1 + } + + function compare_decimal(left, right) { + if (length(left) != length(right)) return length(left) < length(right) ? -1 : 1 + if (("x" left) == ("x" right)) return 0 + return ("x" left) < ("x" right) ? -1 : 1 + } + + function compare_identifiers(left, right, left_numeric, right_numeric, compared) { + left_numeric = left ~ /^[0-9]+$/ + right_numeric = right ~ /^[0-9]+$/ + if (left_numeric && right_numeric) return compare_decimal(left, right) + if (left_numeric != right_numeric) return left_numeric ? -1 : 1 + if (("x" left) == ("x" right)) return 0 + return ("x" left) < ("x" right) ? -1 : 1 + } + + function compare(left, right, compared, left_parts, right_parts, left_count, right_count, count, i) { + compared = compare_decimal(left["major"], right["major"]) + if (compared != 0) return compared + compared = compare_decimal(left["minor"], right["minor"]) + if (compared != 0) return compared + compared = compare_decimal(left["patch"], right["patch"]) + if (compared != 0) return compared + + if (left["prerelease"] == "" && right["prerelease"] == "") return 0 + if (left["prerelease"] == "") return 1 + if (right["prerelease"] == "") return -1 + + left_count = split(left["prerelease"], left_parts, ".") + right_count = split(right["prerelease"], right_parts, ".") + count = left_count < right_count ? left_count : right_count + for (i = 1; i <= count; i++) { + compared = compare_identifiers(left_parts[i], right_parts[i]) + if (compared != 0) return compared + } + if (left_count == right_count) return 0 + return left_count < right_count ? -1 : 1 + } + + BEGIN { + if (!parse(current, current_version) || !parse(minimum, minimum_version)) exit 2 + exit compare(current_version, minimum_version) >= 0 ? 0 : 1 + } + ' +} + +go_version_as_semver() { + local version="$1" + local major minor patch extra + IFS='.' read -r major minor patch extra <<<"$version" + patch="${patch:-0}" + [[ -n "$major" && -n "$minor" && -z "$extra" ]] || return 1 + [[ "$major" =~ ^(0|[1-9][0-9]*)$ ]] || return 1 + [[ "$minor" =~ ^(0|[1-9][0-9]*)$ ]] || return 1 + [[ "$patch" =~ ^(0|[1-9][0-9]*)$ ]] || return 1 + printf 'v%s.%s.%s\n' "$major" "$minor" "$patch" +} + +go_version_at_least() { + local current minimum + current="$(go_version_as_semver "$1")" || return 1 + minimum="$(go_version_as_semver "$2")" || return 1 + semver_at_least "$current" "$minimum" +} + +module_is_excluded() { + local dir="$1" + local excluded + for excluded in "${EXCLUDES[@]}"; do + if [[ "$dir" == "$excluded" ]] || [[ "$dir" == "$excluded/"* ]]; then + return 0 + fi + done + return 1 +} + +json_value() { + local key="$1" + awk -v key="$key" ' + $0 ~ "\"" key "\"[[:space:]]*:" { + line = $0 + sub("^.*\"" key "\"[[:space:]]*:[[:space:]]*\"", "", line) + sub("\".*$", "", line) + print line + exit + } + ' +} + +module_path_from_json() { + awk ' + /"Module"[[:space:]]*:/ { in_module = 1; next } + in_module && /"Path"[[:space:]]*:/ { + line = $0 + sub("^.*\"Path\"[[:space:]]*:[[:space:]]*\"", "", line) + sub("\".*$", "", line) + print line + exit + } + ' +} + +parse_edges() { + local owner="$1" + awk -v owner="$owner" ' + function value(line, key) { + sub("^.*\"" key "\"[[:space:]]*:[[:space:]]*\"", "", line) + sub("\".*$", "", line) + return line + } + + /^\t"Require"[[:space:]]*:/ { section = "require"; next } + /^\t"Replace"[[:space:]]*:/ { section = "replace"; next } + /^\t"(Exclude|Retract|Tool|Ignore)"[[:space:]]*:/ { section = ""; next } + + section == "require" && /^\t\t\{/ { + req_path = "" + req_version = "" + next + } + section == "require" && /"Path"[[:space:]]*:/ { + req_path = value($0, "Path") + next + } + section == "require" && /"Version"[[:space:]]*:/ { + req_version = value($0, "Version") + next + } + section == "require" && /^\t\t\},?$/ { + if (req_path != "") { + print "require|" owner "|" req_path "|" req_version + } + next + } + + section == "replace" && /"Old"[[:space:]]*:/ { + replace_part = "old" + old_path = "" + old_version = "" + new_path = "" + new_version = "" + next + } + section == "replace" && /"New"[[:space:]]*:/ { + replace_part = "new" + next + } + section == "replace" && /"Path"[[:space:]]*:/ { + if (replace_part == "old") { + old_path = value($0, "Path") + } else if (replace_part == "new") { + new_path = value($0, "Path") + } + next + } + section == "replace" && /"Version"[[:space:]]*:/ { + if (replace_part == "old") { + old_version = value($0, "Version") + } else if (replace_part == "new") { + new_version = value($0, "Version") + } + next + } + section == "replace" && /^\t\t\},?$/ { + if (old_path != "") { + print "replace|" owner "|" old_path "|" old_version "|" new_path "|" new_version + } + replace_part = "" + next + } + ' +} + +VERSION_VALIDATOR="$ROOT_DIR/scripts/release-version.sh" +[[ -r "$VERSION_VALIDATOR" ]] || fail "scripts/release-version.sh is missing or unreadable" +# shellcheck source=scripts/release-version.sh +source "$VERSION_VALIDATOR" + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + usage + exit 0 + ;; + --release-version) + RELEASE_VERSION="${2:-}" + [[ -n "$RELEASE_VERSION" ]] || fail "--release-version requires a value" + require_version "$RELEASE_VERSION" + shift 2 + ;; + --tag-version) + TAG_VERSION="${2:-}" + [[ -n "$TAG_VERSION" ]] || fail "--tag-version requires a value" + require_version "$TAG_VERSION" + shift 2 + ;; + --exclude) + excluded="${2:-}" + [[ -n "$excluded" ]] || fail "--exclude requires a module directory value" + EXCLUDES+=("$(normalize_dir "$excluded")") + shift 2 + ;; + *) + fail "unknown argument: $1" + ;; + esac +done + +if [[ ${#EXCLUDES[@]} -gt 0 && -z "$RELEASE_VERSION" ]]; then + fail "--exclude requires --release-version" +fi + +[[ -f "$ROOT_DIR/go.mod" ]] || fail "root go.mod is missing" +[[ -f "$ROOT_DIR/go.work" ]] || fail "go.work is missing" +[[ -f "$ROOT_DIR/scripts/module-go-versions.tsv" ]] || fail "scripts/module-go-versions.tsv is missing" +[[ -f "$ROOT_DIR/scripts/dependency-minimums.tsv" ]] || fail "scripts/dependency-minimums.tsv is missing" +[[ -x "$ROOT_DIR/scripts/tag-all-modules.sh" ]] || fail "scripts/tag-all-modules.sh is missing or not executable" +[[ -x "$ROOT_DIR/scripts/plan-module-release-tags.sh" ]] || fail "scripts/plan-module-release-tags.sh is missing or not executable" + +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT + +MODULES_FILE="$TMP_DIR/modules.tsv" +EDGES_FILE="$TMP_DIR/edges.tsv" +REQUIRES_FILE="$TMP_DIR/requires.tsv" +REPLACEMENTS_FILE="$TMP_DIR/replacements.tsv" +EXPECTED_DIRS_FILE="$TMP_DIR/expected-dirs.txt" +WORKSPACE_DIRS_FILE="$TMP_DIR/workspace-dirs.txt" +WORKSPACE_DIRS_RAW_FILE="$TMP_DIR/workspace-dirs-raw.txt" +EXPECTED_RACE_DIRS_FILE="$TMP_DIR/expected-race-dirs.txt" +RACE_DIRS_FILE="$TMP_DIR/race-dirs.txt" +RACE_DIRS_RAW_FILE="$TMP_DIR/race-dirs-raw.txt" +EXPECTED_MINIMUM_GO_VERSIONS_FILE="$TMP_DIR/expected-minimum-go-versions.txt" +MINIMUM_GO_VERSIONS_FILE="$TMP_DIR/minimum-go-versions.txt" +MINIMUM_GO_VERSIONS_RAW_FILE="$TMP_DIR/minimum-go-versions-raw.txt" +WORKSPACE_REPLACEMENTS_FILE="$TMP_DIR/workspace-replacements.tsv" +GO_VERSIONS_FILE="$TMP_DIR/module-go-versions.tsv" +DEPENDENCY_MINIMUMS_FILE="$TMP_DIR/dependency-minimums.tsv" + +if ! awk ' + /^[[:space:]]*(#|$)/ { next } + NF != 2 { exit 1 } + { print $1 "\t" $2 } +' "$ROOT_DIR/scripts/module-go-versions.tsv" >"$GO_VERSIONS_FILE"; then + fail "scripts/module-go-versions.tsv must contain module-directory and Go-version pairs" +fi +[[ -s "$GO_VERSIONS_FILE" ]] || fail "scripts/module-go-versions.tsv has no module policies" + +duplicate_go_policy="$(awk -F '\t' 'seen[$1]++ { print $1; exit }' "$GO_VERSIONS_FILE")" +[[ -z "$duplicate_go_policy" ]] || fail "scripts/module-go-versions.tsv contains duplicate module $duplicate_go_policy" + +if ! awk ' + /^[[:space:]]*(#|$)/ { next } + NF != 2 { exit 1 } + { print $1 "\t" $2 } +' "$ROOT_DIR/scripts/dependency-minimums.tsv" >"$DEPENDENCY_MINIMUMS_FILE"; then + fail "scripts/dependency-minimums.tsv must contain dependency and minimum-version pairs" +fi +[[ -s "$DEPENDENCY_MINIMUMS_FILE" ]] || fail "scripts/dependency-minimums.tsv has no dependency policies" + +duplicate_dependency_policy="$(awk -F '\t' 'seen[$1]++ { print $1; exit }' "$DEPENDENCY_MINIMUMS_FILE")" +[[ -z "$duplicate_dependency_policy" ]] || fail "scripts/dependency-minimums.tsv contains duplicate policy $duplicate_dependency_policy" + +while IFS= read -r mod_file; do + rel_file="${mod_file#./}" + dir="$(normalize_dir "$(dirname "$rel_file")")" + json_file="$TMP_DIR/module-${dir//\//_}.json" + if [[ "$dir" == "." ]]; then + json_file="$TMP_DIR/module-root.json" + fi + + (cd "$ROOT_DIR" && GOWORK=off go mod edit -json "$rel_file") >"$json_file" + module_path="$(module_path_from_json <"$json_file")" + go_version="$(json_value Go <"$json_file")" + [[ -n "$module_path" ]] || fail "could not read module path from $rel_file" + [[ -n "$go_version" ]] || fail "could not read Go version from $rel_file" + + printf '%s\t%s\t%s\n' "$dir" "$module_path" "$go_version" >>"$MODULES_FILE" + parse_edges "$dir" <"$json_file" >>"$EDGES_FILE" +done < <( + cd "$ROOT_DIR" + find . -name go.mod -type f \ + -not -path './.git/*' \ + -not -path './*/.git/*' \ + -not -path './*/vendor/*' \ + | sort +) + +[[ -s "$MODULES_FILE" ]] || fail "no Go modules discovered" +awk -F '|' '$1 == "require"' "$EDGES_FILE" >"$REQUIRES_FILE" +awk -F '|' '$1 == "replace"' "$EDGES_FILE" >"$REPLACEMENTS_FILE" +root_module="$(awk -F '\t' '$1 == "." { print $2; exit }' "$MODULES_FILE")" +root_go_version="$(awk -F '\t' '$1 == "." { print $3; exit }' "$MODULES_FILE")" +[[ -n "$root_module" ]] || fail "root module was not discovered" + +highest_go_version="$root_go_version" + +module_count=0 +while IFS=$'\t' read -r dir module_path go_version; do + module_count=$((module_count + 1)) + expected_path="$root_module" + if [[ "$dir" != "." ]]; then + expected_path="$root_module/$dir" + fi + [[ "$module_path" == "$expected_path" ]] || fail "$dir declares $module_path; expected $expected_path" + expected_go_version="$(awk -F '\t' -v dir="$dir" '$1 == dir { print $2; exit }' "$GO_VERSIONS_FILE")" + [[ -n "$expected_go_version" ]] || fail "scripts/module-go-versions.tsv has no policy for $dir" + go_version_as_semver "$expected_go_version" >/dev/null || fail "$dir has invalid policy Go version $expected_go_version" + [[ "$go_version" == "$expected_go_version" ]] || fail "$dir uses Go $go_version; policy requires Go $expected_go_version" + if [[ "$go_version" != "$highest_go_version" ]] && go_version_at_least "$go_version" "$highest_go_version"; then + highest_go_version="$go_version" + fi + module_release_version="${RELEASE_VERSION:-$TAG_VERSION}" + if [[ -n "$module_release_version" ]] && ! module_is_excluded "$dir"; then + if ! validate_release_module_path "$module_path" "$module_release_version"; then + fail "$RELEASE_VERSION_ERROR" + fi + fi + printf '%s\n' "$dir" >>"$EXPECTED_DIRS_FILE" +done <"$MODULES_FILE" +sort -u -o "$EXPECTED_DIRS_FILE" "$EXPECTED_DIRS_FILE" + +while IFS=$'\t' read -r policy_dir _; do + normalized_policy_dir="$(normalize_dir "$policy_dir")" + [[ "$policy_dir" == "$normalized_policy_dir" ]] || fail "scripts/module-go-versions.tsv uses non-canonical directory $policy_dir" + awk -F '\t' -v dir="$policy_dir" '$1 == dir { found = 1 } END { exit !found }' "$MODULES_FILE" || \ + fail "scripts/module-go-versions.tsv references unknown module $policy_dir" +done <"$GO_VERSIONS_FILE" + +workspace_json="$TMP_DIR/workspace.json" +(cd "$ROOT_DIR" && go work edit -json "$ROOT_DIR/go.work") >"$workspace_json" +parse_edges "go.work" <"$workspace_json" | awk -F '|' '$1 == "replace"' >"$WORKSPACE_REPLACEMENTS_FILE" +workspace_go_version="$(json_value Go <"$workspace_json")" +[[ "$workspace_go_version" == "$highest_go_version" ]] || fail "go.work uses Go $workspace_go_version; highest module policy requires Go $highest_go_version" + +while IFS= read -r disk_path; do + [[ -n "$disk_path" ]] || continue + if [[ "$disk_path" == /* ]]; then + absolute_path="$disk_path" + else + absolute_path="$ROOT_DIR/$disk_path" + fi + [[ -d "$absolute_path" ]] || fail "go.work references missing directory: $disk_path" + canonical_path="$(cd "$absolute_path" && pwd -P)" + if [[ "$canonical_path" == "$ROOT_DIR" ]]; then + workspace_dir="." + elif [[ "$canonical_path" == "$ROOT_DIR/"* ]]; then + workspace_dir="${canonical_path#"$ROOT_DIR/"}" + else + fail "go.work references a module outside the repository: $disk_path" + fi + printf '%s\n' "$workspace_dir" >>"$WORKSPACE_DIRS_RAW_FILE" +done < <(awk '/"DiskPath"[[:space:]]*:/ { line = $0; sub("^.*\"DiskPath\"[[:space:]]*:[[:space:]]*\"", "", line); sub("\".*$", "", line); print line }' "$workspace_json") +sort -u "$WORKSPACE_DIRS_RAW_FILE" >"$WORKSPACE_DIRS_FILE" +workspace_entry_count="$(wc -l <"$WORKSPACE_DIRS_RAW_FILE" | tr -d ' ')" +workspace_unique_count="$(wc -l <"$WORKSPACE_DIRS_FILE" | tr -d ' ')" +[[ "$workspace_entry_count" -eq "$workspace_unique_count" ]] || fail "go.work contains duplicate module entries" + +if ! diff -u "$EXPECTED_DIRS_FILE" "$WORKSPACE_DIRS_FILE" >"$TMP_DIR/workspace.diff"; then + cat "$TMP_DIR/workspace.diff" >&2 + fail "go.work membership differs from the discovered module inventory" +fi + +while IFS=$'\t' read -r dependency minimum_version; do + if ! semver_at_least "$minimum_version" "$minimum_version"; then + fail "$dependency has invalid policy minimum $minimum_version" + fi + + dependency_requirements="$(awk -F '|' -v dependency="$dependency" \ + '$1 == "require" && $3 == dependency { print }' "$REQUIRES_FILE")" + [[ -n "$dependency_requirements" ]] || fail "policy dependency $dependency is not directly required by any module" + while IFS='|' read -r _ owner _ required_version; do + if semver_at_least "$required_version" "$minimum_version"; then + continue + else + comparison_status=$? + fi + if [[ "$comparison_status" -eq 2 ]]; then + fail "$owner requires $dependency at invalid semantic version $required_version" + fi + fail "$owner requires $dependency at $required_version; policy requires at least $minimum_version" + done <<<"$dependency_requirements" + + replacement_owner="$(awk -F '|' -v dependency="$dependency" \ + '$1 == "replace" && $3 == dependency { print $2; exit }' "$REPLACEMENTS_FILE")" + [[ -z "$replacement_owner" ]] || fail "$replacement_owner must not replace policy dependency $dependency" + if awk -F '|' -v dependency="$dependency" '$1 == "replace" && $3 == dependency { found = 1 } END { exit !found }' "$WORKSPACE_REPLACEMENTS_FILE"; then + fail "go.work must not replace policy dependency $dependency" + fi +done <"$DEPENDENCY_MINIMUMS_FILE" + +RACE_WORKFLOW="$ROOT_DIR/.github/workflows/test.yml" +[[ -f "$RACE_WORKFLOW" ]] || fail ".github/workflows/test.yml is missing" + +awk -F '\t' '{ print $2 }' "$GO_VERSIONS_FILE" | sort -u >"$EXPECTED_MINIMUM_GO_VERSIONS_FILE" +if ! awk ' + /minimum_go_version:[[:space:]]*\[/ { + line = $0 + sub(/^.*minimum_go_version:[[:space:]]*\[/, "", line) + sub(/\].*$/, "", line) + count = split(line, entries, ",") + for (i = 1; i <= count; i++) { + value = entries[i] + gsub(/^[[:space:]"]+|[[:space:]"]+$/, "", value) + if (value != "") print value + } + found = 1 + exit + } + END { if (!found) exit 1 } +' "$RACE_WORKFLOW" >"$MINIMUM_GO_VERSIONS_RAW_FILE"; then + fail ".github/workflows/test.yml has no inline minimum_go_version matrix" +fi +sort -u "$MINIMUM_GO_VERSIONS_RAW_FILE" >"$MINIMUM_GO_VERSIONS_FILE" +minimum_go_entry_count="$(wc -l <"$MINIMUM_GO_VERSIONS_RAW_FILE" | tr -d ' ')" +minimum_go_unique_count="$(wc -l <"$MINIMUM_GO_VERSIONS_FILE" | tr -d ' ')" +[[ "$minimum_go_entry_count" -eq "$minimum_go_unique_count" ]] || fail "CI minimum Go matrix contains duplicate versions" +if ! diff -u "$EXPECTED_MINIMUM_GO_VERSIONS_FILE" "$MINIMUM_GO_VERSIONS_FILE" >"$TMP_DIR/minimum-go-matrix.diff"; then + cat "$TMP_DIR/minimum-go-matrix.diff" >&2 + fail "CI minimum Go matrix must match the module Go version policy" +fi + +{ + printf '.\n' + awk -F '\t' 'index($1, "driver/") == 1 { print $1 }' "$MODULES_FILE" +} | sort -u >"$EXPECTED_RACE_DIRS_FILE" + +if ! awk ' + /race_module:[[:space:]]*\[/ { + line = $0 + sub(/^.*race_module:[[:space:]]*\[/, "", line) + sub(/\].*$/, "", line) + count = split(line, entries, ",") + for (i = 1; i <= count; i++) { + value = entries[i] + gsub(/^[[:space:]"]+|[[:space:]"]+$/, "", value) + if (value != "") { + print value + } + } + found = 1 + exit + } + END { if (!found) exit 1 } +' "$RACE_WORKFLOW" >"$RACE_DIRS_RAW_FILE"; then + fail ".github/workflows/test.yml has no inline race_module matrix" +fi + +sort -u "$RACE_DIRS_RAW_FILE" >"$RACE_DIRS_FILE" +race_entry_count="$(wc -l <"$RACE_DIRS_RAW_FILE" | tr -d ' ')" +race_unique_count="$(wc -l <"$RACE_DIRS_FILE" | tr -d ' ')" +[[ "$race_entry_count" -eq "$race_unique_count" ]] || fail "CI race matrix contains duplicate module entries" + +if ! diff -u "$EXPECTED_RACE_DIRS_FILE" "$RACE_DIRS_FILE" >"$TMP_DIR/race-matrix.diff"; then + cat "$TMP_DIR/race-matrix.diff" >&2 + fail "CI race matrix must contain root and every discovered driver module" +fi + +LOCAL_VERSIONS_FILE="$TMP_DIR/local-versions.txt" +while IFS='|' read -r kind owner required_path required_version; do + [[ "$kind" == "require" ]] || continue + target_dir="$(awk -F '\t' -v path="$required_path" '$2 == path { print $1; exit }' "$MODULES_FILE")" + [[ -n "$target_dir" ]] || continue + + printf '%s\n' "$required_version" >>"$LOCAL_VERSIONS_FILE" + replacement="$(awk -F '|' -v owner="$owner" -v path="$required_path" '$2 == owner && $3 == path && $4 == "" { print; exit }' "$REPLACEMENTS_FILE")" + [[ -n "$replacement" ]] || fail "$owner requires sibling $required_path without an unversioned local replacement" + + IFS='|' read -r _ _ _ _ new_path new_version <<<"$replacement" + [[ -z "$new_version" ]] || fail "$owner replaces sibling $required_path with version $new_version instead of a local directory" + [[ -n "$new_path" ]] || fail "$owner has an empty replacement target for sibling $required_path" + + owner_path="$ROOT_DIR" + if [[ "$owner" != "." ]]; then + owner_path="$ROOT_DIR/$owner" + fi + if [[ "$new_path" == /* ]]; then + replacement_path="$new_path" + else + replacement_path="$owner_path/$new_path" + fi + [[ -d "$replacement_path" ]] || fail "$owner replacement for $required_path points to missing directory $new_path" + replacement_path="$(cd "$replacement_path" && pwd -P)" + expected_target_path="$ROOT_DIR" + if [[ "$target_dir" != "." ]]; then + expected_target_path="$ROOT_DIR/$target_dir" + fi + expected_target_path="$(cd "$expected_target_path" && pwd -P)" + [[ "$replacement_path" == "$expected_target_path" ]] || fail "$owner replacement for $required_path points to $new_path, not $target_dir" + + if [[ -n "$RELEASE_VERSION" ]] && ! module_is_excluded "$owner"; then + if [[ "$required_version" != "$RELEASE_VERSION" ]]; then + fail "$owner requires sibling $required_path at $required_version; release $RELEASE_VERSION requires a resolvable $RELEASE_VERSION pin" + fi + if module_is_excluded "$target_dir"; then + required_tag="$RELEASE_VERSION" + if [[ "$target_dir" != "." ]]; then + required_tag="$target_dir/$RELEASE_VERSION" + fi + fail "$owner is included but requires excluded sibling $required_path; release $RELEASE_VERSION would omit required tag $required_tag" + fi + fi +done <"$REQUIRES_FILE" + +if [[ -s "$LOCAL_VERSIONS_FILE" ]]; then + sort -u -o "$LOCAL_VERSIONS_FILE" "$LOCAL_VERSIONS_FILE" + local_version_count="$(wc -l <"$LOCAL_VERSIONS_FILE" | tr -d ' ')" + if [[ "$local_version_count" -ne 1 ]]; then + echo "sibling requirement versions:" >&2 + sed 's/^/ - /' "$LOCAL_VERSIONS_FILE" >&2 + fail "sibling module requirement versions have drifted" + fi +fi + +while IFS='|' read -r kind owner old_path old_version new_path new_version; do + [[ "$kind" == "replace" ]] || continue + target_dir="$(awk -F '\t' -v path="$old_path" '$2 == path { print $1; exit }' "$MODULES_FILE")" + [[ -n "$target_dir" ]] || continue + [[ -z "$old_version" ]] || fail "$owner uses a version-specific replacement for sibling $old_path" + [[ -z "$new_version" ]] || fail "$owner replaces sibling $old_path with non-local version $new_version" + [[ -n "$new_path" ]] || fail "$owner has an empty replacement target for sibling $old_path" + + owner_path="$ROOT_DIR" + if [[ "$owner" != "." ]]; then + owner_path="$ROOT_DIR/$owner" + fi + if [[ "$new_path" == /* ]]; then + replacement_path="$new_path" + else + replacement_path="$owner_path/$new_path" + fi + [[ -d "$replacement_path" ]] || fail "$owner replacement for $old_path points to missing directory $new_path" + replacement_path="$(cd "$replacement_path" && pwd -P)" + expected_target_path="$ROOT_DIR" + if [[ "$target_dir" != "." ]]; then + expected_target_path="$ROOT_DIR/$target_dir" + fi + expected_target_path="$(cd "$expected_target_path" && pwd -P)" + [[ "$replacement_path" == "$expected_target_path" ]] || fail "$owner replacement for $old_path points to $new_path, not $target_dir" +done <"$REPLACEMENTS_FILE" + +guard_version="${RELEASE_VERSION:-$TAG_VERSION}" +if [[ -z "$guard_version" ]]; then + guard_major="0" + if [[ "$root_module" == gopkg.in/* && "$root_module" =~ \.v(0|[1-9][0-9]*)$ ]]; then + guard_major="${BASH_REMATCH[1]}" + elif [[ "$root_module" =~ /v([2-9][0-9]*)$ ]]; then + guard_major="${BASH_REMATCH[1]}" + fi + guard_version="v$guard_major.0.0-module-inventory-guard" +fi +release_output="$TMP_DIR/release-output.txt" +if ! "$ROOT_DIR/scripts/plan-module-release-tags.sh" "$guard_version" >"$release_output" 2>&1; then + cat "$release_output" >&2 + fail "release tag planner failed" +fi + +EXPECTED_TAGS_FILE="$TMP_DIR/expected-tags.txt" +ACTUAL_TAGS_FILE="$TMP_DIR/actual-tags.txt" +while IFS=$'\t' read -r dir _; do + if [[ "$dir" == "." ]]; then + printf '%s\n' "$guard_version" >>"$EXPECTED_TAGS_FILE" + else + printf '%s/%s\n' "$dir" "$guard_version" >>"$EXPECTED_TAGS_FILE" + fi +done <"$MODULES_FILE" +sort -u "$release_output" >"$ACTUAL_TAGS_FILE" +sort -u -o "$EXPECTED_TAGS_FILE" "$EXPECTED_TAGS_FILE" +if ! diff -u "$EXPECTED_TAGS_FILE" "$ACTUAL_TAGS_FILE" >"$TMP_DIR/tags.diff"; then + cat "$TMP_DIR/tags.diff" >&2 + fail "release tag planner does not cover the discovered module inventory" +fi + +if [[ -n "$TAG_VERSION" ]]; then + root_tag_commit="$(git -C "$ROOT_DIR" rev-parse -q --verify "refs/tags/$TAG_VERSION^{commit}")" || fail "missing root tag $TAG_VERSION" + while IFS=$'\t' read -r dir _; do + tag="$TAG_VERSION" + if [[ "$dir" != "." ]]; then + tag="$dir/$TAG_VERSION" + fi + tag_commit="$(git -C "$ROOT_DIR" rev-parse -q --verify "refs/tags/$tag^{commit}")" || fail "missing module tag $tag" + [[ "$tag_commit" == "$root_tag_commit" ]] || fail "$tag points to $tag_commit; root $TAG_VERSION points to $root_tag_commit" + done <"$MODULES_FILE" +fi + +echo "module inventory guard: $module_count modules, Go $root_go_version root/$highest_go_version workspace, dependencies/minimum-CI/race/replacements/release coverage OK" +if [[ -s "$LOCAL_VERSIONS_FILE" ]]; then + echo "module inventory guard: sibling requirement version $(head -n 1 "$LOCAL_VERSIONS_FILE")" +fi +if [[ -n "$TAG_VERSION" ]]; then + echo "module inventory guard: tag family $TAG_VERSION resolves to $root_tag_commit" +fi diff --git a/scripts/coverage-codecov.sh b/scripts/coverage-codecov.sh index 2f44b29..d9fdf16 100755 --- a/scripts/coverage-codecov.sh +++ b/scripts/coverage-codecov.sh @@ -1,31 +1,235 @@ #!/usr/bin/env bash set -euo pipefail +# Collects deterministic Codecov profiles across this repository's Go modules. +# +# Unit mode runs every buildable module without crossing go.mod boundaries and +# emits one profile plus a module manifest. Integration mode runs the real, +# tagged integration module for one backend (or "all") and emits one profile. +# Both modes collapse duplicate source ranges produced when -coverpkg spans +# multiple test binaries. +# +# Usage: +# scripts/coverage-codecov.sh unit +# INTEGRATION_BACKEND=redis scripts/coverage-codecov.sh integration + ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" -cd "$ROOT_DIR" +MODE="${1:-unit}" +OUTPUT_DIR="${COVERAGE_OUTPUT_DIR:-$ROOT_DIR/coverage}" +TMP_PARENT="${COVERAGE_TMP_DIR:-/tmp}" +GOCACHE_DIR="${GOCACHE:-${GOCACHE_DIR:-/tmp/gocache}}" +GOMODCACHE_DIR="${GOMODCACHE:-/tmp/gomodcache}" +ROOT_MODULE="$(awk '$1 == "module" { print $2; exit }' "$ROOT_DIR/go.mod")" + +fail() { + echo "coverage collection: $*" >&2 + exit 1 +} + +absolute_output_path() { + local path="$1" + if [[ "$path" == /* ]]; then + printf '%s\n' "$path" + return + fi + printf '%s/%s\n' "$ROOT_DIR" "$path" +} + +validate_profile() { + local profile="$1" + [[ -s "$profile" ]] || fail "profile is missing or empty: $profile" + [[ "$(head -n 1 "$profile")" == "mode: atomic" ]] || fail "profile is not atomic: $profile" + [[ "$(grep -c '^mode:' "$profile")" -eq 1 ]] || fail "profile has multiple mode headers: $profile" + + awk -v prefix="$ROOT_MODULE/" ' + NR == 1 { next } + index($1, prefix) != 1 { + print "coverage collection: profile path is outside the repository: " $1 > "/dev/stderr" + exit 1 + } + seen[$1]++ { + print "coverage collection: duplicate source range in profile: " $1 > "/dev/stderr" + exit 1 + } + ' "$profile" +} + +merge_profiles() { + local output="$1" + shift + local profiles=("$@") + [[ "${#profiles[@]}" -gt 0 ]] || fail "no profiles were provided for $output" + + local records_file="$TMP_ROOT/records.txt" + local sorted_file="$TMP_ROOT/records.sorted.txt" + local merged_file="$TMP_ROOT/merged.out" + : >"$records_file" + + local profile + for profile in "${profiles[@]}"; do + [[ -s "$profile" ]] || fail "raw profile is missing or empty: $profile" + [[ "$(head -n 1 "$profile")" == "mode: atomic" ]] || fail "raw profile is not atomic: $profile" + tail -n +2 "$profile" >>"$records_file" + done + + LC_ALL=C sort -k1,1 "$records_file" >"$sorted_file" + { + printf 'mode: atomic\n' + awk ' + function emit() { + if (range_key != "") { + printf "%s %s %.0f\n", range_key, statements, hits + } + } + { + if ($1 != range_key) { + emit() + range_key = $1 + statements = $2 + hits = $3 + next + } + if ($2 != statements) { + print "coverage collection: statement count mismatch for " range_key > "/dev/stderr" + exit 1 + } + hits += $3 + } + END { emit() } + ' "$sorted_file" + } >"$merged_file" + + mkdir -p "$(dirname "$output")" + cp "$merged_file" "$output" + validate_profile "$output" +} + +print_summary() { + local profile="$1" + local ranges + ranges="$(awk 'END { print NR - 1 }' "$profile")" + echo "==> wrote $profile ($ranges source ranges)" + (cd "$ROOT_DIR" && GOWORK="$ROOT_DIR/go.work" go tool cover -func="$profile" | tail -n 1) +} + +collect_unit() { + local output + output="$(absolute_output_path "${COVERAGE_OUTPUT:-$OUTPUT_DIR/coverage-unit.out}")" + local manifest + manifest="$(absolute_output_path "${COVERAGE_MODULE_MANIFEST:-$OUTPUT_DIR/coverage-unit-modules.tsv}")" + local manifest_tmp="$TMP_ROOT/unit-modules.tsv" + printf '# module_directory\tmodule_path\n' >"$manifest_tmp" + + "$ROOT_DIR/scripts/check-module-inventory.sh" + + local raw_profiles=() + local module_count=0 + local module_file module_dir relative_dir module_path package_list slug raw_profile list_stderr + while IFS= read -r module_file; do + module_count=$((module_count + 1)) + module_dir="$(dirname "$module_file")" + relative_dir="${module_dir#"$ROOT_DIR"}" + relative_dir="${relative_dir#/}" + if [[ -z "$relative_dir" ]]; then + relative_dir="." + fi + module_path="$(awk '$1 == "module" { print $2; exit }' "$module_file")" + [[ -n "$module_path" ]] || fail "could not read module path from $module_file" + + list_stderr="$TMP_ROOT/list-${relative_dir//\//-}.stderr" + if ! package_list="$({ + cd "$module_dir" + GOWORK=off GOCACHE="$GOCACHE_DIR" GOMODCACHE="$GOMODCACHE_DIR" go list ./... + } 2>"$list_stderr")"; then + cat "$list_stderr" >&2 + fail "could not list packages in $relative_dir" + fi + if [[ -z "$package_list" ]]; then + if [[ "$relative_dir" == "docs" ]]; then + local tagged_profile="$TMP_ROOT/unit-docs-testcounts-tag.out" + echo "==> unit coverage: docs tagged test-count evidence" + ( + cd "$module_dir" + GOWORK=off GOCACHE="$GOCACHE_DIR" GOMODCACHE="$GOMODCACHE_DIR" \ + go test -count=1 -tags=testcounts -covermode=atomic -coverpkg=./readme/testcounts \ + -coverprofile="$tagged_profile" ./readme/testcounts + ) + raw_profiles+=("$tagged_profile") + printf '%s\t%s\n' "$relative_dir" "$module_path" >>"$manifest_tmp" + continue + fi + echo "==> $relative_dir has no buildable packages; skipping coverage" + continue + fi + + slug="${relative_dir//\//-}" + if [[ "$slug" == "." ]]; then + slug="root" + fi + raw_profile="$TMP_ROOT/unit-$slug.out" + echo "==> unit coverage: $relative_dir" + ( + cd "$module_dir" + GOWORK=off GOCACHE="$GOCACHE_DIR" GOMODCACHE="$GOMODCACHE_DIR" \ + go test -count=1 -covermode=atomic -coverpkg=./... -coverprofile="$raw_profile" ./... + ) + raw_profiles+=("$raw_profile") + + if [[ "$relative_dir" == "." ]]; then + local tagged_profile="$TMP_ROOT/unit-root-integration-tag.out" + echo "==> unit coverage: root integration-tagged bus fixtures" + ( + cd "$module_dir" + GOWORK=off GOCACHE="$GOCACHE_DIR" GOMODCACHE="$GOMODCACHE_DIR" \ + go test -count=1 -tags=integration -covermode=atomic -coverpkg=./... \ + -coverprofile="$tagged_profile" ./bus + ) + raw_profiles+=("$tagged_profile") + fi + printf '%s\t%s\n' "$relative_dir" "$module_path" >>"$manifest_tmp" + done < <( + find "$ROOT_DIR" -type f -name go.mod \ + -not -path '*/.git/*' \ + -not -path '*/vendor/*' \ + -print | LC_ALL=C sort + ) + [[ "$module_count" -gt 0 ]] || fail "no Go modules were discovered" -OUTPUT_FILE="${COVERAGE_OUTPUT:-coverage.txt}" -TMP_ROOT="${COVERAGE_TMP_DIR:-/tmp/cache-coverage}" -GOCACHE_DIR="${GOCACHE_DIR:-/tmp/go-build-cache}" -INTEGRATION_BACKEND="${INTEGRATION_BACKEND:-all}" + merge_profiles "$output" "${raw_profiles[@]}" + mkdir -p "$(dirname "$manifest")" + cp "$manifest_tmp" "$manifest" + print_summary "$output" +} -UNIT_DIR="$TMP_ROOT/unit" -INT_DIR="$TMP_ROOT/integration" -MERGED_DIR="$TMP_ROOT/merged" +collect_integration() { + local backend="${INTEGRATION_BACKEND:-all}" + case "$backend" in + all|null|sync|workerpool|redis|mysql|postgres|sqlite|nats|sqs|rabbitmq) ;; + *) fail "unsupported INTEGRATION_BACKEND for coverage: $backend" ;; + esac -rm -rf "$TMP_ROOT" -mkdir -p "$UNIT_DIR" "$INT_DIR" "$MERGED_DIR" + local output + output="$(absolute_output_path "${COVERAGE_OUTPUT:-$OUTPUT_DIR/coverage-integration-$backend.out}")" + local raw_profile="$TMP_ROOT/integration-$backend.raw.out" -echo "==> Unit coverage" -GOCACHE="$GOCACHE_DIR" \ -go test -cover -coverpkg=./... ./... -args -test.gocoverdir="$UNIT_DIR" + echo "==> integration coverage: $backend" + ( + cd "$ROOT_DIR/integration" + INTEGRATION_BACKEND="$backend" GOWORK=off GOCACHE="$GOCACHE_DIR" GOMODCACHE="$GOMODCACHE_DIR" \ + go test -p=1 -count=1 -tags=integration -covermode=atomic \ + -coverpkg="$ROOT_MODULE/..." -coverprofile="$raw_profile" ./... + ) -echo "==> Integration coverage (backend=${INTEGRATION_BACKEND})" -INTEGRATION_BACKEND="$INTEGRATION_BACKEND" GOCACHE="$GOCACHE_DIR" \ -go test -cover -tags integration -coverpkg=./... ./... -args -test.gocoverdir="$INT_DIR" + merge_profiles "$output" "$raw_profile" + print_summary "$output" +} -echo "==> Merge coverage" -go tool covdata merge -i="$UNIT_DIR,$INT_DIR" -o="$MERGED_DIR" -go tool covdata textfmt -i="$MERGED_DIR" -o="$OUTPUT_FILE" +mkdir -p "$TMP_PARENT" +TMP_ROOT="$(mktemp -d "$TMP_PARENT/queue-codecov.XXXXXX")" +trap 'rm -rf -- "$TMP_ROOT"' EXIT -go tool cover -func="$OUTPUT_FILE" | tail -n 1 +case "$MODE" in + unit) collect_unit ;; + integration) collect_integration ;; + *) fail "usage: scripts/coverage-codecov.sh [unit|integration]" ;; +esac diff --git a/scripts/dependency-minimums.tsv b/scripts/dependency-minimums.tsv new file mode 100644 index 0000000..2488b83 --- /dev/null +++ b/scripts/dependency-minimums.tsv @@ -0,0 +1,2 @@ +# Direct dependency floors that protect repository security or compatibility constraints. +github.com/jackc/pgx/v5 v5.9.2 diff --git a/scripts/integration-flake-repeat.sh b/scripts/integration-flake-repeat.sh index 413879d..55f89e7 100755 --- a/scripts/integration-flake-repeat.sh +++ b/scripts/integration-flake-repeat.sh @@ -3,7 +3,24 @@ set -euo pipefail repeat_count="${FLAKE_REPEAT_COUNT:-5}" artifacts_dir="${FLAKE_ARTIFACTS_DIR:-.artifacts}" -backend_label="${INTEGRATION_BACKEND:-all}" +backend_label="${INTEGRATION_BACKEND:-}" + +case "${backend_label}" in + redis | mysql | postgres | sqlite | nats | sqs | rabbitmq) ;; + "" | all | *,*) + echo "INTEGRATION_BACKEND must name exactly one backend" >&2 + exit 2 + ;; + *) + echo "unsupported INTEGRATION_BACKEND: ${backend_label}" >&2 + exit 2 + ;; +esac + +if ! command -v jq >/dev/null 2>&1; then + echo "jq is required to validate go test JSON events" >&2 + exit 2 +fi default_scenarios=( "scenario_multi_worker_contention" @@ -47,35 +64,57 @@ attempts_tsv="${artifacts_dir}/integration-flake-${backend_label}-attempts.tsv" echo "| Repeat count | \`${repeat_count}\` |" echo "| Scenario count | \`${#scenarios[@]}\` |" echo - echo "| Scenario | Pass | Fail | Flake Rate |" - echo "|---|---:|---:|---:|" + echo "Skipped attempts are documented capability gates and are excluded from the flake rate. A missing expected test event is a failure." + echo + echo "| Scenario | Pass | Fail | Skip | Missing | Flake Rate |" + echo "|---|---:|---:|---:|---:|---:|" } >"${summary_md}" total_pass=0 total_fail=0 +total_skip=0 +total_missing=0 overall_exit=0 for scenario in "${scenarios[@]}"; do scenario_pass=0 scenario_fail=0 + scenario_skip=0 + scenario_missing=0 for attempt in $(seq 1 "${repeat_count}"); do safe_name="$(tr '/:' '__' <<<"${scenario}")" - log_file="${artifacts_dir}/integration-flake-${backend_label}-${safe_name}-run${attempt}.log" - run_pattern="^TestIntegrationScenarios_AllBackends/.*/${scenario}$" + log_file="${artifacts_dir}/integration-flake-${backend_label}-${safe_name}-run${attempt}.jsonl" + run_pattern="^TestIntegrationScenarios_AllBackends$/^${backend_label}$/${scenario}$" + expected_test="TestIntegrationScenarios_AllBackends/${backend_label}/${scenario}" - status="pass" - if go test -tags=integration ./integration/... -run "${run_pattern}" -count=1 -v >"${log_file}" 2>&1; then + test_exit=0 + go test -tags=integration ./integration/... -run "${run_pattern}" -count=1 -json >"${log_file}" 2>&1 || test_exit=$? + + status="fail" + if [[ "${test_exit}" -ne 0 ]]; then + scenario_fail=$((scenario_fail + 1)) + total_fail=$((total_fail + 1)) + overall_exit=1 + elif jq -e --arg test "${expected_test}" 'select(.Action == "pass" and .Test == $test)' "${log_file}" >/dev/null; then + status="pass" scenario_pass=$((scenario_pass + 1)) total_pass=$((total_pass + 1)) + elif jq -e --arg test "${expected_test}" 'select(.Action == "skip" and .Test == $test)' "${log_file}" >/dev/null; then + status="skip" + scenario_skip=$((scenario_skip + 1)) + total_skip=$((total_skip + 1)) else - status="fail" - scenario_fail=$((scenario_fail + 1)) - total_fail=$((total_fail + 1)) + status="missing" + scenario_missing=$((scenario_missing + 1)) + total_missing=$((total_missing + 1)) overall_exit=1 fi - duration="$(grep -Eo '\[[^]]+\]\[[^]]+\] duration=[^[:space:]]+' "${log_file}" | tail -n 1 | sed -E 's/^.* duration=([^[:space:]]+)$/\1/' || true)" + duration="$(jq -r 'select(.Action == "output") | .Output // empty' "${log_file}" \ + | grep -Eo '\[[^]]+\]\[[^]]+\] duration=[^[:space:]]+' \ + | tail -n 1 \ + | sed -E 's/^.* duration=([^[:space:]]+)$/\1/' || true)" if [[ -z "${duration}" ]]; then duration="n/a" fi @@ -86,7 +125,9 @@ for scenario in "${scenarios[@]}"; do done flake_rate="$(awk -v p="${scenario_pass}" -v f="${scenario_fail}" 'BEGIN { t=p+f; if (t==0) { print "n/a" } else { printf "%.1f%%", (f*100)/t } }')" - printf '| `%s` | %d | %d | `%s` |\n' "${scenario}" "${scenario_pass}" "${scenario_fail}" "${flake_rate}" >>"${summary_md}" + printf "| \`%s\` | %d | %d | %d | %d | \`%s\` |\n" \ + "${scenario}" "${scenario_pass}" "${scenario_fail}" "${scenario_skip}" "${scenario_missing}" "${flake_rate}" \ + >>"${summary_md}" done { @@ -98,14 +139,16 @@ done echo "| Backend | Scenario | Attempt | Status | Duration | Test Output |" echo "|---|---|---:|---|---|---|" tail -n +2 "${attempts_tsv}" | while IFS=$'\t' read -r b s a st d out; do - printf '| `%s` | `%s` | %s | %s | `%s` | `%s` |\n' "${b}" "${s}" "${a}" "${st}" "${d}" "${out}" + printf "| \`%s\` | \`%s\` | %s | %s | \`%s\` | \`%s\` |\n" "${b}" "${s}" "${a}" "${st}" "${d}" "${out}" done echo echo "### Totals" echo - echo "- Total attempts: $((total_pass + total_fail))" + echo "- Total attempts: $((total_pass + total_fail + total_skip + total_missing))" echo "- Pass: ${total_pass}" echo "- Fail: ${total_fail}" + echo "- Skip: ${total_skip}" + echo "- Missing expected event: ${total_missing}" } >>"${summary_md}" echo "Wrote ${summary_md}" @@ -120,16 +163,18 @@ cat "${summary_md}" if [[ "${overall_exit}" -ne 0 ]]; then echo echo "=== Failing Attempts (${backend_label}) ===" - awk -F '\t' 'NR==1 || $4=="fail"' "${attempts_tsv}" + awk -F '\t' 'NR==1 || $4=="fail" || $4=="missing"' "${attempts_tsv}" echo echo "=== Failing Log Tails (${backend_label}) ===" - awk -F '\t' 'NR>1 && $4=="fail" {print $6}' "${attempts_tsv}" | while IFS= read -r out; do + awk -F '\t' 'NR>1 && ($4=="fail" || $4=="missing") {print $6}' "${attempts_tsv}" | while IFS= read -r out; do [[ -z "${out}" ]] && continue file="${artifacts_dir}/${out}" echo "--- ${file} ---" if [[ -f "${file}" ]]; then - tail -n 120 "${file}" + if ! jq -r 'select(.Action == "output") | .Output // empty' "${file}" | tail -n 120; then + tail -n 120 "${file}" + fi else echo "missing log file: ${file}" fi diff --git a/scripts/module-go-versions.tsv b/scripts/module-go-versions.tsv new file mode 100644 index 0000000..21a27c3 --- /dev/null +++ b/scripts/module-go-versions.tsv @@ -0,0 +1,13 @@ +# Exact module Go versions. The workspace uses the highest version in this file. +. 1.24.4 +docs 1.24.4 +driver/mysqlqueue 1.24.4 +driver/natsqueue 1.24.4 +driver/postgresqueue 1.25.0 +driver/rabbitmqqueue 1.24.4 +driver/redisqueue 1.24.4 +driver/sqlitequeue 1.24.4 +driver/sqlqueuecore 1.24.4 +driver/sqsqueue 1.24.4 +examples 1.25.0 +integration 1.25.0 diff --git a/scripts/plan-module-release-tags.sh b/scripts/plan-module-release-tags.sh new file mode 100755 index 0000000..931f456 --- /dev/null +++ b/scripts/plan-module-release-tags.sh @@ -0,0 +1,128 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" +VERSION="" +EXCLUDES=() +VERSION_VALIDATOR="$ROOT_DIR/scripts/release-version.sh" + +if [[ ! -r "$VERSION_VALIDATOR" ]]; then + echo "error: release version validator is missing or unreadable: $VERSION_VALIDATOR" >&2 + exit 1 +fi +# shellcheck source=scripts/release-version.sh +source "$VERSION_VALIDATOR" + +usage() { + cat <<'USAGE' +Usage: + scripts/plan-module-release-tags.sh [--exclude ]... + +Prints the release tag for every included Go module without inspecting or +mutating Git tags. +USAGE +} + +normalize_module_dir() { + local dir="$1" + dir="${dir#./}" + dir="${dir%/}" + if [[ -z "$dir" ]]; then + dir="." + fi + printf '%s\n' "$dir" +} + +module_is_excluded() { + local dir="$1" + local excluded + for excluded in "${EXCLUDES[@]}"; do + if [[ "$dir" == "$excluded" ]] || [[ "$dir" == "$excluded/"* ]]; then + return 0 + fi + done + return 1 +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -h|--help) + usage + exit 0 + ;; + --exclude) + excluded="${2:-}" + if [[ -z "$excluded" ]]; then + echo "error: --exclude requires a module directory value" >&2 + exit 1 + fi + EXCLUDES+=("$(normalize_module_dir "$excluded")") + shift 2 + ;; + v*) + if [[ -n "$VERSION" ]]; then + echo "error: multiple versions provided" >&2 + exit 1 + fi + VERSION="$1" + shift + ;; + *) + echo "error: unknown argument: $1" >&2 + usage + exit 1 + ;; + esac +done + +if [[ -z "$VERSION" ]]; then + echo "error: version is required (example: v0.1.3)" >&2 + exit 1 +fi + +if ! validate_release_version "$VERSION"; then + echo "error: $RELEASE_VERSION_ERROR" >&2 + exit 1 +fi + +module_count=0 +while IFS= read -r discovered_dir; do + module_count=$((module_count + 1)) + dir="$(normalize_module_dir "$discovered_dir")" + if module_is_excluded "$dir"; then + continue + fi + + relative_file="go.mod" + if [[ "$dir" != "." ]]; then + relative_file="$dir/go.mod" + fi + + module_path="$(awk '$1 == "module" { value = $2; gsub(/^"|"$/, "", value); print value; exit }' "$ROOT_DIR/$relative_file")" + if [[ -z "$module_path" ]]; then + echo "error: could not read module path from $relative_file" >&2 + exit 1 + fi + if ! validate_release_module_path "$module_path" "$VERSION"; then + echo "error: $RELEASE_VERSION_ERROR" >&2 + exit 1 + fi + + if [[ "$dir" == "." ]]; then + printf '%s\n' "$VERSION" + else + printf '%s/%s\n' "$dir" "$VERSION" + fi +done < <( + cd "$ROOT_DIR" + find . -name go.mod -type f \ + -not -path './.git/*' \ + -not -path './*/.git/*' \ + -not -path './*/vendor/*' \ + -exec dirname {} \; | sed 's#^\./##' | sort +) + +if [[ "$module_count" -eq 0 ]]; then + echo "error: no modules discovered" >&2 + exit 1 +fi diff --git a/scripts/release-version.sh b/scripts/release-version.sh new file mode 100644 index 0000000..4e008df --- /dev/null +++ b/scripts/release-version.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash + +# shellcheck disable=SC2034 # Callers consume these diagnostics after sourcing this file. + +RELEASE_VERSION_ERROR="" +RELEASE_VERSION_MAJOR="" + +validate_release_version() { + local version="$1" + local without_build + local prerelease + local identifier + local -a prerelease_identifiers=() + + RELEASE_VERSION_ERROR="" + RELEASE_VERSION_MAJOR="" + if [[ ! "$version" =~ ^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?(\+([0-9A-Za-z-]+(\.[0-9A-Za-z-]+)*))?$ ]]; then + RELEASE_VERSION_ERROR="invalid Go release version $version: expected vMAJOR.MINOR.PATCH with valid SemVer prerelease/build identifiers" + return 1 + fi + + RELEASE_VERSION_MAJOR="${BASH_REMATCH[1]}" + without_build="${version%%+*}" + if [[ "$without_build" == *-* ]]; then + prerelease="${without_build#*-}" + IFS='.' read -r -a prerelease_identifiers <<<"$prerelease" + for identifier in "${prerelease_identifiers[@]}"; do + if [[ "$identifier" =~ ^[0-9]+$ && "$identifier" == 0[0-9]* ]]; then + RELEASE_VERSION_ERROR="invalid Go release version $version: numeric prerelease identifier $identifier must not contain leading zeroes" + return 1 + fi + done + fi +} + +validate_release_module_path() { + local module_path="$1" + local version="$2" + local version_major + local path_major + + if ! validate_release_version "$version"; then + return 1 + fi + version_major="$RELEASE_VERSION_MAJOR" + + if [[ "$module_path" == gopkg.in/* ]]; then + if [[ ! "$module_path" =~ \.v(0|[1-9][0-9]*)$ ]]; then + RELEASE_VERSION_ERROR="module path $module_path requires .v$version_major for release $version" + return 1 + fi + path_major="${BASH_REMATCH[1]}" + if [[ "$path_major" != "$version_major" ]]; then + RELEASE_VERSION_ERROR="module path $module_path declares major v$path_major but release $version uses v$version_major" + return 1 + fi + return 0 + fi + + if [[ "$module_path" =~ /v([0-9]+)$ ]]; then + path_major="${BASH_REMATCH[1]}" + if [[ "$path_major" == 0* || "$path_major" == "1" ]]; then + RELEASE_VERSION_ERROR="module path $module_path has an invalid semantic import suffix for release $version" + return 1 + fi + if [[ "$path_major" != "$version_major" ]]; then + RELEASE_VERSION_ERROR="module path $module_path declares major v$path_major but release $version uses v$version_major" + return 1 + fi + return 0 + fi + + if [[ "$version_major" != "0" && "$version_major" != "1" ]]; then + RELEASE_VERSION_ERROR="module path $module_path requires /v$version_major for release $version" + return 1 + fi +} diff --git a/scripts/tag-all-modules.sh b/scripts/tag-all-modules.sh index a14c798..8332041 100755 --- a/scripts/tag-all-modules.sh +++ b/scripts/tag-all-modules.sh @@ -1,6 +1,15 @@ #!/usr/bin/env bash set -euo pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" +VERSION_VALIDATOR="$SCRIPT_DIR/release-version.sh" +if [[ ! -r "$VERSION_VALIDATOR" ]]; then + echo "error: release version validator is missing or unreadable: $VERSION_VALIDATOR" >&2 + exit 1 +fi +# shellcheck source=scripts/release-version.sh +source "$VERSION_VALIDATOR" + usage() { cat <<'USAGE' Usage: @@ -15,7 +24,9 @@ Examples: Behavior: - Tags root module as: vX.Y.Z - Tags each submodule as: /vX.Y.Z - - Uses the current HEAD commit for all tags + - Validates and tags one captured HEAD commit; dry runs preview the working tree + - Refuses to tag modules whose sibling requirements are not pinned to the release version + - --allow-dirty is accepted only with --dry-run because real tags always target HEAD - --exclude supports exact module dirs and prefixes (for example: driver excludes all driver/* modules) USAGE } @@ -32,6 +43,14 @@ dry_run=0 allow_dirty=0 skip_existing=0 excludes=() +release_view_dir="" + +cleanup() { + if [[ -n "$release_view_dir" && -d "$release_view_dir" ]]; then + rm -rf -- "$release_view_dir" + fi +} +trap cleanup EXIT normalize_module_dir() { local dir="$1" @@ -43,21 +62,53 @@ normalize_module_dir() { printf '%s\n' "$dir" } -module_is_excluded() { - local dir="$1" - local ex - for ex in "${excludes[@]-}"; do - if [[ "$dir" == "$ex" ]] || [[ "$dir" == "$ex/"* ]]; then +REMOTE_TAG_EXISTS=0 +REMOTE_TAG_COMMIT="" + +inspect_remote_tag() { + local remote_name="$1" + local tag="$2" + local output + local status + local direct_ref="refs/tags/$tag" + local peeled_ref="refs/tags/$tag^{}" + + REMOTE_TAG_EXISTS=0 + REMOTE_TAG_COMMIT="" + if output="$(git ls-remote --exit-code --tags "$remote_name" "$direct_ref" "$peeled_ref")"; then + REMOTE_TAG_EXISTS=1 + else + status=$? + if [[ "$status" -eq 2 ]]; then return 0 fi - done - return 1 + echo "error: failed to query remote for tag $tag (git ls-remote exit $status)" >&2 + return "$status" + fi + + REMOTE_TAG_COMMIT="$(awk -v ref="$peeled_ref" '$2 == ref { print $1; exit }' <<<"$output")" + if [[ -z "$REMOTE_TAG_COMMIT" ]]; then + REMOTE_TAG_COMMIT="$(awk -v ref="$direct_ref" '$2 == ref { print $1; exit }' <<<"$output")" + fi + if [[ -z "$REMOTE_TAG_COMMIT" ]]; then + echo "error: remote returned no resolvable object for tag $tag" >&2 + return 1 + fi } -remote_tag_exists() { - local remote_name="$1" - local tag="$2" - git ls-remote --exit-code --tags --refs "$remote_name" "refs/tags/$tag" >/dev/null 2>&1 +WORKTREE_STATUS="" + +capture_worktree_status() { + local phase="$1" + local status + + if WORKTREE_STATUS="$(git status --porcelain)"; then + return 0 + else + status=$? + fi + echo "error: failed to inspect working tree $phase (git status exit $status)" >&2 + return "$status" } while [[ $# -gt 0 ]]; do @@ -120,8 +171,13 @@ if [[ -z "$version" ]]; then exit 1 fi -if [[ ! "$version" =~ ^v[0-9]+\.[0-9]+\.[0-9]+(-[0-9A-Za-z.-]+)?(\+[0-9A-Za-z.-]+)?$ ]]; then - echo "error: version must look like vX.Y.Z (optionally with -prerelease and/or +build suffix)" >&2 +if ! validate_release_version "$version"; then + echo "error: $RELEASE_VERSION_ERROR" >&2 + exit 1 +fi + +if [[ "$allow_dirty" -eq 1 && "$dry_run" -eq 0 ]]; then + echo "error: --allow-dirty is only supported with --dry-run; real tags must be validated from HEAD" >&2 exit 1 fi @@ -133,55 +189,102 @@ if [[ ! -f go.mod ]]; then exit 1 fi -if [[ "$allow_dirty" -eq 0 ]] && [[ -n "$(git status --porcelain)" ]]; then - echo "error: working tree is dirty. commit/stash or pass --allow-dirty" >&2 +if [[ "$allow_dirty" -eq 0 ]]; then + if capture_worktree_status "before release planning"; then + if [[ -n "$WORKTREE_STATUS" ]]; then + echo "error: working tree is dirty. commit/stash before tagging or pass --allow-dirty with --dry-run" >&2 + exit 1 + fi + else + exit $? + fi +fi + +head_commit="$(git rev-parse --verify HEAD)" + +release_root="$root" +if [[ "$dry_run" -eq 0 ]]; then + release_view_dir="$(mktemp -d "${TMPDIR:-/tmp}/queue-release-head.XXXXXX")" + if ! git archive --format=tar "$head_commit" | tar -xf - -C "$release_view_dir"; then + echo "error: failed to extract captured HEAD $head_commit for release validation" >&2 + exit 1 + fi + release_root="$release_view_dir" +fi + +inventory_guard="$release_root/scripts/check-module-inventory.sh" +if [[ ! -x "$inventory_guard" ]]; then + echo "error: release preflight is missing or not executable: $inventory_guard" >&2 exit 1 fi -module_dirs=() -while IFS= read -r dir; do - dir="$(normalize_module_dir "$dir")" - module_dirs+=("$dir") -done <&2 +tag_planner="$release_root/scripts/plan-module-release-tags.sh" +if [[ ! -x "$tag_planner" ]]; then + echo "error: release tag planner is missing or not executable: $tag_planner" >&2 exit 1 fi -tags_to_create=() -tags_to_push=() -for dir in "${module_dirs[@]}"; do - if module_is_excluded "$dir"; then - continue - fi +preflight_args=(--release-version "$version") +tag_plan_args=("$version") +for excluded in "${excludes[@]}"; do + preflight_args+=(--exclude "$excluded") + tag_plan_args+=(--exclude "$excluded") +done +if ! preflight_output="$("$inventory_guard" "${preflight_args[@]}" 2>&1)"; then + printf '%s\n' "$preflight_output" >&2 + exit 1 +fi - if [[ "$dir" == "." ]]; then - tag="$version" - else - tag="$dir/$version" - fi +if ! tag_plan="$("$tag_planner" "${tag_plan_args[@]}")"; then + exit 1 +fi +planned_tags=() +if [[ -n "$tag_plan" ]]; then + while IFS= read -r tag; do + planned_tags+=("$tag") + done <<<"$tag_plan" +fi + +tags_to_create=() +tags_to_push=() +for tag in "${planned_tags[@]}"; do if ! git check-ref-format "refs/tags/$tag" >/dev/null 2>&1; then - echo "error: computed invalid tag ref: $tag (from module dir: $dir)" >&2 + echo "error: computed invalid tag ref: $tag" >&2 exit 1 fi local_exists=0 + local_tag_commit="" remote_exists=0 + remote_tag_commit="" if git rev-parse -q --verify "refs/tags/$tag" >/dev/null 2>&1; then local_exists=1 + if [[ "$skip_existing" -eq 1 ]]; then + if ! local_tag_commit="$(git rev-parse -q --verify "refs/tags/$tag^{commit}")"; then + echo "error: local tag $tag does not resolve to a commit" >&2 + exit 1 + fi + if [[ "$local_tag_commit" != "$head_commit" ]]; then + echo "error: local tag $tag resolves to $local_tag_commit; --skip-existing requires HEAD $head_commit" >&2 + exit 1 + fi + fi fi - if [[ "$push" -eq 1 ]] && remote_tag_exists "$remote" "$tag"; then - remote_exists=1 + if [[ "$push" -eq 1 ]]; then + if inspect_remote_tag "$remote" "$tag"; then + remote_exists="$REMOTE_TAG_EXISTS" + remote_tag_commit="$REMOTE_TAG_COMMIT" + else + remote_status=$? + exit "$remote_status" + fi + if [[ "$remote_exists" -eq 1 && "$skip_existing" -eq 1 && "$remote_tag_commit" != "$head_commit" ]]; then + echo "error: remote tag $tag resolves to $remote_tag_commit; --skip-existing requires HEAD $head_commit" >&2 + exit 1 + fi fi if [[ "$local_exists" -eq 1 ]] || [[ "$remote_exists" -eq 1 ]]; then @@ -215,7 +318,7 @@ if [[ ${#tags_to_create[@]} -eq 0 ]] && [[ ${#tags_to_push[@]} -eq 0 ]]; then fi echo "repo: $root" -echo "head: $(git rev-parse --short HEAD)" +echo "head: $(git rev-parse --short "$head_commit")" echo "version: $version" if [[ ${#excludes[@]} -gt 0 ]]; then echo "excluded modules: ${excludes[*]}" @@ -238,9 +341,23 @@ if [[ "$dry_run" -eq 1 ]]; then exit 0 fi +current_head="$(git rev-parse --verify HEAD)" +if [[ "$current_head" != "$head_commit" ]]; then + echo "error: HEAD changed during release planning; expected $head_commit, found $current_head" >&2 + exit 1 +fi +if capture_worktree_status "immediately before tag mutation"; then + if [[ -n "$WORKTREE_STATUS" ]]; then + echo "error: working tree changed during release planning; refusing to mutate tags" >&2 + exit 1 + fi +else + exit $? +fi + if [[ ${#tags_to_create[@]} -gt 0 ]]; then for t in "${tags_to_create[@]}"; do - git tag -a "$t" -m "release $t" + git tag -a "$t" -m "release $t" "$head_commit" done fi @@ -249,7 +366,11 @@ if [[ ${#tags_to_create[@]} -gt 0 ]]; then fi if [[ "$push" -eq 1 ]]; then - git push "$remote" "${tags_to_push[@]}" + tag_refspecs=() + for t in "${tags_to_push[@]}"; do + tag_refspecs+=("refs/tags/$t:refs/tags/$t") + done + git push --atomic "$remote" "${tag_refspecs[@]}" echo "pushed ${#tags_to_push[@]} tags to $remote" else echo "not pushed (use --push)" diff --git a/scripts/test-all-modules.sh b/scripts/test-all-modules.sh index 09b1f67..9ffc0e0 100755 --- a/scripts/test-all-modules.sh +++ b/scripts/test-all-modules.sh @@ -1,19 +1,22 @@ #!/usr/bin/env bash set -euo pipefail -# Runs root + driver module tests from a single entrypoint. +# Runs root and every nested module test suite from a single entrypoint. # # Default mode is compile-only (fast) to validate module wiring. # Set FULL=1 to run full test suites. +# Set VET=1 to run vet after each module's tests. # # Examples: # ./scripts/test-all-modules.sh # FULL=1 ./scripts/test-all-modules.sh +# FULL=1 VET=1 ./scripts/test-all-modules.sh # GOCACHE=/tmp/queue-gocache ./scripts/test-all-modules.sh ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" GOCACHE_DIR="${GOCACHE:-/tmp/queue-gocache}" FULL="${FULL:-0}" +VET="${VET:-0}" ROOT_TEST_ARGS=("./..." "-count=1") MODULE_TEST_ARGS=("./..." "-count=1") @@ -26,27 +29,45 @@ fi run_root() { echo "==> root module" (cd "$ROOT_DIR" && GOCACHE="$GOCACHE_DIR" go test "${ROOT_TEST_ARGS[@]}") + if [[ "$VET" == "1" ]]; then + (cd "$ROOT_DIR" && GOCACHE="$GOCACHE_DIR" go vet ./...) + fi } -run_driver_module() { +run_module() { local mod_dir="$1" + local vet_module="${2:-$VET}" echo "==> ${mod_dir} (GOWORK=off)" ( cd "$ROOT_DIR/$mod_dir" && \ GOWORK=off GOCACHE="$GOCACHE_DIR" go test "${MODULE_TEST_ARGS[@]}" ) + if [[ "$vet_module" == "1" ]]; then + (cd "$ROOT_DIR/$mod_dir" && GOWORK=off GOCACHE="$GOCACHE_DIR" go vet ./...) + fi } +run_tooling_module() { + local mod_dir="$1" + echo "==> ${mod_dir} tooling module (GOWORK=off)" + (cd "$ROOT_DIR/$mod_dir" && GOWORK=off GOCACHE="$GOCACHE_DIR" go mod verify) +} + +"$ROOT_DIR/scripts/check-module-inventory.sh" +"$ROOT_DIR/scripts/test-release-scripts.sh" +"$ROOT_DIR/scripts/check-generated-docs.sh" run_root -run_driver_module "driver/redisqueue" -run_driver_module "driver/natsqueue" -run_driver_module "driver/sqsqueue" -run_driver_module "driver/rabbitmqqueue" -run_driver_module "driver/sqlqueuecore" -run_driver_module "driver/mysqlqueue" -run_driver_module "driver/postgresqueue" -run_driver_module "driver/sqlitequeue" -run_driver_module "examples" -run_driver_module "integration" +run_module "driver/redisqueue" +run_module "driver/natsqueue" +run_module "driver/sqsqueue" +run_module "driver/rabbitmqqueue" +run_module "driver/sqlqueuecore" +run_module "driver/mysqlqueue" +run_module "driver/postgresqueue" +run_module "driver/sqlitequeue" +run_module "examples" +run_module "integration" +# The generated-doc check compiles the build-ignored tools; module verification covers their dependency graph. +run_tooling_module "docs" echo "==> all module tests completed" diff --git a/scripts/test-release-scripts.sh b/scripts/test-release-scripts.sh new file mode 100755 index 0000000..e5ff778 --- /dev/null +++ b/scripts/test-release-scripts.sh @@ -0,0 +1,870 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" +TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/queue-release-scripts.XXXXXX")" +FIXTURE_DIR="$TMP_DIR/repository" +RELEASE_VERSION="v0.3.0" +GIT_BIN="$(command -v git)" + +cleanup() { + rm -rf "$TMP_DIR" +} +trap cleanup EXIT + +fail() { + echo "release script contract: $*" >&2 + exit 1 +} + +run_tag() { + run_tag_in "$FIXTURE_DIR" "$@" +} + +run_tag_in() { + local repository="$1" + shift + ( + cd "$repository" + ./scripts/tag-all-modules.sh "$@" + ) +} + +clone_fixture() { + local destination="$1" + git clone -q "$FIXTURE_DIR" "$destination" + git -C "$destination" config user.name "Release Contract" + git -C "$destination" config user.email "release-contract@example.invalid" +} + +create_bare_remote() { + local destination="$1" + git clone -q --bare "$FIXTURE_DIR" "$destination" + git -C "$destination" config user.name "Release Contract" + git -C "$destination" config user.email "release-contract@example.invalid" +} + +# create_fixture mirrors the minimum repository surfaces guarded by the real +# release path so the contract stays offline and independent of working-tree state. +create_fixture() { + mkdir -p "$FIXTURE_DIR/scripts" "$FIXTURE_DIR/driver/mockqueue" "$FIXTURE_DIR/.github/workflows" + cp "$ROOT_DIR/scripts/check-module-inventory.sh" "$FIXTURE_DIR/scripts/check-module-inventory.sh" + cp "$ROOT_DIR/scripts/plan-module-release-tags.sh" "$FIXTURE_DIR/scripts/plan-module-release-tags.sh" + cp "$ROOT_DIR/scripts/release-version.sh" "$FIXTURE_DIR/scripts/release-version.sh" + cp "$ROOT_DIR/scripts/tag-all-modules.sh" "$FIXTURE_DIR/scripts/tag-all-modules.sh" + chmod +x "$FIXTURE_DIR/scripts/check-module-inventory.sh" "$FIXTURE_DIR/scripts/plan-module-release-tags.sh" "$FIXTURE_DIR/scripts/tag-all-modules.sh" + + cat >"$FIXTURE_DIR/go.mod" <<'EOF_ROOT_MOD' +module example.com/queue-release-fixture + +go 1.24.4 +EOF_ROOT_MOD + + cat >"$FIXTURE_DIR/driver/mockqueue/go.mod" <<'EOF_DRIVER_MOD' +module example.com/queue-release-fixture/driver/mockqueue + +go 1.25.0 + +require ( + example.com/dependency v1.2.3 + example.com/queue-release-fixture v0.0.0 +) + +replace example.com/queue-release-fixture => ../.. +EOF_DRIVER_MOD + + cat >"$FIXTURE_DIR/go.work" <<'EOF_WORK' +go 1.25.0 + +use ( + . + ./driver/mockqueue +) +EOF_WORK + + cat >"$FIXTURE_DIR/scripts/module-go-versions.tsv" <<'EOF_GO_VERSIONS' +. 1.24.4 +driver/mockqueue 1.25.0 +EOF_GO_VERSIONS + + cat >"$FIXTURE_DIR/scripts/dependency-minimums.tsv" <<'EOF_DEPENDENCIES' +example.com/dependency v1.2.3 +EOF_DEPENDENCIES + + cat >"$FIXTURE_DIR/.github/workflows/test.yml" <<'EOF_WORKFLOW' +jobs: + minimum_go: + strategy: + matrix: + minimum_go_version: ["1.24.4", "1.25.0"] + race_modules: + strategy: + matrix: + race_module: [".", "driver/mockqueue"] +EOF_WORKFLOW + + git -C "$FIXTURE_DIR" init -q + git -C "$FIXTURE_DIR" config user.name "Release Contract" + git -C "$FIXTURE_DIR" config user.email "release-contract@example.invalid" + git -C "$FIXTURE_DIR" add go.mod go.work driver/mockqueue/go.mod scripts .github/workflows/test.yml + git -C "$FIXTURE_DIR" commit -qm "test: initialize release fixture" +} + +create_fixture + +if ! ( + cd "$FIXTURE_DIR" + ./scripts/check-module-inventory.sh +) >"$TMP_DIR/mixed-go-policy.log" 2>&1; then + cat "$TMP_DIR/mixed-go-policy.log" >&2 + fail "the inventory guard rejected an explicit mixed module Go policy" +fi + +LOW_WORKSPACE_DIR="$TMP_DIR/low-workspace" +clone_fixture "$LOW_WORKSPACE_DIR" +( + cd "$LOW_WORKSPACE_DIR" + go work edit -go=1.24.4 +) +if ( + cd "$LOW_WORKSPACE_DIR" + ./scripts/check-module-inventory.sh +) >"$TMP_DIR/low-workspace.log" 2>&1; then + fail "the inventory guard accepted a workspace below its highest module Go version" +fi +if ! grep -Fq "highest module policy requires Go 1.25.0" "$TMP_DIR/low-workspace.log"; then + cat "$TMP_DIR/low-workspace.log" >&2 + fail "the low-workspace rejection omitted the expected diagnostic" +fi + +LOW_MODULE_DIR="$TMP_DIR/low-module" +clone_fixture "$LOW_MODULE_DIR" +GOWORK=off go mod edit -go=1.24.4 "$LOW_MODULE_DIR/driver/mockqueue/go.mod" +if ( + cd "$LOW_MODULE_DIR" + ./scripts/check-module-inventory.sh +) >"$TMP_DIR/low-module.log" 2>&1; then + fail "the inventory guard accepted a module below its exact Go policy" +fi +if ! grep -Fq "driver/mockqueue uses Go 1.24.4; policy requires Go 1.25.0" "$TMP_DIR/low-module.log"; then + cat "$TMP_DIR/low-module.log" >&2 + fail "the low-module rejection omitted the expected diagnostic" +fi + +LOW_DEPENDENCY_DIR="$TMP_DIR/low-dependency" +clone_fixture "$LOW_DEPENDENCY_DIR" +GOWORK=off go mod edit -require=example.com/dependency@v1.2.2 "$LOW_DEPENDENCY_DIR/driver/mockqueue/go.mod" +if ( + cd "$LOW_DEPENDENCY_DIR" + ./scripts/check-module-inventory.sh +) >"$TMP_DIR/low-dependency.log" 2>&1; then + fail "the inventory guard accepted a dependency below its policy floor" +fi +if ! grep -Fq "driver/mockqueue requires example.com/dependency at v1.2.2; policy requires at least v1.2.3" "$TMP_DIR/low-dependency.log"; then + cat "$TMP_DIR/low-dependency.log" >&2 + fail "the dependency-floor rejection omitted the expected diagnostic" +fi + +PRERELEASE_DEPENDENCY_DIR="$TMP_DIR/prerelease-dependency" +clone_fixture "$PRERELEASE_DEPENDENCY_DIR" +GOWORK=off go mod edit -require=example.com/dependency@v1.3.0-rc.1 "$PRERELEASE_DEPENDENCY_DIR/driver/mockqueue/go.mod" +if ! ( + cd "$PRERELEASE_DEPENDENCY_DIR" + ./scripts/check-module-inventory.sh +) >"$TMP_DIR/prerelease-dependency.log" 2>&1; then + cat "$TMP_DIR/prerelease-dependency.log" >&2 + fail "the dependency floor rejected a valid newer semantic prerelease" +fi + +INVALID_DEPENDENCY_POLICY_DIR="$TMP_DIR/invalid-dependency-policy" +clone_fixture "$INVALID_DEPENDENCY_POLICY_DIR" +cat >"$INVALID_DEPENDENCY_POLICY_DIR/scripts/dependency-minimums.tsv" <<'EOF_INVALID_DEPENDENCY_POLICY' +example.com/dependency v1.2 +EOF_INVALID_DEPENDENCY_POLICY +if ( + cd "$INVALID_DEPENDENCY_POLICY_DIR" + ./scripts/check-module-inventory.sh +) >"$TMP_DIR/invalid-dependency-policy.log" 2>&1; then + fail "the inventory guard accepted a malformed dependency policy floor" +fi +if ! grep -Fq "example.com/dependency has invalid policy minimum v1.2" "$TMP_DIR/invalid-dependency-policy.log"; then + cat "$TMP_DIR/invalid-dependency-policy.log" >&2 + fail "the malformed dependency policy rejection omitted the expected diagnostic" +fi + +LARGE_SEMVER_DIR="$TMP_DIR/large-semver" +clone_fixture "$LARGE_SEMVER_DIR" +cat >"$LARGE_SEMVER_DIR/scripts/dependency-minimums.tsv" <<'EOF_LARGE_SEMVER_POLICY' +example.com/dependency v1.9007199254740993.0 +EOF_LARGE_SEMVER_POLICY +GOWORK=off go mod edit \ + -require=example.com/dependency@v1.9007199254740992.0 \ + "$LARGE_SEMVER_DIR/driver/mockqueue/go.mod" +if ( + cd "$LARGE_SEMVER_DIR" + ./scripts/check-module-inventory.sh +) >"$TMP_DIR/large-semver.log" 2>&1; then + fail "the dependency floor lost precision for large semantic version components" +fi +if ! grep -Fq "policy requires at least v1.9007199254740993.0" "$TMP_DIR/large-semver.log"; then + cat "$TMP_DIR/large-semver.log" >&2 + fail "the large semantic version rejection omitted the expected diagnostic" +fi + +LEXICAL_SEMVER_DIR="$TMP_DIR/lexical-semver" +clone_fixture "$LEXICAL_SEMVER_DIR" +cat >"$LEXICAL_SEMVER_DIR/scripts/dependency-minimums.tsv" <<'EOF_LEXICAL_SEMVER_POLICY' +example.com/dependency v1.2.3-1e3 +EOF_LEXICAL_SEMVER_POLICY +GOWORK=off go mod edit \ + -require=example.com/dependency@v1.2.3-1000e0 \ + "$LEXICAL_SEMVER_DIR/driver/mockqueue/go.mod" +if ( + cd "$LEXICAL_SEMVER_DIR" + ./scripts/check-module-inventory.sh +) >"$TMP_DIR/lexical-semver.log" 2>&1; then + fail "the dependency floor compared nonnumeric prerelease identifiers numerically" +fi +if ! grep -Fq "policy requires at least v1.2.3-1e3" "$TMP_DIR/lexical-semver.log"; then + cat "$TMP_DIR/lexical-semver.log" >&2 + fail "the lexical semantic version rejection omitted the expected diagnostic" +fi + +REPLACED_DEPENDENCY_DIR="$TMP_DIR/replaced-dependency" +clone_fixture "$REPLACED_DEPENDENCY_DIR" +GOWORK=off go mod edit \ + -replace=example.com/dependency=example.com/dependency@v1.2.2 \ + "$REPLACED_DEPENDENCY_DIR/driver/mockqueue/go.mod" +if ( + cd "$REPLACED_DEPENDENCY_DIR" + ./scripts/check-module-inventory.sh +) >"$TMP_DIR/replaced-dependency.log" 2>&1; then + fail "the inventory guard accepted a replacement below a protected dependency floor" +fi +if ! grep -Fq "driver/mockqueue must not replace policy dependency example.com/dependency" "$TMP_DIR/replaced-dependency.log"; then + cat "$TMP_DIR/replaced-dependency.log" >&2 + fail "the protected dependency replacement rejection omitted the expected diagnostic" +fi + +WORKSPACE_REPLACED_DEPENDENCY_DIR="$TMP_DIR/workspace-replaced-dependency" +clone_fixture "$WORKSPACE_REPLACED_DEPENDENCY_DIR" +( + cd "$WORKSPACE_REPLACED_DEPENDENCY_DIR" + go work edit -replace=example.com/dependency=example.com/dependency@v1.2.2 +) +if ( + cd "$WORKSPACE_REPLACED_DEPENDENCY_DIR" + ./scripts/check-module-inventory.sh +) >"$TMP_DIR/workspace-replaced-dependency.log" 2>&1; then + fail "the inventory guard accepted a workspace replacement below a protected dependency floor" +fi +if ! grep -Fq "go.work must not replace policy dependency example.com/dependency" "$TMP_DIR/workspace-replaced-dependency.log"; then + cat "$TMP_DIR/workspace-replaced-dependency.log" >&2 + fail "the protected workspace replacement rejection omitted the expected diagnostic" +fi + +INCOMPLETE_MINIMUM_GO_DIR="$TMP_DIR/incomplete-minimum-go" +clone_fixture "$INCOMPLETE_MINIMUM_GO_DIR" +awk ' + { gsub(/minimum_go_version: \["1.24.4", "1.25.0"\]/, "minimum_go_version: [\"1.24.4\"]"); print } +' "$INCOMPLETE_MINIMUM_GO_DIR/.github/workflows/test.yml" >"$INCOMPLETE_MINIMUM_GO_DIR/.github/workflows/test.yml.tmp" +mv "$INCOMPLETE_MINIMUM_GO_DIR/.github/workflows/test.yml.tmp" "$INCOMPLETE_MINIMUM_GO_DIR/.github/workflows/test.yml" +if ( + cd "$INCOMPLETE_MINIMUM_GO_DIR" + ./scripts/check-module-inventory.sh +) >"$TMP_DIR/incomplete-minimum-go.log" 2>&1; then + fail "the inventory guard accepted an incomplete minimum Go CI matrix" +fi +if ! grep -Fq "CI minimum Go matrix must match the module Go version policy" "$TMP_DIR/incomplete-minimum-go.log"; then + cat "$TMP_DIR/incomplete-minimum-go.log" >&2 + fail "the incomplete minimum Go matrix rejection omitted the expected diagnostic" +fi + +invalid_versions=( + v01.2.3 + v0.3.0-01 + v0.3.0-alpha..1 + v0.3.0-alpha_1 +) +version_case=0 +for invalid_version in "${invalid_versions[@]}"; do + version_case=$((version_case + 1)) + for entrypoint in tag planner inventory; do + version_output="$TMP_DIR/version-$version_case-$entrypoint.log" + case "$entrypoint" in + tag) + if run_tag "$invalid_version" --dry-run --allow-dirty >"$version_output" 2>&1; then + fail "tag entrypoint accepted invalid version $invalid_version" + fi + ;; + planner) + if ( + cd "$FIXTURE_DIR" + ./scripts/plan-module-release-tags.sh "$invalid_version" + ) >"$version_output" 2>&1; then + fail "planner entrypoint accepted invalid version $invalid_version" + fi + ;; + inventory) + if ( + cd "$FIXTURE_DIR" + ./scripts/check-module-inventory.sh --release-version "$invalid_version" + ) >"$version_output" 2>&1; then + fail "inventory entrypoint accepted invalid version $invalid_version" + fi + ;; + esac + if ! grep -Fq "invalid Go release version $invalid_version" "$version_output"; then + cat "$version_output" >&2 + fail "$entrypoint entrypoint rejected $invalid_version without the shared diagnostic" + fi + done +done + +for entrypoint in tag planner inventory; do + major_output="$TMP_DIR/major-$entrypoint.log" + case "$entrypoint" in + tag) + if run_tag v2.0.0 --dry-run --allow-dirty >"$major_output" 2>&1; then + fail "tag entrypoint accepted v2 for unsuffixed module paths" + fi + ;; + planner) + if ( + cd "$FIXTURE_DIR" + ./scripts/plan-module-release-tags.sh v2.0.0 + ) >"$major_output" 2>&1; then + fail "planner entrypoint accepted v2 for unsuffixed module paths" + fi + ;; + inventory) + if ( + cd "$FIXTURE_DIR" + ./scripts/check-module-inventory.sh --release-version v2.0.0 + ) >"$major_output" 2>&1; then + fail "inventory entrypoint accepted v2 for unsuffixed module paths" + fi + ;; + esac + if ! grep -Fq "requires /v2 for release v2.0.0" "$major_output"; then + cat "$major_output" >&2 + fail "$entrypoint entrypoint rejected v2 without the module-path diagnostic" + fi +done + +V2_DIR="$TMP_DIR/v2-module" +mkdir -p "$V2_DIR/.github/workflows" +cp -R "$FIXTURE_DIR/scripts" "$V2_DIR/scripts" +cat >"$V2_DIR/go.mod" <<'EOF_V2_MOD' +module example.com/queue-release-fixture/v2 + +go 1.24.4 + +require example.com/dependency v1.2.3 +EOF_V2_MOD +cat >"$V2_DIR/go.work" <<'EOF_V2_WORK' +go 1.24.4 + +use . +EOF_V2_WORK +cat >"$V2_DIR/scripts/module-go-versions.tsv" <<'EOF_V2_GO_VERSIONS' +. 1.24.4 +EOF_V2_GO_VERSIONS +cat >"$V2_DIR/scripts/dependency-minimums.tsv" <<'EOF_V2_DEPENDENCIES' +example.com/dependency v1.2.3 +EOF_V2_DEPENDENCIES +cat >"$V2_DIR/.github/workflows/test.yml" <<'EOF_V2_WORKFLOW' +jobs: + minimum_go: + strategy: + matrix: + minimum_go_version: ["1.24.4"] + race_modules: + strategy: + matrix: + race_module: ["."] +EOF_V2_WORKFLOW +git -C "$V2_DIR" init -q +git -C "$V2_DIR" config user.name "Release Contract" +git -C "$V2_DIR" config user.email "release-contract@example.invalid" +git -C "$V2_DIR" add go.mod go.work scripts .github/workflows/test.yml +git -C "$V2_DIR" commit -qm "test: initialize v2 release fixture" +if ! run_tag_in "$V2_DIR" v2.1.0 >"$TMP_DIR/v2-tag.log" 2>&1; then + cat "$TMP_DIR/v2-tag.log" >&2 + fail "tag entrypoint rejected a matching /v2 module path" +fi +if ! ( + cd "$V2_DIR" + ./scripts/plan-module-release-tags.sh v2.1.0 +) >"$TMP_DIR/v2-planner.log" 2>&1; then + cat "$TMP_DIR/v2-planner.log" >&2 + fail "planner entrypoint rejected a matching /v2 module path" +fi +if ! ( + cd "$V2_DIR" + ./scripts/check-module-inventory.sh --release-version v2.1.0 +) >"$TMP_DIR/v2-inventory.log" 2>&1; then + cat "$TMP_DIR/v2-inventory.log" >&2 + fail "inventory entrypoint rejected a matching /v2 module path" +fi +if [[ "$(git -C "$V2_DIR" tag --list)" != "v2.1.0" ]]; then + fail "matching /v2 module path produced the wrong tag family" +fi + +HIDDEN_PIN_DIR="$TMP_DIR/hidden-pin" +clone_fixture "$HIDDEN_PIN_DIR" +GOWORK=off go mod edit \ + -require="example.com/queue-release-fixture@$RELEASE_VERSION" \ + "$HIDDEN_PIN_DIR/driver/mockqueue/go.mod" +git -C "$HIDDEN_PIN_DIR" update-index --assume-unchanged driver/mockqueue/go.mod +if [[ -n "$(git -C "$HIDDEN_PIN_DIR" status --porcelain)" ]]; then + fail "hidden-pin fixture is not clean according to Git" +fi + +hidden_preview_output="$TMP_DIR/hidden-pin-preview.log" +if ! run_tag_in "$HIDDEN_PIN_DIR" "$RELEASE_VERSION" --dry-run --allow-dirty >"$hidden_preview_output" 2>&1; then + cat "$hidden_preview_output" >&2 + fail "dry-run did not preview hidden working-tree module pins" +fi +if ! grep -Fq "create tags (2):" "$hidden_preview_output"; then + cat "$hidden_preview_output" >&2 + fail "hidden-pin dry-run produced the wrong working-tree tag plan" +fi + +hidden_release_output="$TMP_DIR/hidden-pin-release.log" +if run_tag_in "$HIDDEN_PIN_DIR" "$RELEASE_VERSION" >"$hidden_release_output" 2>&1; then + fail "hidden working-tree pins validated a release that captured HEAD does not contain" +fi +if ! grep -Fq "requires sibling example.com/queue-release-fixture at v0.0.0; release $RELEASE_VERSION requires a resolvable $RELEASE_VERSION pin" "$hidden_release_output"; then + cat "$hidden_release_output" >&2 + fail "captured-HEAD pin rejection omitted the expected diagnostic" +fi +if [[ -n "$(git -C "$HIDDEN_PIN_DIR" tag --list)" ]]; then + fail "captured-HEAD pin rejection created local tags" +fi + +invalid_output="$TMP_DIR/invalid-pins.log" +if run_tag "$RELEASE_VERSION" >"$invalid_output" 2>&1; then + fail "tagging unexpectedly accepted a v0.0.0 sibling pin" +fi +if [[ -n "$(git -C "$FIXTURE_DIR" tag --list)" ]]; then + fail "invalid release pins created tags before the preflight failed" +fi +if ! grep -Fq "requires sibling example.com/queue-release-fixture at v0.0.0; release $RELEASE_VERSION requires a resolvable $RELEASE_VERSION pin" "$invalid_output"; then + cat "$invalid_output" >&2 + fail "invalid release pins failed without the expected diagnostic" +fi + +# Excluding the unpublished driver should retain the documented root-only +# dry-run path even while that driver's own pin remains intentionally invalid. +exclude_output="$TMP_DIR/excluded-driver.log" +if ! run_tag "$RELEASE_VERSION" --dry-run --exclude driver >"$exclude_output" 2>&1; then + cat "$exclude_output" >&2 + fail "an excluded invalid module blocked a legitimate dry-run" +fi +if ! grep -Fq "create tags (1):" "$exclude_output" || \ + ! grep -Fq " - $RELEASE_VERSION" "$exclude_output" || \ + grep -Fq "driver/mockqueue/$RELEASE_VERSION" "$exclude_output"; then + cat "$exclude_output" >&2 + fail "exclude dry-run produced the wrong tag plan" +fi +if [[ -n "$(git -C "$FIXTURE_DIR" tag --list)" ]]; then + fail "exclude dry-run created a tag" +fi + +GOWORK=off go mod edit \ + -require="example.com/queue-release-fixture@$RELEASE_VERSION" \ + "$FIXTURE_DIR/driver/mockqueue/go.mod" +git -C "$FIXTURE_DIR" add driver/mockqueue/go.mod +git -C "$FIXTURE_DIR" commit -qm "test: pin the release fixture" + +VALID_VERSION="v0.3.0-rc.1+build.01" +VALID_VERSION_DIR="$TMP_DIR/valid-version" +clone_fixture "$VALID_VERSION_DIR" +valid_version_output="$TMP_DIR/valid-version-tag.log" +if ! run_tag_in "$VALID_VERSION_DIR" "$VALID_VERSION" --exclude driver >"$valid_version_output" 2>&1; then + cat "$valid_version_output" >&2 + fail "tag entrypoint rejected valid prerelease/build metadata" +fi +if ! ( + cd "$VALID_VERSION_DIR" + ./scripts/plan-module-release-tags.sh "$VALID_VERSION" --exclude driver +) >"$TMP_DIR/valid-version-planner.log" 2>&1; then + cat "$TMP_DIR/valid-version-planner.log" >&2 + fail "planner entrypoint rejected valid prerelease/build metadata" +fi +if ! ( + cd "$VALID_VERSION_DIR" + ./scripts/check-module-inventory.sh --release-version "$VALID_VERSION" --exclude driver +) >"$TMP_DIR/valid-version-inventory.log" 2>&1; then + cat "$TMP_DIR/valid-version-inventory.log" >&2 + fail "inventory entrypoint rejected valid prerelease/build metadata" +fi + +excluded_target_output="$TMP_DIR/excluded-target.log" +if run_tag "$RELEASE_VERSION" --dry-run --exclude . >"$excluded_target_output" 2>&1; then + fail "tagging accepted an included module whose required sibling tag was excluded" +fi +if ! grep -Fq "driver/mockqueue is included but requires excluded sibling example.com/queue-release-fixture; release $RELEASE_VERSION would omit required tag $RELEASE_VERSION" "$excluded_target_output"; then + cat "$excluded_target_output" >&2 + fail "an excluded required sibling failed without the expected diagnostic" +fi +if [[ -n "$(git -C "$FIXTURE_DIR" tag --list)" ]]; then + fail "an incomplete dependency tag plan created tags before the preflight failed" +fi + +printf 'uncommitted fixture state\n' >"$FIXTURE_DIR/dirty-marker.txt" +dirty_dry_run_output="$TMP_DIR/dirty-dry-run.log" +if ! run_tag "$RELEASE_VERSION" --dry-run --allow-dirty >"$dirty_dry_run_output" 2>&1; then + cat "$dirty_dry_run_output" >&2 + fail "--allow-dirty no longer supports its safe dry-run path" +fi +if ! grep -Fq "dry-run: no tags created" "$dirty_dry_run_output"; then + cat "$dirty_dry_run_output" >&2 + fail "dirty dry-run omitted its no-mutation result" +fi + +dirty_tag_output="$TMP_DIR/dirty-tag.log" +if run_tag "$RELEASE_VERSION" --allow-dirty >"$dirty_tag_output" 2>&1; then + fail "--allow-dirty permitted real tags against uncommitted module state" +fi +if ! grep -Fq -- "--allow-dirty is only supported with --dry-run" "$dirty_tag_output"; then + cat "$dirty_tag_output" >&2 + fail "dirty real-tag rejection omitted the expected diagnostic" +fi +if [[ -n "$(git -C "$FIXTURE_DIR" tag --list)" ]]; then + fail "dirty real-tag rejection created tags" +fi +git -C "$FIXTURE_DIR" add dirty-marker.txt +git -C "$FIXTURE_DIR" commit -qm "test: restore a clean release fixture" + +FAKE_GIT_DIR="$TMP_DIR/fake-git" +mkdir -p "$FAKE_GIT_DIR" +cat >"$FAKE_GIT_DIR/git" <<'EOF_FAKE_GIT' +#!/usr/bin/env bash +set -euo pipefail + +real_git="${RELEASE_TEST_REAL_GIT:?}" +state_file="${RELEASE_TEST_STATE_FILE:?}" +mode="${RELEASE_TEST_MUTATION_MODE:?}" +matched=0 +if [[ "$mode" == "head" && "$#" -eq 3 && "$1" == "rev-parse" && "$2" == "--verify" && "$3" == "HEAD" ]]; then + matched=1 +elif [[ ( "$mode" == "dirty" || "$mode" == status-error-* ) && "$#" -eq 2 && "$1" == "status" && "$2" == "--porcelain" ]]; then + matched=1 +fi + +if [[ "$matched" -eq 1 ]]; then + count=0 + if [[ -f "$state_file" ]]; then + count="$(<"$state_file")" + fi + count=$((count + 1)) + printf '%s\n' "$count" >"$state_file" + if [[ "$mode" == "status-error-initial" && "$count" -eq 1 ]]; then + exit 73 + fi + if [[ "$mode" == "status-error-final" && "$count" -eq 2 ]]; then + exit 74 + fi + if [[ "$count" -eq 2 ]]; then + if [[ "$mode" == "head" ]]; then + "$real_git" update-ref HEAD "${RELEASE_TEST_TARGET_HEAD:?}" "${RELEASE_TEST_ORIGINAL_HEAD:?}" + else + printf 'changed during release planning\n' >"${RELEASE_TEST_REPOSITORY:?}/final-state-dirty.txt" + fi + fi +fi + +exec "$real_git" "$@" +EOF_FAKE_GIT +chmod +x "$FAKE_GIT_DIR/git" + +HEAD_CHANGE_DIR="$TMP_DIR/head-change" +clone_fixture "$HEAD_CHANGE_DIR" +original_head="$(git -C "$HEAD_CHANGE_DIR" rev-parse HEAD)" +original_tree="$(git -C "$HEAD_CHANGE_DIR" rev-parse "HEAD^{tree}")" +changed_head="$(printf 'test: concurrent head change\n' | git -C "$HEAD_CHANGE_DIR" commit-tree "$original_tree" -p "$original_head")" +head_change_output="$TMP_DIR/head-change.log" +if ( + cd "$HEAD_CHANGE_DIR" + PATH="$FAKE_GIT_DIR:$PATH" \ + RELEASE_TEST_REAL_GIT="$GIT_BIN" \ + RELEASE_TEST_STATE_FILE="$TMP_DIR/head-change.state" \ + RELEASE_TEST_MUTATION_MODE="head" \ + RELEASE_TEST_ORIGINAL_HEAD="$original_head" \ + RELEASE_TEST_TARGET_HEAD="$changed_head" \ + RELEASE_TEST_REPOSITORY="$HEAD_CHANGE_DIR" \ + ./scripts/tag-all-modules.sh "$RELEASE_VERSION" +) >"$head_change_output" 2>&1; then + fail "a concurrent HEAD change passed the final release-state check" +fi +if ! grep -Fq "HEAD changed during release planning; expected $original_head, found $changed_head" "$head_change_output"; then + cat "$head_change_output" >&2 + fail "concurrent HEAD rejection omitted the expected diagnostic" +fi +if [[ -n "$(git -C "$HEAD_CHANGE_DIR" tag --list)" ]]; then + fail "concurrent HEAD rejection created local tags" +fi + +FINAL_DIRTY_DIR="$TMP_DIR/final-dirty" +clone_fixture "$FINAL_DIRTY_DIR" +final_dirty_output="$TMP_DIR/final-dirty.log" +if ( + cd "$FINAL_DIRTY_DIR" + PATH="$FAKE_GIT_DIR:$PATH" \ + RELEASE_TEST_REAL_GIT="$GIT_BIN" \ + RELEASE_TEST_STATE_FILE="$TMP_DIR/final-dirty.state" \ + RELEASE_TEST_MUTATION_MODE="dirty" \ + RELEASE_TEST_ORIGINAL_HEAD="unused" \ + RELEASE_TEST_TARGET_HEAD="unused" \ + RELEASE_TEST_REPOSITORY="$FINAL_DIRTY_DIR" \ + ./scripts/tag-all-modules.sh "$RELEASE_VERSION" +) >"$final_dirty_output" 2>&1; then + fail "a concurrent working-tree change passed the final release-state check" +fi +if ! grep -Fq "working tree changed during release planning; refusing to mutate tags" "$final_dirty_output"; then + cat "$final_dirty_output" >&2 + fail "concurrent working-tree rejection omitted the expected diagnostic" +fi +if [[ -n "$(git -C "$FINAL_DIRTY_DIR" tag --list)" ]]; then + fail "concurrent working-tree rejection created local tags" +fi + +STATUS_INITIAL_DIR="$TMP_DIR/status-initial" +clone_fixture "$STATUS_INITIAL_DIR" +status_initial_output="$TMP_DIR/status-initial.log" +if ( + cd "$STATUS_INITIAL_DIR" + PATH="$FAKE_GIT_DIR:$PATH" \ + RELEASE_TEST_REAL_GIT="$GIT_BIN" \ + RELEASE_TEST_STATE_FILE="$TMP_DIR/status-initial.state" \ + RELEASE_TEST_MUTATION_MODE="status-error-initial" \ + RELEASE_TEST_ORIGINAL_HEAD="unused" \ + RELEASE_TEST_TARGET_HEAD="unused" \ + RELEASE_TEST_REPOSITORY="$STATUS_INITIAL_DIR" \ + ./scripts/tag-all-modules.sh "$RELEASE_VERSION" +) >"$status_initial_output" 2>&1; then + fail "an initial git status failure was treated as a clean tree" +fi +if ! grep -Fq "failed to inspect working tree before release planning (git status exit 73)" "$status_initial_output"; then + cat "$status_initial_output" >&2 + fail "initial git status failure omitted the expected diagnostic" +fi +if [[ -n "$(git -C "$STATUS_INITIAL_DIR" tag --list)" ]]; then + fail "initial git status failure created local tags" +fi + +STATUS_FINAL_DIR="$TMP_DIR/status-final" +clone_fixture "$STATUS_FINAL_DIR" +status_final_output="$TMP_DIR/status-final.log" +if ( + cd "$STATUS_FINAL_DIR" + PATH="$FAKE_GIT_DIR:$PATH" \ + RELEASE_TEST_REAL_GIT="$GIT_BIN" \ + RELEASE_TEST_STATE_FILE="$TMP_DIR/status-final.state" \ + RELEASE_TEST_MUTATION_MODE="status-error-final" \ + RELEASE_TEST_ORIGINAL_HEAD="unused" \ + RELEASE_TEST_TARGET_HEAD="unused" \ + RELEASE_TEST_REPOSITORY="$STATUS_FINAL_DIR" \ + ./scripts/tag-all-modules.sh "$RELEASE_VERSION" +) >"$status_final_output" 2>&1; then + fail "a final git status failure was treated as a clean tree" +fi +if ! grep -Fq "failed to inspect working tree immediately before tag mutation (git status exit 74)" "$status_final_output"; then + cat "$status_final_output" >&2 + fail "final git status failure omitted the expected diagnostic" +fi +if [[ -n "$(git -C "$STATUS_FINAL_DIR" tag --list)" ]]; then + fail "final git status failure created local tags" +fi + +REMOTE_DIR="$TMP_DIR/remote.git" +BROKEN_REMOTE_DIR="$TMP_DIR/missing-remote.git" +git init --bare -q "$REMOTE_DIR" +git -C "$FIXTURE_DIR" remote add origin "$REMOTE_DIR" +git -C "$FIXTURE_DIR" remote add broken "$BROKEN_REMOTE_DIR" + +empty_remote_output="$TMP_DIR/empty-remote.log" +if ! run_tag "$RELEASE_VERSION" --dry-run --push >"$empty_remote_output" 2>&1; then + cat "$empty_remote_output" >&2 + fail "an empty reachable remote was not treated as having no release tags" +fi +if ! grep -Fq "push tags (2):" "$empty_remote_output"; then + cat "$empty_remote_output" >&2 + fail "empty-remote dry-run produced the wrong push plan" +fi + +broken_remote_output="$TMP_DIR/broken-remote.log" +if run_tag "$RELEASE_VERSION" --dry-run --push --remote broken >"$broken_remote_output" 2>&1; then + fail "an unreachable remote was treated as an absent tag during dry-run" +fi +if ! grep -Fq "failed to query remote for tag $RELEASE_VERSION" "$broken_remote_output"; then + cat "$broken_remote_output" >&2 + fail "unreachable-remote dry-run omitted the expected diagnostic" +fi +if [[ -n "$(git -C "$FIXTURE_DIR" tag --list)" ]]; then + fail "remote-query dry-run created local tags" +fi + +broken_push_output="$TMP_DIR/broken-push.log" +if run_tag "$RELEASE_VERSION" --push --remote broken >"$broken_push_output" 2>&1; then + fail "an unreachable remote allowed a real tag operation to continue" +fi +if ! grep -Fq "failed to query remote for tag $RELEASE_VERSION" "$broken_push_output"; then + cat "$broken_push_output" >&2 + fail "unreachable real push omitted the expected diagnostic" +fi +if [[ -n "$(git -C "$FIXTURE_DIR" tag --list)" ]]; then + fail "remote query failure created local tags" +fi + +STALE_LOCAL_DIR="$TMP_DIR/stale-local" +clone_fixture "$STALE_LOCAL_DIR" +stale_local_commit="$(git -C "$STALE_LOCAL_DIR" rev-parse HEAD)" +git -C "$STALE_LOCAL_DIR" tag -a "$RELEASE_VERSION" -m "stale root" "$stale_local_commit" +git -C "$STALE_LOCAL_DIR" tag -a "driver/mockqueue/$RELEASE_VERSION" -m "stale driver" "$stale_local_commit" +printf 'advance past local tags\n' >"$STALE_LOCAL_DIR/stale-local-marker.txt" +git -C "$STALE_LOCAL_DIR" add stale-local-marker.txt +git -C "$STALE_LOCAL_DIR" commit -qm "test: advance past local tags" +stale_local_head="$(git -C "$STALE_LOCAL_DIR" rev-parse HEAD)" +stale_local_output="$TMP_DIR/stale-local.log" +if run_tag_in "$STALE_LOCAL_DIR" "$RELEASE_VERSION" --skip-existing >"$stale_local_output" 2>&1; then + fail "--skip-existing accepted a stale local release tag" +fi +if ! grep -Fq "local tag $RELEASE_VERSION resolves to $stale_local_commit; --skip-existing requires HEAD $stale_local_head" "$stale_local_output"; then + cat "$stale_local_output" >&2 + fail "stale local tag rejection omitted the expected peeled commit diagnostic" +fi +if [[ "$(git -C "$STALE_LOCAL_DIR" tag --list | wc -l | tr -d ' ')" != "2" ]]; then + fail "stale local tag rejection mutated the local tag family" +fi + +STALE_REMOTE_DIR="$TMP_DIR/stale-remote" +STALE_REMOTE_BARE_DIR="$TMP_DIR/stale-remote.git" +clone_fixture "$STALE_REMOTE_DIR" +create_bare_remote "$STALE_REMOTE_BARE_DIR" +stale_remote_commit="$(git -C "$STALE_REMOTE_DIR" rev-parse HEAD)" +git -C "$STALE_REMOTE_BARE_DIR" tag -a "$RELEASE_VERSION" -m "stale remote root" "$stale_remote_commit" +git -C "$STALE_REMOTE_BARE_DIR" tag -a "driver/mockqueue/$RELEASE_VERSION" -m "stale remote driver" "$stale_remote_commit" +git -C "$STALE_REMOTE_DIR" remote set-url origin "$STALE_REMOTE_BARE_DIR" +printf 'advance past remote tags\n' >"$STALE_REMOTE_DIR/stale-remote-marker.txt" +git -C "$STALE_REMOTE_DIR" add stale-remote-marker.txt +git -C "$STALE_REMOTE_DIR" commit -qm "test: advance past remote tags" +stale_remote_head="$(git -C "$STALE_REMOTE_DIR" rev-parse HEAD)" +stale_remote_output="$TMP_DIR/stale-remote.log" +if run_tag_in "$STALE_REMOTE_DIR" "$RELEASE_VERSION" --skip-existing --dry-run --push >"$stale_remote_output" 2>&1; then + fail "--skip-existing accepted a stale remote release tag" +fi +if ! grep -Fq "remote tag $RELEASE_VERSION resolves to $stale_remote_commit; --skip-existing requires HEAD $stale_remote_head" "$stale_remote_output"; then + cat "$stale_remote_output" >&2 + fail "stale remote tag rejection omitted the expected peeled commit diagnostic" +fi +if [[ -n "$(git -C "$STALE_REMOTE_DIR" tag --list)" ]]; then + fail "stale remote tag rejection created local tags" +fi + +REUSE_DIR="$TMP_DIR/reuse" +REUSE_REMOTE_DIR="$TMP_DIR/reuse.git" +clone_fixture "$REUSE_DIR" +create_bare_remote "$REUSE_REMOTE_DIR" +reuse_head="$(git -C "$REUSE_DIR" rev-parse HEAD)" +git -C "$REUSE_DIR" tag -a "$RELEASE_VERSION" -m "reusable root" "$reuse_head" +git -C "$REUSE_REMOTE_DIR" tag -a "driver/mockqueue/$RELEASE_VERSION" -m "reusable driver" "$reuse_head" +git -C "$REUSE_DIR" remote set-url origin "$REUSE_REMOTE_DIR" +reuse_output="$TMP_DIR/reuse.log" +if ! run_tag_in "$REUSE_DIR" "$RELEASE_VERSION" --skip-existing --dry-run --push >"$reuse_output" 2>&1; then + cat "$reuse_output" >&2 + fail "--skip-existing rejected same-HEAD local or remote tags" +fi +if ! grep -Fq "reuse local tag for push: $RELEASE_VERSION" "$reuse_output" || \ + ! grep -Fq "skip existing: driver/mockqueue/$RELEASE_VERSION" "$reuse_output"; then + cat "$reuse_output" >&2 + fail "same-HEAD reuse produced the wrong push plan" +fi + +ATOMIC_DIR="$TMP_DIR/atomic" +ATOMIC_REMOTE_DIR="$TMP_DIR/atomic.git" +clone_fixture "$ATOMIC_DIR" +create_bare_remote "$ATOMIC_REMOTE_DIR" +git -C "$ATOMIC_DIR" remote set-url origin "$ATOMIC_REMOTE_DIR" +atomic_trace="$TMP_DIR/atomic-push.trace" +atomic_output="$TMP_DIR/atomic-push.log" +if ! GIT_TRACE="$atomic_trace" run_tag_in "$ATOMIC_DIR" "$RELEASE_VERSION" --push >"$atomic_output" 2>&1; then + cat "$atomic_output" >&2 + fail "the synchronized atomic family push failed" +fi +if ! grep -Fq "git push --atomic origin refs/tags/$RELEASE_VERSION:refs/tags/$RELEASE_VERSION refs/tags/driver/mockqueue/$RELEASE_VERSION:refs/tags/driver/mockqueue/$RELEASE_VERSION" "$atomic_trace"; then + cat "$atomic_trace" >&2 + fail "the family push was not atomic with fully qualified tag refspecs" +fi +expected_tags=$'driver/mockqueue/v0.3.0\nv0.3.0' +atomic_local_tags="$(git -C "$ATOMIC_DIR" tag --list | LC_ALL=C sort)" +atomic_remote_tags="$(git -C "$ATOMIC_REMOTE_DIR" tag --list | LC_ALL=C sort)" +if [[ "$atomic_local_tags" != "$expected_tags" || "$atomic_remote_tags" != "$expected_tags" ]]; then + printf 'expected tags:\n%s\nlocal tags:\n%s\nremote tags:\n%s\n' "$expected_tags" "$atomic_local_tags" "$atomic_remote_tags" >&2 + fail "the atomic push did not publish the complete tag family" +fi + +ATOMIC_REJECT_DIR="$TMP_DIR/atomic-reject" +ATOMIC_REJECT_REMOTE_DIR="$TMP_DIR/atomic-reject.git" +clone_fixture "$ATOMIC_REJECT_DIR" +create_bare_remote "$ATOMIC_REJECT_REMOTE_DIR" +git -C "$ATOMIC_REJECT_DIR" remote set-url origin "$ATOMIC_REJECT_REMOTE_DIR" +cat >"$ATOMIC_REJECT_REMOTE_DIR/hooks/update" <"$atomic_reject_output" 2>&1; then + fail "a rejected member unexpectedly allowed the atomic family push" +fi +if [[ -n "$(git -C "$ATOMIC_REJECT_REMOTE_DIR" tag --list)" ]]; then + cat "$atomic_reject_output" >&2 + fail "a rejected atomic family push partially updated the remote" +fi + +valid_output="$TMP_DIR/valid-pins.log" +if ! run_tag "$RELEASE_VERSION" >"$valid_output" 2>&1; then + cat "$valid_output" >&2 + fail "valid synchronized pins did not produce a tag family" +fi +actual_tags="$(git -C "$FIXTURE_DIR" tag --list | LC_ALL=C sort)" +if [[ "$actual_tags" != "$expected_tags" ]]; then + printf 'expected tags:\n%s\nactual tags:\n%s\n' "$expected_tags" "$actual_tags" >&2 + fail "valid synchronized pins produced the wrong tag family" +fi + +if ! ( + cd "$FIXTURE_DIR" + ./scripts/check-module-inventory.sh --tag-version "$RELEASE_VERSION" +) >"$TMP_DIR/tag-family.log" 2>&1; then + cat "$TMP_DIR/tag-family.log" >&2 + fail "the synchronized fixture tags failed the ordinary tag-family guard" +fi + +if ! run_tag "$RELEASE_VERSION" --skip-existing >"$TMP_DIR/skip-existing.log" 2>&1; then + cat "$TMP_DIR/skip-existing.log" >&2 + fail "the release preflight broke the documented skip-existing path" +fi + +inventory_stdout="$TMP_DIR/inventory-stdout.log" +inventory_stderr="$TMP_DIR/inventory-stderr.log" +if ! ( + cd "$FIXTURE_DIR" + ./scripts/check-module-inventory.sh +) >"$inventory_stdout" 2>"$inventory_stderr"; then + cat "$inventory_stdout" >&2 + cat "$inventory_stderr" >&2 + fail "the ordinary inventory guard failed after the release contracts" +fi +if [[ -s "$inventory_stderr" ]]; then + cat "$inventory_stderr" >&2 + fail "the ordinary inventory guard emitted non-portable parser diagnostics" +fi + +echo "release script contract: strict versions, module compatibility policies, path majors, captured-HEAD planning, fail-closed status, final-state checks, dependency closure, dirty-tree safety, remote queries, safe reuse, atomic push, excludes, dry-run, and synchronized tags OK" diff --git a/test-plan.md b/test-plan.md index 69c8374..c38a4ad 100644 --- a/test-plan.md +++ b/test-plan.md @@ -83,15 +83,21 @@ Related documentation and CI checks: ## 4. Coverage aggregation (supporting signal) -Script: +Commands: ```bash -scripts/coverage-codecov.sh +scripts/coverage-codecov.sh unit +INTEGRATION_BACKEND=redis scripts/coverage-codecov.sh integration ``` Current role: -- merges unit + integration-tagged coverage +- unit mode runs the root module and every buildable nested module independently with `GOWORK=off`; it also executes the root module's lightweight integration-tagged `bus` fixture tests, while the tooling-only `docs` module is inventory-checked but has no package to cover +- integration mode runs the tagged tests from the actual `integration` module and instruments repository root, integration, and driver packages +- CI runs one integration coverage command in each existing backend matrix leg, then fans the unit profile and all ten backend profiles into one guarded Codecov upload +- emitted atomic text profiles use module-qualified source paths and collapse duplicate source ranges produced by broad `-coverpkg` runs +- the fan-in guard requires every expected unit/backend artifact, all buildable module records, the root integration-tagged bus fixture, and covered representative root, driver, and integration-module source before upload +- Codecov upload errors fail CI; project and patch status checks compare to the base at the repository's established 1% threshold - tracks broad regressions - not used as a substitute for guarantee validation @@ -137,12 +143,10 @@ This section is the core of the plan. Each area should have: - duplicate-delivery idempotency scenario - restart recovery scenarios - broker fault / recovery scenarios (capability-gated) +- Redis handler-time disconnect, lost acknowledgement, and same-task redelivery with one idempotent side effect ### Gaps to add -- Explicit ack-boundary invariants under worker interruption - - example: handler side effect committed, ack path interrupted, duplicate delivery occurs; verify idempotency pattern and state consistency -- “success exactly once” is not promised; test and document “side-effect idempotency required” with reference scenario - Delayed job + restart + recovery invariants for all backends that claim durable delay/retry behavior ## B. Retry, Delay, and Scheduling Semantics @@ -316,6 +320,24 @@ This is a trust-critical area. Users will assume high-level workflow helpers enc - Fuzz/property tests for payload decoding and queue-name normalization - Config defaults invariants (documented defaults should be test-locked) +## J. Public Fake Semantics + +### Guarantees enforced today + +- `queue.NewFake` is the only fake state owner; deprecated `bus.Fake` and `queuefake.Fake` are compatibility views over it. +- Direct dispatch uses the same typed-value conversion and `Job` validation as production runtimes. +- Chain and batch builders use the production workflow engine, record only from `Dispatch`, retain queue/name/failure policy, and expose isolated canonical records plus lookup state. +- Closure callbacks remain fluent compatibility inputs but are not retained in fake runtime state or executed by the recording fake. +- Invalid, abandoned, and canceled builders cannot satisfy workflow assertions. +- Direct, chain, batch, lookup, assertion, Prune, and Reset access is safe across concurrent handles and covered under the race detector. +- Reset clears direct records, workflow records, and workflow-store state for every shared view. + +### Compatibility coverage + +- `queue.NewFake() *queue.FakeQueue`, `queuefake.Fake.Queue() *queue.FakeQueue`, `queuefake.Fake.Workflow() *bus.Fake`, and `bus.NewFake() *bus.Fake` retain their established signatures. +- `bus.Fake` and the one-field `bus.BatchSpec` retain their physical package identity and keyed/unkeyed source forms. +- Legacy bus builders retain shallow job snapshots and Dispatch-time payload JSON encoding before entering the canonical fake. + ## Test Types We Should Add or Expand (Prioritized) ## P0 (Before v1 tag) @@ -412,7 +434,8 @@ Why: - no double-advance / double-terminal transition under concurrent processing in covered scenarios - Notes: - this is a trust-critical P0 item, not optional polish - - Progress: cross-backend callback failure semantics (catch/finally + terminal state) are covered in `integration/bus/integration_test.go`; SQL runtime/store integration now covers chain + batch duplicate callback suppression, callback replay after callback-dispatch fault (chain final callback), and chain/batch dispatch failure state consistency (including batch partial-dispatch-failure-after-progress) + - Progress: cross-backend callback failure semantics (catch/finally + terminal state) are covered in `integration/bus/integration_test.go`; SQL runtime/store integration covers chain + batch duplicate callback suppression, callback replay after callback-dispatch fault (chain final callback), and chain/batch dispatch failure state consistency (including batch partial-dispatch-failure-after-progress). Shared public and real-dialect workflow-store contracts prove concurrent duplicate chain advancement, first-writer outcome categories, suppression of contradictory logical facts, and simultaneous batch aggregation without lost state across memory, SQLite, MySQL, and PostgreSQL. Focused private built-in contracts distinguish response-local `claimedNow` from immutable transition receipts and validate receipt identity on memory and SQLite. `TestTransitionReceiptUnknownVersionsFailClosed` and `TestUnknownTransitionReceiptVersionsBlockRecoveredApplicationExecution` prove unsupported `receipt_version` or observer `event_schema_version` values produce an uncommitted outcome without acknowledgement, application execution, state-commit signaling, or facts; the event schema is independent from the workflow-envelope protocol. `TestDeliveryApplicationStateCommittedSignal`, `TestDatabasePendingRecoveryTokenPreservesPendingRecovery`, and `TestChainPostTransitionFailureMarksCurrentGenerationForRecovery` prove that a receipt-owning generation supersedes inherited provenance when later infrastructure requests same-attempt redelivery. `TestChainSuccessorRejectionRecoversWithoutPredecessorReplay` proves active exact-receipt recovery re-dispatches the immediate successor after definite rejection. `TestChainRecoveryWithoutExactReceiptOwnershipPreservesOnlyLiveContinuation` covers receipt absence, a decorated no-capability store, and supported receipts with different or legacy generation provenance: only the live immediate successor is dispatched, predecessor handlers/facts/callbacks remain suppressed, progressed or terminal state is a no-op, and rejection remains uncommitted for retry. `TestChainSuccessRecoveryRejectsInvalidReceiptShapeBeforeLiveness` proves cancellation/completion corruption fails uncommitted before dispatch or effects. These paths retain the documented at-least-once duplicate ambiguity. The SQLite recovery group plus `mysql_workflow_receipt_recovery` and `postgres_workflow_receipt_recovery` prove supported receipts suppress handler replay and exact recovered-generation ownership gates reconstructed success facts. `TestChainCommittedFailureRecoveryPreservesOneApplicationOccurrence`, its invalid/legacy companions, the first-cause store contract, and the receipt rollback test prove failed-chain receipts return authoritative permanent state across generation variants without repeated handlers, callbacks, or facts. `sqlite_failed_chain_recovery_archives_without_reexecution` extends that proof through repeated real archive failure and best-effort lineage restoration to a final `dead` row with the persisted cause. SQLite also covers completed predecessors, aggregate non-inference, later-attempt ownership, two-member completion ownership, and failed-batch generic permanent archive. Driver tests cover the lineage repair's fence, delay, malformed token, stale owner, and inapplicable branches. `mysql_concurrent_batch_receipt_owner` and `postgres_concurrent_batch_receipt_owner` race twelve fail-fast members through separate workers and prove one aggregate-owner receipt and one failed/cancelled terminal fact pair. `TestWorkflowStoreIntegration_MySQLAutoMigratesMissingReceiptAtLegacyWidths` is the real upgrade gate: it drops only the receipt table beside 512-byte legacy state, proves ordinary startup derives 512-byte receipt identities without altering existing tables, and exercises chain, batch, callback, and receipt keys above fresh defaults. The managed-width fixture separately covers a complete pre-existing wider schema. Still open: managed-schema migration/rollback and real cross-dialect pruning/physical-commit-readback coverage, remaining custom/decorated/raw-store fallback contracts, cross-driver provenance, durable callback/continuation intents, exact successor-enqueue ownership, and a settlement outbox for the no-surviving-row window. + - Final receipt regression gate: `TestRecoveredTransitionReceiptLogicalValidationSeparatesPhysicalOwnership`, `TestChainSuccessRecoveryAllowsDifferentPhysicalDeliveryIdentity`, `TestChainFailureRecoveryAllowsDifferentPhysicalDeliveryIdentity`, and `TestBatchRecoverySettlesNonFactOwnersWithoutFacts` prove a complete persisted owner must retain a nonnegative attempt while a logical duplicate's current attempt may differ or be negative. Chain physical `JobID` may differ; batch `JobID` remains the logical member key. Logical nonowners do not execute handlers, callbacks, or facts; chain success preserves only the live immediate successor, chain failure returns the persisted permanent cause, batch success settles silently, and batch failure returns a generic permanent cause. Exact recovered generation, current attempt, and physical `JobID` remain mandatory for fact reconstruction. `TestSQLStoreBatchAggregateOwnershipMismatchFailsClosed`, `TestSQLStoreBatchAggregateIncarnationMismatchFailsClosed`, and `TestBatchRecoveryRejectsInvalidAggregateReceiptShape` cover missing completion/member, stale incarnation, success-owned cancellation, owner/outcome mismatch, and aggregate/live-state disagreement; every branch is uncommitted and produces no partial effects. Extend workflow integration scenarios to cover: @@ -483,7 +506,7 @@ Why: - scenario results are visible in CI artifacts/logs with backend + scenario naming - Notes: - capability-gate unsupported fault injection paths explicitly - - Implemented in `.github/workflows/soak.yml` `integration-chaos` subset with shared scenario names aligned to current suite (`scenario_dispatch_during_broker_fault`, `scenario_consume_after_broker_recovery`, `scenario_worker_restart_recovery`, `scenario_worker_restart_delay_recovery`, plus contention/shutdown race probes); results are emitted with backend+scenario duration lines and uploaded per-backend logs + - Implemented in the scheduled and manually runnable `.github/workflows/soak.yml` `integration-chaos` subset. `TestIntegrationChaos_RedisBrokerDisconnectRedelivery` stops Redis while an idempotent handler is active, proves its successful return cannot be acknowledged, retains the same active task, and exercises Asynq lease recovery without consuming the application's zero-retry budget. The subset also runs `scenario_dispatch_during_broker_fault`, `scenario_consume_after_broker_recovery`, `scenario_worker_restart_recovery`, `scenario_worker_restart_delay_recovery`, and contention/shutdown race probes. Results include backend and scenario duration lines and per-backend log artifacts. Expand scheduled integration scenarios for: @@ -506,10 +529,11 @@ Why: - Acceptance: - selected scenarios run repeatedly per backend (or backend subsets) - flake rate is recorded by backend/scenario + - capability-gated scenarios are recorded as skips, while a missing expected scenario event fails the repeat job - release candidates require manual review of recent flake results - Notes: - focus on contention, retry timing, shutdown races, ordering - - Implemented via `.github/workflows/soak.yml` `integration-flake-repeat` (scheduled + manual) using `scripts/integration-flake-repeat.sh`; current backend subset is `redis`, `rabbitmq`, `sqs` with per-scenario flake-rate summaries/artifacts in `docs/flake-log.md` review format + - Implemented via `.github/workflows/soak.yml` `integration-flake-repeat` (scheduled + manual) using `scripts/integration-flake-repeat.sh`; current backend subset is `redis`, `rabbitmq`, `sqs` with per-scenario executed pass/fail and explicit capability-skip summaries/artifacts in `docs/flake-log.md` review format Run critical scenarios repeatedly (nightly/RC gate): @@ -612,8 +636,8 @@ Minimum gate for a v1 release candidate: 1. `GOCACHE=/tmp/queue-gocache go test ./...` 2. `GOCACHE=/tmp/queue-gocache ./scripts/test-all-modules.sh` 3. `GOCACHE=/tmp/queue-gocache FULL=1 ./scripts/test-all-modules.sh` (or equivalent split full runs) -4. `INTEGRATION_BACKEND=all GOCACHE=/tmp/queue-gocache go test -tags=integration ./integration/... -count=1` -5. `scripts/coverage-codecov.sh` +4. `scripts/coverage-codecov.sh unit` +5. `INTEGRATION_BACKEND=all scripts/coverage-codecov.sh integration` (runs the full integration suite with coverage; CI parallelizes this by backend) 6. `GOCACHE=/tmp/queue-gocache go run ./docs/readme/main.go` 7. `GOCACHE=/tmp/queue-gocache go run ./docs/examplegen/main.go` 8. `cd examples && GOCACHE=/tmp/queue-gocache go test ./... -run '^TestExamplesBuild$' -count=1` diff --git a/unified_observer_test.go b/unified_observer_test.go new file mode 100644 index 0000000..eb27062 --- /dev/null +++ b/unified_observer_test.go @@ -0,0 +1,146 @@ +package queue + +import ( + "context" + "testing" +) + +// TestWithObserverReceivesEveryEventLayer verifies the root option spans the composed runtime. +func TestWithObserverReceivesEveryEventLayer(t *testing.T) { + var events []Event + observer := ObserverFunc(func(_ context.Context, event Event) { + events = append(events, event) + }) + + q, err := NewSync(WithObserver(observer)) + if err != nil { + t.Fatalf("new sync queue: %v", err) + } + q.Register("reports:build", func(context.Context, Message) error { return nil }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + t.Cleanup(func() { + if err := q.Shutdown(context.Background()); err != nil { + t.Errorf("shutdown: %v", err) + } + }) + + if _, err := q.Dispatch(NewJob("reports:build").OnQueue("default")); err != nil { + t.Fatalf("dispatch: %v", err) + } + + required := map[EventKind]EventLayer{ + EventDispatchStarted: EventLayerQueue, + EventEnqueueAccepted: EventLayerQueue, + EventProcessStarted: EventLayerWorker, + EventJobStarted: EventLayerWorkflow, + } + for kind, layer := range required { + event, ok := findEvent(events, kind) + if !ok { + t.Errorf("missing %q event in %+v", kind, events) + continue + } + if event.Layer != layer { + t.Errorf("event %q layer = %q, want %q", kind, event.Layer, layer) + } + if event.SchemaVersion == 0 || event.EventID == "" || event.Time.IsZero() { + t.Errorf("event %q missing envelope metadata: %+v", kind, event) + } + } +} + +// TestConfigObserverReceivesWorkflowEvents preserves the compatibility configuration path during migration. +func TestConfigObserverReceivesWorkflowEvents(t *testing.T) { + var events []Event + q, err := New(Config{ + Driver: DriverSync, + Observer: ObserverFunc(func(_ context.Context, event Event) { + events = append(events, event) + }), + }) + if err != nil { + t.Fatalf("new sync queue: %v", err) + } + q.Register("reports:build", func(context.Context, Message) error { return nil }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + t.Cleanup(func() { + if err := q.Shutdown(context.Background()); err != nil { + t.Errorf("shutdown: %v", err) + } + }) + + if _, err := q.Dispatch(NewJob("reports:build").OnQueue("default")); err != nil { + t.Fatalf("dispatch: %v", err) + } + if _, ok := findEvent(events, EventJobSucceeded); !ok { + t.Fatalf("config observer did not receive workflow events: %+v", events) + } +} + +// TestConfigAndOptionObserversShareOneEventIdentity prevents nested wrappers from describing one fact twice. +func TestConfigAndOptionObserversShareOneEventIdentity(t *testing.T) { + var configEvents []Event + var optionEvents []Event + q, err := New( + Config{ + Driver: DriverSync, + Observer: ObserverFunc(func(_ context.Context, event Event) { + configEvents = append(configEvents, event) + }), + }, + WithObserver(ObserverFunc(func(_ context.Context, event Event) { + optionEvents = append(optionEvents, event) + })), + ) + if err != nil { + t.Fatalf("new observed queue: %v", err) + } + q.Register("reports:identity", func(context.Context, Message) error { return nil }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + t.Cleanup(func() { + if err := q.Shutdown(context.Background()); err != nil { + t.Errorf("shutdown: %v", err) + } + }) + + if _, err := q.Dispatch(NewJob("reports:identity").OnQueue("default")); err != nil { + t.Fatalf("dispatch: %v", err) + } + for _, kind := range []EventKind{EventEnqueueAccepted, EventProcessStarted, EventJobSucceeded} { + configMatches := eventsOfKind(configEvents, kind) + optionMatches := eventsOfKind(optionEvents, kind) + if len(configMatches) != 1 || len(optionMatches) != 1 { + t.Fatalf("event %q counts = config:%d option:%d, want 1/1", kind, len(configMatches), len(optionMatches)) + } + if configMatches[0].EventID != optionMatches[0].EventID || !configMatches[0].Time.Equal(optionMatches[0].Time) { + t.Fatalf("event %q identity differs: config=%+v option=%+v", kind, configMatches[0], optionMatches[0]) + } + } +} + +// findEvent keeps assertions focused on the unified contract rather than incidental event ordering that will change when enqueue ordering is corrected. +func findEvent(events []Event, kind EventKind) (Event, bool) { + for _, event := range events { + if event.Kind == kind { + return event, true + } + } + return Event{}, false +} + +// eventsOfKind returns every matching fact so duplicate emission is part of the observer contract assertion. +func eventsOfKind(events []Event, kind EventKind) []Event { + matches := make([]Event, 0, 1) + for _, event := range events { + if event.Kind == kind { + matches = append(matches, event) + } + } + return matches +} diff --git a/workflow_adapters.go b/workflow_adapters.go new file mode 100644 index 0000000..e3276c2 --- /dev/null +++ b/workflow_adapters.go @@ -0,0 +1,356 @@ +package queue + +import ( + "context" + + "github.com/goforj/queue/internal/workflow" +) + +// cloneWorkflowPayload preserves nil-versus-empty payload semantics while isolating mutable bytes. +func cloneWorkflowPayload(payload []byte) []byte { + if payload == nil { + return nil + } + cloned := make([]byte, len(payload)) + copy(cloned, payload) + return cloned +} + +// messageFromWorkflow converts an engine context into the root-owned public message model. +func messageFromWorkflow(message workflow.Context) Message { + return Message{ + SchemaVersion: message.SchemaVersion, + DispatchID: message.DispatchID, + JobID: message.JobID, + ChainID: message.ChainID, + BatchID: message.BatchID, + Attempt: message.Attempt, + JobType: message.JobType, + payload: message.PayloadBytes(), + } +} + +// messageToWorkflow converts a public message into the engine's private context model. +func messageToWorkflow(message Message) workflow.Context { + return workflow.NewContext( + message.SchemaVersion, + message.DispatchID, + message.JobID, + message.ChainID, + message.BatchID, + message.Attempt, + message.JobType, + message.payload, + ) +} + +// dispatchResultFromWorkflow converts the engine's dispatch receipt into the root-owned result. +func dispatchResultFromWorkflow(result workflow.DispatchResult) DispatchResult { + return DispatchResult{DispatchID: result.DispatchID} +} + +// storedJobOptionsToWorkflow converts the root-owned delivery policy to its engine representation. +func storedJobOptionsToWorkflow(options StoredJobOptions) workflow.JobOptions { + return workflow.JobOptions{ + Queue: options.Queue, + Delay: options.Delay, + Timeout: options.Timeout, + Retry: options.Retry, + Backoff: options.Backoff, + UniqueFor: options.UniqueFor, + } +} + +// storedJobOptionsFromWorkflow converts the engine delivery policy to its root-owned representation. +func storedJobOptionsFromWorkflow(options workflow.JobOptions) StoredJobOptions { + return StoredJobOptions{ + Queue: options.Queue, + Delay: options.Delay, + Timeout: options.Timeout, + Retry: options.Retry, + Backoff: options.Backoff, + UniqueFor: options.UniqueFor, + } +} + +// storedJobToWorkflow converts a persisted public job without changing its version-one JSON shape. +func storedJobToWorkflow(job StoredJob) workflow.StoredJob { + return workflow.StoredJob{ + Type: job.Type, + Payload: cloneWorkflowPayload(job.Payload), + Options: storedJobOptionsToWorkflow(job.Options), + } +} + +// storedJobFromWorkflow converts a persisted engine job into the public root-owned shape. +func storedJobFromWorkflow(job workflow.StoredJob) StoredJob { + return StoredJob{ + Type: job.Type, + Payload: cloneWorkflowPayload(job.Payload), + Options: storedJobOptionsFromWorkflow(job.Options), + } +} + +// chainNodeToWorkflow converts one public chain node into the engine model. +func chainNodeToWorkflow(node ChainNode) workflow.ChainNode { + return workflow.ChainNode{ + NodeID: node.NodeID, + Job: storedJobToWorkflow(node.Job), + } +} + +// chainNodeFromWorkflow converts one engine chain node into the public model. +func chainNodeFromWorkflow(node workflow.ChainNode) ChainNode { + return ChainNode{ + NodeID: node.NodeID, + Job: storedJobFromWorkflow(node.Job), + } +} + +// chainNodesToWorkflow converts a public node slice while preserving nil slices. +func chainNodesToWorkflow(nodes []ChainNode) []workflow.ChainNode { + if nodes == nil { + return nil + } + converted := make([]workflow.ChainNode, len(nodes)) + for i, node := range nodes { + converted[i] = chainNodeToWorkflow(node) + } + return converted +} + +// chainNodesFromWorkflow converts an engine node slice while preserving nil slices. +func chainNodesFromWorkflow(nodes []workflow.ChainNode) []ChainNode { + if nodes == nil { + return nil + } + converted := make([]ChainNode, len(nodes)) + for i, node := range nodes { + converted[i] = chainNodeFromWorkflow(node) + } + return converted +} + +// chainRecordToWorkflow converts a public chain creation record into the engine model. +func chainRecordToWorkflow(record ChainRecord) workflow.ChainRecord { + return workflow.ChainRecord{ + ChainID: record.ChainID, + DispatchID: record.DispatchID, + Queue: record.Queue, + Nodes: chainNodesToWorkflow(record.Nodes), + CreatedAt: record.CreatedAt, + } +} + +// chainRecordFromWorkflow converts an engine chain creation record into the public model. +func chainRecordFromWorkflow(record workflow.ChainRecord) ChainRecord { + return ChainRecord{ + ChainID: record.ChainID, + DispatchID: record.DispatchID, + Queue: record.Queue, + Nodes: chainNodesFromWorkflow(record.Nodes), + CreatedAt: record.CreatedAt, + } +} + +// chainStateToWorkflow converts a public chain state into the engine model. +func chainStateToWorkflow(state ChainState) workflow.ChainState { + return workflow.ChainState{ + ChainID: state.ChainID, + DispatchID: state.DispatchID, + Queue: state.Queue, + Nodes: chainNodesToWorkflow(state.Nodes), + NextIndex: state.NextIndex, + Completed: state.Completed, + Failed: state.Failed, + Failure: state.Failure, + CreatedAt: state.CreatedAt, + UpdatedAt: state.UpdatedAt, + } +} + +// chainStateFromWorkflow converts an engine chain state into the public model. +func chainStateFromWorkflow(state workflow.ChainState) ChainState { + return ChainState{ + ChainID: state.ChainID, + DispatchID: state.DispatchID, + Queue: state.Queue, + Nodes: chainNodesFromWorkflow(state.Nodes), + NextIndex: state.NextIndex, + Completed: state.Completed, + Failed: state.Failed, + Failure: state.Failure, + CreatedAt: state.CreatedAt, + UpdatedAt: state.UpdatedAt, + } +} + +// batchJobToWorkflow converts one public batch member into the engine model. +func batchJobToWorkflow(job BatchJob) workflow.BatchJob { + return workflow.BatchJob{ + JobID: job.JobID, + Job: storedJobToWorkflow(job.Job), + } +} + +// batchJobFromWorkflow converts one engine batch member into the public model. +func batchJobFromWorkflow(job workflow.BatchJob) BatchJob { + return BatchJob{ + JobID: job.JobID, + Job: storedJobFromWorkflow(job.Job), + } +} + +// batchJobsToWorkflow converts a public batch member slice while preserving nil slices. +func batchJobsToWorkflow(jobs []BatchJob) []workflow.BatchJob { + if jobs == nil { + return nil + } + converted := make([]workflow.BatchJob, len(jobs)) + for i, job := range jobs { + converted[i] = batchJobToWorkflow(job) + } + return converted +} + +// batchJobsFromWorkflow converts an engine batch member slice while preserving nil slices. +func batchJobsFromWorkflow(jobs []workflow.BatchJob) []BatchJob { + if jobs == nil { + return nil + } + converted := make([]BatchJob, len(jobs)) + for i, job := range jobs { + converted[i] = batchJobFromWorkflow(job) + } + return converted +} + +// batchRecordToWorkflow converts a public batch creation record into the engine model. +func batchRecordToWorkflow(record BatchRecord) workflow.BatchRecord { + return workflow.BatchRecord{ + BatchID: record.BatchID, + DispatchID: record.DispatchID, + Name: record.Name, + Queue: record.Queue, + AllowFailed: record.AllowFailed, + Jobs: batchJobsToWorkflow(record.Jobs), + CreatedAt: record.CreatedAt, + } +} + +// batchRecordFromWorkflow converts an engine batch creation record into the public model. +func batchRecordFromWorkflow(record workflow.BatchRecord) BatchRecord { + return BatchRecord{ + BatchID: record.BatchID, + DispatchID: record.DispatchID, + Name: record.Name, + Queue: record.Queue, + AllowFailed: record.AllowFailed, + Jobs: batchJobsFromWorkflow(record.Jobs), + CreatedAt: record.CreatedAt, + } +} + +// batchStateToWorkflow converts a public aggregate state into the engine model. +func batchStateToWorkflow(state BatchState) workflow.BatchState { + return workflow.BatchState{ + BatchID: state.BatchID, + DispatchID: state.DispatchID, + Name: state.Name, + Queue: state.Queue, + AllowFailed: state.AllowFailed, + Total: state.Total, + Pending: state.Pending, + Processed: state.Processed, + Failed: state.Failed, + Cancelled: state.Cancelled, + Completed: state.Completed, + CreatedAt: state.CreatedAt, + UpdatedAt: state.UpdatedAt, + } +} + +// batchStateFromWorkflow converts an engine aggregate state into the public model. +func batchStateFromWorkflow(state workflow.BatchState) BatchState { + return BatchState{ + BatchID: state.BatchID, + DispatchID: state.DispatchID, + Name: state.Name, + Queue: state.Queue, + AllowFailed: state.AllowFailed, + Total: state.Total, + Pending: state.Pending, + Processed: state.Processed, + Failed: state.Failed, + Cancelled: state.Cancelled, + Completed: state.Completed, + CreatedAt: state.CreatedAt, + UpdatedAt: state.UpdatedAt, + } +} + +// workflowMiddlewareAdapter translates physical root messages around one public middleware. +type workflowMiddlewareAdapter struct { + middleware Middleware +} + +// Handle preserves middleware message replacement while crossing the private engine boundary. +func (a workflowMiddlewareAdapter) Handle(ctx context.Context, message workflow.Context, next workflow.Next) error { + return a.middleware.Handle(ctx, messageFromWorkflow(message), func(nextContext context.Context, nextMessage Message) error { + return next(nextContext, messageToWorkflow(nextMessage)) + }) +} + +// middlewaresToWorkflow converts public middleware into private engine adapters. +func middlewaresToWorkflow(middlewares []Middleware) []workflow.Middleware { + if middlewares == nil { + return nil + } + converted := make([]workflow.Middleware, 0, len(middlewares)) + for _, middleware := range middlewares { + if middleware != nil { + converted = append(converted, workflowMiddlewareAdapter{middleware: middleware}) + } + } + return converted +} + +// chainCatchToWorkflow adapts an optional public chain failure callback to engine state. +func chainCatchToWorkflow(callback func(context.Context, ChainState, error) error) func(context.Context, workflow.ChainState, error) error { + if callback == nil { + return nil + } + return func(ctx context.Context, state workflow.ChainState, err error) error { + return callback(ctx, chainStateFromWorkflow(state), err) + } +} + +// chainFinallyToWorkflow adapts an optional public chain terminal callback to engine state. +func chainFinallyToWorkflow(callback func(context.Context, ChainState) error) func(context.Context, workflow.ChainState) error { + if callback == nil { + return nil + } + return func(ctx context.Context, state workflow.ChainState) error { + return callback(ctx, chainStateFromWorkflow(state)) + } +} + +// batchStateCallbackToWorkflow adapts an optional public batch callback to engine state. +func batchStateCallbackToWorkflow(callback func(context.Context, BatchState) error) func(context.Context, workflow.BatchState) error { + if callback == nil { + return nil + } + return func(ctx context.Context, state workflow.BatchState) error { + return callback(ctx, batchStateFromWorkflow(state)) + } +} + +// batchCatchToWorkflow adapts an optional public batch failure callback to engine state. +func batchCatchToWorkflow(callback func(context.Context, BatchState, error) error) func(context.Context, workflow.BatchState, error) error { + if callback == nil { + return nil + } + return func(ctx context.Context, state workflow.BatchState, err error) error { + return callback(ctx, batchStateFromWorkflow(state), err) + } +} diff --git a/workflow_forwarding_contract_test.go b/workflow_forwarding_contract_test.go new file mode 100644 index 0000000..7cc021d --- /dev/null +++ b/workflow_forwarding_contract_test.go @@ -0,0 +1,412 @@ +package queue + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/goforj/queue/internal/workflow" +) + +type workflowRateLimiterFunc func(context.Context, string) (bool, time.Duration, error) + +// Allow invokes the test rate-limiter function. +func (f workflowRateLimiterFunc) Allow(ctx context.Context, key string) (bool, time.Duration, error) { + return f(ctx, key) +} + +type workflowLockerFunc func(context.Context, string, time.Duration) (Lock, bool, error) + +// Acquire invokes the test locker function. +func (f workflowLockerFunc) Acquire(ctx context.Context, key string, ttl time.Duration) (Lock, bool, error) { + return f(ctx, key, ttl) +} + +type workflowTestLock struct { + release func(context.Context) error +} + +// Release invokes the test lock release function. +func (l *workflowTestLock) Release(ctx context.Context) error { + return l.release(ctx) +} + +type workflowCancelStore struct { + workflow.Store + receivedContext context.Context + receivedBatchID string + err error +} + +// CancelBatch records the forwarded call and preserves its configured error identity. +func (s *workflowCancelStore) CancelBatch(ctx context.Context, batchID string) error { + s.receivedContext = ctx + s.receivedBatchID = batchID + return s.err +} + +// TestNewSQLStoreForwardsConstructionError verifies the root constructor does not reinterpret engine errors. +func TestNewSQLStoreForwardsConstructionError(t *testing.T) { + store, err := NewSQLStore(SQLStoreConfig{}) + if store != nil { + t.Fatalf("store = %T, want nil on construction failure", store) + } + want := "sql store driver name is required" + if err == nil || err.Error() != want { + t.Fatalf("construction error = %v, want %q", err, want) + } +} + +// TestWorkflowStoreViewCancelBatchForwardsUnchanged verifies the built-in view is transparent to callers. +func TestWorkflowStoreViewCancelBatchForwardsUnchanged(t *testing.T) { + type contextKey struct{} + + wantErr := errors.New("cancel failed") + store := &workflowCancelStore{err: wantErr} + view := &workflowStoreView{store: store} + ctx := context.WithValue(context.Background(), contextKey{}, "request") + + if err := view.CancelBatch(ctx, "batch-17"); err != wantErr { + t.Fatalf("cancel error = %v, want exact error %v", err, wantErr) + } + if store.receivedBatchID != "batch-17" { + t.Fatalf("batch id = %q, want batch-17", store.receivedBatchID) + } + if got := store.receivedContext.Value(contextKey{}); got != "request" { + t.Fatalf("context value = %v, want request", got) + } +} + +// TestSkipWhenBranches pins pass-through and suppression behavior for every predicate outcome. +func TestSkipWhenBranches(t *testing.T) { + message := NewMessage("reports:build", []byte(`{"id":7}`)) + nextErr := errors.New("next failed") + + tests := []struct { + name string + predicate func(context.Context, Message) bool + wantErr error + wantCalled bool + }{ + {name: "nil predicate", wantErr: nextErr, wantCalled: true}, + {name: "false predicate", predicate: func(context.Context, Message) bool { return false }, wantErr: nextErr, wantCalled: true}, + {name: "true predicate", predicate: func(context.Context, Message) bool { return true }, wantCalled: false}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + called := false + err := (SkipWhen{Predicate: test.predicate}).Handle(context.Background(), message, func(context.Context, Message) error { + called = true + return nextErr + }) + if err != test.wantErr { + t.Fatalf("handle error = %v, want exact error %v", err, test.wantErr) + } + if called != test.wantCalled { + t.Fatalf("next called = %t, want %t", called, test.wantCalled) + } + }) + } +} + +// TestFailOnErrorBranches pins successful, retryable, and permanent error paths. +func TestFailOnErrorBranches(t *testing.T) { + message := NewMessage("reports:build", nil) + nextErr := errors.New("next failed") + + t.Run("success bypasses predicate", func(t *testing.T) { + predicateCalled := false + middleware := FailOnError{When: func(error) bool { + predicateCalled = true + return true + }} + if err := middleware.Handle(context.Background(), message, func(context.Context, Message) error { return nil }); err != nil { + t.Fatalf("handle success: %v", err) + } + if predicateCalled { + t.Fatal("predicate called for successful execution") + } + }) + + t.Run("unmatched error remains unchanged", func(t *testing.T) { + middleware := FailOnError{When: func(error) bool { return false }} + if err := middleware.Handle(context.Background(), message, func(context.Context, Message) error { return nextErr }); err != nextErr { + t.Fatalf("handle error = %v, want exact error %v", err, nextErr) + } + }) + + for _, test := range []struct { + name string + when func(error) bool + }{ + {name: "nil predicate"}, + {name: "matched predicate", when: func(error) bool { return true }}, + } { + t.Run(test.name, func(t *testing.T) { + err := (FailOnError{When: test.when}).Handle(context.Background(), message, func(context.Context, Message) error { return nextErr }) + if !IsPermanent(err) { + t.Fatalf("handle error = %v, want permanent classification", err) + } + if !errors.Is(err, nextErr) { + t.Fatalf("handle error = %v, want wrapped error %v", err, nextErr) + } + }) + } +} + +// TestRateLimitBranches pins key resolution, limiter failures, denial, and pass-through behavior. +func TestRateLimitBranches(t *testing.T) { + message := NewMessage("reports:build", nil) + nextErr := errors.New("next failed") + limiterErr := errors.New("limiter failed") + + t.Run("nil limiter", func(t *testing.T) { + called := false + err := (RateLimit{}).Handle(context.Background(), message, func(context.Context, Message) error { + called = true + return nextErr + }) + if err != nextErr || !called { + t.Fatalf("handle result = (%v, %t), want exact next error and call", err, called) + } + }) + + for _, test := range []struct { + name string + key func(context.Context, Message) string + wantKey string + }{ + {name: "default key", wantKey: "reports:build"}, + {name: "empty resolved key", key: func(context.Context, Message) string { return "" }, wantKey: "reports:build"}, + {name: "custom key", key: func(context.Context, Message) string { return "tenant:17" }, wantKey: "tenant:17"}, + } { + t.Run(test.name, func(t *testing.T) { + var gotKey string + limiter := workflowRateLimiterFunc(func(_ context.Context, key string) (bool, time.Duration, error) { + gotKey = key + return true, time.Second, nil + }) + err := (RateLimit{Key: test.key, Limiter: limiter}).Handle(context.Background(), message, func(context.Context, Message) error { return nextErr }) + if err != nextErr { + t.Fatalf("handle error = %v, want exact next error %v", err, nextErr) + } + if gotKey != test.wantKey { + t.Fatalf("limiter key = %q, want %q", gotKey, test.wantKey) + } + }) + } + + t.Run("limiter error", func(t *testing.T) { + called := false + limiter := workflowRateLimiterFunc(func(context.Context, string) (bool, time.Duration, error) { + return false, 0, limiterErr + }) + err := (RateLimit{Limiter: limiter}).Handle(context.Background(), message, func(context.Context, Message) error { + called = true + return nil + }) + if err != limiterErr || called { + t.Fatalf("handle result = (%v, %t), want exact limiter error without next call", err, called) + } + }) + + t.Run("denied", func(t *testing.T) { + called := false + limiter := workflowRateLimiterFunc(func(context.Context, string) (bool, time.Duration, error) { + return false, time.Second, nil + }) + err := (RateLimit{Limiter: limiter}).Handle(context.Background(), message, func(context.Context, Message) error { + called = true + return nil + }) + if err != ErrRateLimited || called { + t.Fatalf("handle result = (%v, %t), want exact ErrRateLimited without next call", err, called) + } + }) +} + +// TestWithoutOverlappingBranches pins key resolution, acquisition outcomes, and release behavior. +func TestWithoutOverlappingBranches(t *testing.T) { + message := NewMessage("reports:build", nil) + nextErr := errors.New("next failed") + lockerErr := errors.New("locker failed") + releaseErr := errors.New("release failed") + + t.Run("nil locker", func(t *testing.T) { + called := false + err := (WithoutOverlapping{}).Handle(context.Background(), message, func(context.Context, Message) error { + called = true + return nextErr + }) + if err != nextErr || !called { + t.Fatalf("handle result = (%v, %t), want exact next error and call", err, called) + } + }) + + t.Run("acquire error uses default key", func(t *testing.T) { + var gotKey string + locker := workflowLockerFunc(func(_ context.Context, key string, _ time.Duration) (Lock, bool, error) { + gotKey = key + return nil, false, lockerErr + }) + err := (WithoutOverlapping{Locker: locker}).Handle(context.Background(), message, func(context.Context, Message) error { return nil }) + if err != lockerErr || gotKey != "reports:build" { + t.Fatalf("handle result = (%v, %q), want exact locker error and default key", err, gotKey) + } + }) + + t.Run("denied uses default for empty resolved key", func(t *testing.T) { + called := false + var gotKey string + locker := workflowLockerFunc(func(_ context.Context, key string, _ time.Duration) (Lock, bool, error) { + gotKey = key + return nil, false, nil + }) + err := (WithoutOverlapping{ + Key: func(context.Context, Message) string { return "" }, + Locker: locker, + }).Handle(context.Background(), message, func(context.Context, Message) error { + called = true + return nil + }) + if err != ErrOverlapping || called || gotKey != "reports:build" { + t.Fatalf("handle result = (%v, %t, %q), want exact ErrOverlapping, no next call, and default key", err, called, gotKey) + } + }) + + t.Run("acquired custom key releases after next", func(t *testing.T) { + const ttl = 3 * time.Second + var gotKey string + var gotTTL time.Duration + released := false + lock := &workflowTestLock{release: func(context.Context) error { + released = true + return releaseErr + }} + locker := workflowLockerFunc(func(_ context.Context, key string, requestedTTL time.Duration) (Lock, bool, error) { + gotKey = key + gotTTL = requestedTTL + return lock, true, nil + }) + err := (WithoutOverlapping{ + Key: func(context.Context, Message) string { return "tenant:17" }, + TTL: ttl, + Locker: locker, + }).Handle(context.Background(), message, func(context.Context, Message) error { return nextErr }) + if err != nextErr { + t.Fatalf("handle error = %v, want exact next error %v", err, nextErr) + } + if gotKey != "tenant:17" || gotTTL != ttl || !released { + t.Fatalf("lock lifecycle = (%q, %s, %t), want custom key, ttl, and release", gotKey, gotTTL, released) + } + }) +} + +// TestRetryPolicyPassesThroughUnchanged verifies retry ownership stays with the worker runtime. +func TestRetryPolicyPassesThroughUnchanged(t *testing.T) { + type contextKey struct{} + + wantErr := errors.New("next failed") + wantMessage := NewMessage("reports:build", []byte(`{"id":7}`)) + ctx := context.WithValue(context.Background(), contextKey{}, "request") + called := false + err := (RetryPolicy{}).Handle(ctx, wantMessage, func(gotContext context.Context, gotMessage Message) error { + called = true + if got := gotContext.Value(contextKey{}); got != "request" { + t.Errorf("context value = %v, want request", got) + } + if gotMessage.JobType != wantMessage.JobType || string(gotMessage.PayloadBytes()) != string(wantMessage.PayloadBytes()) { + t.Errorf("message = %+v, want unchanged message %+v", gotMessage, wantMessage) + } + return wantErr + }) + if err != wantErr || !called { + t.Fatalf("handle result = (%v, %t), want exact next error and call", err, called) + } +} + +// TestWorkflowAdaptersPreserveNilAndEmptyCollectionsAndPayloads pins nil as distinct from an allocated empty value. +func TestWorkflowAdaptersPreserveNilAndEmptyCollectionsAndPayloads(t *testing.T) { + if payload := NewMessage("reports:build", nil).PayloadBytes(); payload != nil { + t.Fatalf("new message payload = %#v, want nil", payload) + } + if payload := messageFromWorkflow(workflow.NewContext(1, "", "", "", "", 0, "reports:build", nil)).PayloadBytes(); payload != nil { + t.Fatalf("message from workflow payload = %#v, want nil", payload) + } + if payload := messageToWorkflow(NewMessage("reports:build", nil)).PayloadBytes(); payload != nil { + t.Fatalf("message to workflow payload = %#v, want nil", payload) + } + if payload := storedJobToWorkflow(StoredJob{}).Payload; payload != nil { + t.Fatalf("stored job to workflow payload = %#v, want nil", payload) + } + if payload := storedJobFromWorkflow(workflow.StoredJob{}).Payload; payload != nil { + t.Fatalf("stored job from workflow payload = %#v, want nil", payload) + } + emptyMessage := NewMessage("reports:build", []byte{}) + if payload := messageToWorkflow(emptyMessage).PayloadBytes(); payload == nil || len(payload) != 0 { + t.Fatalf("empty message to workflow payload = %#v, want non-nil empty slice", payload) + } + if payload := messageFromWorkflow(workflow.NewContext(1, "", "", "", "", 0, "reports:build", []byte{})).PayloadBytes(); payload == nil || len(payload) != 0 { + t.Fatalf("empty message from workflow payload = %#v, want non-nil empty slice", payload) + } + if payload := storedJobToWorkflow(StoredJob{Payload: []byte{}}).Payload; payload == nil || len(payload) != 0 { + t.Fatalf("empty stored job to workflow payload = %#v, want non-nil empty slice", payload) + } + if payload := storedJobFromWorkflow(workflow.StoredJob{Payload: []byte{}}).Payload; payload == nil || len(payload) != 0 { + t.Fatalf("empty stored job from workflow payload = %#v, want non-nil empty slice", payload) + } + + if nodes := chainNodesToWorkflow(nil); nodes != nil { + t.Fatalf("chain nodes to workflow = %#v, want nil", nodes) + } + if nodes := chainNodesFromWorkflow(nil); nodes != nil { + t.Fatalf("chain nodes from workflow = %#v, want nil", nodes) + } + if jobs := batchJobsToWorkflow(nil); jobs != nil { + t.Fatalf("batch jobs to workflow = %#v, want nil", jobs) + } + if jobs := batchJobsFromWorkflow(nil); jobs != nil { + t.Fatalf("batch jobs from workflow = %#v, want nil", jobs) + } +} + +// TestMiddlewaresToWorkflowPreservesNilAndFiltersNilEntries verifies optional middleware lists compose safely. +func TestMiddlewaresToWorkflowPreservesNilAndFiltersNilEntries(t *testing.T) { + if converted := middlewaresToWorkflow(nil); converted != nil { + t.Fatalf("nil middleware conversion = %#v, want nil", converted) + } + + called := false + middleware := MiddlewareFunc(func(ctx context.Context, message Message, next Next) error { + called = true + return next(ctx, message) + }) + converted := middlewaresToWorkflow([]Middleware{nil, middleware, nil}) + if len(converted) != 1 { + t.Fatalf("converted middleware count = %d, want 1", len(converted)) + } + if err := converted[0].Handle(context.Background(), workflow.NewContext(1, "", "", "", "", 0, "reports:build", nil), func(context.Context, workflow.Context) error { return nil }); err != nil { + t.Fatalf("run converted middleware: %v", err) + } + if !called { + t.Fatal("non-nil middleware was not retained") + } +} + +// TestNilWorkflowCallbackAdaptersRemainNil verifies absent callbacks do not become callable wrappers. +func TestNilWorkflowCallbackAdaptersRemainNil(t *testing.T) { + if chainCatchToWorkflow(nil) != nil { + t.Fatal("nil chain catch callback became non-nil") + } + if chainFinallyToWorkflow(nil) != nil { + t.Fatal("nil chain finally callback became non-nil") + } + if batchStateCallbackToWorkflow(nil) != nil { + t.Fatal("nil batch state callback became non-nil") + } + if batchCatchToWorkflow(nil) != nil { + t.Fatal("nil batch catch callback became non-nil") + } +} diff --git a/workflow_middleware.go b/workflow_middleware.go new file mode 100644 index 0000000..a7fa410 --- /dev/null +++ b/workflow_middleware.go @@ -0,0 +1,159 @@ +package queue + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/goforj/queue/busruntime" +) + +// Next invokes the remaining queue middleware and logical job handler. +// @group Queue +type Next func(ctx context.Context, message Message) error + +// Middleware intercepts logical queue job execution. +// @group Queue +type Middleware interface { + // Handle wraps the remaining middleware and handler chain. + Handle(ctx context.Context, message Message, next Next) error +} + +// MiddlewareFunc adapts a function to Middleware. +// @group Queue +type MiddlewareFunc func(ctx context.Context, message Message, next Next) error + +// Handle calls the wrapped middleware function. +func (f MiddlewareFunc) Handle(ctx context.Context, message Message, next Next) error { + return f(ctx, message, next) +} + +var ( + // ErrSkipped identifies a job intentionally suppressed by middleware. + ErrSkipped = errors.New("bus job skipped by middleware") + // ErrRateLimited identifies a job rejected by its configured rate limiter. + ErrRateLimited = errors.New("bus job rate limited") + // ErrOverlapping identifies a job rejected because its execution key is already locked. + ErrOverlapping = errors.New("bus job overlap prevented") +) + +// RetryPolicy leaves retry ownership to the underlying worker runtime. +// @group Queue +type RetryPolicy struct{} + +// Handle passes execution through without modification. +func (RetryPolicy) Handle(ctx context.Context, message Message, next Next) error { + return next(ctx, message) +} + +// SkipWhen suppresses handler execution when its predicate matches. +// @group Queue +type SkipWhen struct { + Predicate func(ctx context.Context, message Message) bool +} + +// Handle skips job execution when Predicate returns true. +func (s SkipWhen) Handle(ctx context.Context, message Message, next Next) error { + if s.Predicate != nil && s.Predicate(ctx, message) { + return nil + } + return next(ctx, message) +} + +// FailOnError marks selected handler failures as permanent. +// @group Queue +type FailOnError struct { + When func(err error) bool +} + +// Handle wraps matched errors as fatal errors to stop retries. +func (f FailOnError) Handle(ctx context.Context, message Message, next Next) error { + err := next(ctx, message) + if err == nil { + return nil + } + if f.When == nil || f.When(err) { + return busruntime.Permanent(fmt.Errorf("fatal bus error: %w", err)) + } + return err +} + +// RateLimiter decides whether a logical key may execute now. +// @group Queue +type RateLimiter interface { + // Allow returns whether key may execute and any suggested retry delay. + Allow(ctx context.Context, key string) (allowed bool, retryAfter time.Duration, err error) +} + +// RateLimit applies a RateLimiter before handler execution. +// @group Queue +type RateLimit struct { + Key func(ctx context.Context, message Message) string + Limiter RateLimiter +} + +// Handle applies limiter checks before executing the next handler. +func (r RateLimit) Handle(ctx context.Context, message Message, next Next) error { + if r.Limiter == nil { + return next(ctx, message) + } + key := message.JobType + if r.Key != nil { + if resolved := r.Key(ctx, message); resolved != "" { + key = resolved + } + } + allowed, _, err := r.Limiter.Allow(ctx, key) + if err != nil { + return err + } + if !allowed { + return ErrRateLimited + } + return next(ctx, message) +} + +// Lock represents an acquired overlap-prevention lease. +// @group Queue +type Lock interface { + // Release relinquishes the acquired lease. + Release(ctx context.Context) error +} + +// Locker acquires keyed leases for overlap prevention. +// @group Queue +type Locker interface { + // Acquire attempts to hold key for ttl. + Acquire(ctx context.Context, key string, ttl time.Duration) (Lock, bool, error) +} + +// WithoutOverlapping serializes executions that resolve to the same key. +// @group Queue +type WithoutOverlapping struct { + Key func(ctx context.Context, message Message) string + TTL time.Duration + Locker Locker +} + +// Handle acquires a lock and prevents concurrent overlap for the same key. +func (w WithoutOverlapping) Handle(ctx context.Context, message Message, next Next) error { + if w.Locker == nil { + return next(ctx, message) + } + key := message.JobType + if w.Key != nil { + if resolved := w.Key(ctx, message); resolved != "" { + key = resolved + } + } + lock, ok, err := w.Locker.Acquire(ctx, key, w.TTL) + if err != nil { + return err + } + if !ok { + return ErrOverlapping + } + defer func() { _ = lock.Release(ctx) }() + return next(ctx, message) +} diff --git a/workflow_models.go b/workflow_models.go new file mode 100644 index 0000000..6dce5bf --- /dev/null +++ b/workflow_models.go @@ -0,0 +1,208 @@ +package queue + +import ( + "context" + "encoding/json" + "time" +) + +// Message is the delivered logical job message passed to queue handlers and middleware. +// Its exported fields carry workflow correlation metadata while its payload remains +// isolated behind PayloadBytes and Bind. +// @group Queue +type Message struct { + SchemaVersion int + DispatchID string + JobID string + ChainID string + BatchID string + Attempt int + JobType string + payload []byte +} + +// NewMessage creates a logical queue message from an application job type and exact payload bytes. +// The payload is copied so callers can safely reuse or mutate their input buffer. +// @group Constructors +func NewMessage(jobType string, payload []byte) Message { + return Message{ + JobType: jobType, + payload: cloneWorkflowPayload(payload), + } +} + +// PayloadBytes returns an isolated copy of the raw job payload. +// @group Queue +func (m Message) PayloadBytes() []byte { + return cloneWorkflowPayload(m.payload) +} + +// Bind unmarshals the raw job payload into dst. +// @group Queue +func (m Message) Bind(dst any) error { + return json.Unmarshal(m.payload, dst) +} + +// DispatchResult identifies an accepted logical dispatch. +// @group Queue +type DispatchResult struct { + DispatchID string +} + +// StoredJobOptions is the stable delivery-policy shape persisted inside workflow records. +// Field names intentionally retain their version-one JSON casing. +// @group Queue +type StoredJobOptions struct { + Queue string + Delay time.Duration + Timeout time.Duration + Retry int + Backoff time.Duration + UniqueFor time.Duration +} + +// StoredJob is the stable logical-job shape persisted inside workflow records. +// @group Queue +type StoredJob struct { + Type string `json:"type"` + Payload []byte `json:"payload"` + Options StoredJobOptions `json:"options"` +} + +// ChainNode is one persisted step in a chain workflow. +// @group Queue +type ChainNode struct { + NodeID string + Job StoredJob +} + +// ChainRecord is the persisted representation used to create a chain workflow. +// @group Queue +type ChainRecord struct { + ChainID string + DispatchID string + Queue string + Nodes []ChainNode + CreatedAt time.Time +} + +// ChainState is the persisted view of a chain workflow. +// @group Queue +type ChainState struct { + ChainID string + DispatchID string + Queue string + Nodes []ChainNode + NextIndex int + Completed bool + Failed bool + Failure string + CreatedAt time.Time + UpdatedAt time.Time +} + +// BatchJob is one persisted member of a batch workflow. +// @group Queue +type BatchJob struct { + JobID string + Job StoredJob +} + +// BatchJobOutcome identifies the durable result that first settled one batch member. +// @group Queue +type BatchJobOutcome string + +const ( + // BatchJobSucceeded records successful member settlement. + // @group Queue + BatchJobSucceeded BatchJobOutcome = "succeeded" + // BatchJobFailed records failed member settlement. + // @group Queue + BatchJobFailed BatchJobOutcome = "failed" +) + +// BatchRecord is the persisted representation used to create a batch workflow. +// @group Queue +type BatchRecord struct { + BatchID string + DispatchID string + Name string + Queue string + AllowFailed bool + Jobs []BatchJob + CreatedAt time.Time +} + +// BatchState is the persisted aggregate execution view of a batch workflow. +// @group Queue +type BatchState struct { + BatchID string + DispatchID string + Name string + Queue string + AllowFailed bool + Total int + Pending int + Processed int + Failed int + Cancelled bool + Completed bool + CreatedAt time.Time + UpdatedAt time.Time +} + +// WorkflowStore persists chain, batch, and callback state for orchestration. +// Implement WorkflowOutcomeStore as well when a custom store must arbitrate +// contradictory physical deliveries atomically; built-in stores provide both. +// @group Queue +type WorkflowStore interface { + // CreateChain persists a newly accepted chain. ChainID and every NodeID must + // be non-empty, Nodes must contain at least one entry, and NodeIDs must be unique. + CreateChain(ctx context.Context, rec ChainRecord) error + // AdvanceChain atomically claims completedNode and returns the current successor. + // Repeating the same (chainID, completedNode) claim must not advance again. + // When done is true, GetChain must immediately expose Completed or Failed state. + AdvanceChain(ctx context.Context, chainID string, completedNode string) (next *ChainNode, done bool, err error) + // FailChain commits terminal failure without replacing completed state. + FailChain(ctx context.Context, chainID string, cause error) error + // GetChain returns current chain state. + GetChain(ctx context.Context, chainID string) (ChainState, error) + + // CreateBatch persists a newly accepted batch. BatchID and every JobID must + // be non-empty, Jobs must contain at least one entry, and JobIDs must be unique. + CreateBatch(ctx context.Context, rec BatchRecord) error + // MarkBatchJobStarted records that one batch member began execution. + MarkBatchJobStarted(ctx context.Context, batchID, jobID string) error + // MarkBatchJobSucceeded commits the first outcome for (batchID, jobID). + // Duplicate outcomes must return current state without changing counters. + MarkBatchJobSucceeded(ctx context.Context, batchID, jobID string) (BatchState, bool, error) + // MarkBatchJobFailed commits the first outcome for (batchID, jobID). + // Duplicate outcomes must return current state without changing counters. + MarkBatchJobFailed(ctx context.Context, batchID, jobID string, cause error) (BatchState, bool, error) + // CancelBatch commits aggregate batch cancellation. + CancelBatch(ctx context.Context, batchID string) error + // GetBatch returns current batch state. + GetBatch(ctx context.Context, batchID string) (BatchState, error) + + // MarkCallbackInvoked atomically claims one callback idempotency key. + MarkCallbackInvoked(ctx context.Context, key string) (bool, error) + // Prune removes terminal workflow state older than before. + Prune(ctx context.Context, before time.Time) error +} + +// WorkflowOutcomeStore strengthens WorkflowStore with first-writer ownership +// when duplicate physical deliveries disagree about a logical job outcome. +// Built-in stores implement this additive capability; established custom +// WorkflowStore implementations remain source-compatible. +// @group Queue +type WorkflowOutcomeStore interface { + WorkflowStore + + // FailChainNode commits failure only while nodeID is the current unsettled node. + // owned remains true on replay while that node's failure owns the chain. + FailChainNode(ctx context.Context, chainID, nodeID string, cause error) (state ChainState, owned bool, err error) + // SettleBatchJob returns the first committed outcome for one batch member. + // owned remains true on same-outcome replay and false when the opposite outcome won. + // Ownership covers the outcome category; BatchState does not retain a per-member cause. + SettleBatchJob(ctx context.Context, batchID, jobID string, outcome BatchJobOutcome, cause error) (state BatchState, owned bool, err error) +} diff --git a/workflow_models_test.go b/workflow_models_test.go new file mode 100644 index 0000000..431519d --- /dev/null +++ b/workflow_models_test.go @@ -0,0 +1,139 @@ +package queue + +import ( + "context" + "encoding/json" + "reflect" + "testing" + "time" + + "github.com/goforj/queue/internal/workflow" +) + +// TestWorkflowPublicModelsAreRootOwned prevents private engine types from leaking back into the public API. +func TestWorkflowPublicModelsAreRootOwned(t *testing.T) { + types := []reflect.Type{ + reflect.TypeOf(Message{}), + reflect.TypeOf(DispatchResult{}), + reflect.TypeOf(StoredJobOptions{}), + reflect.TypeOf(StoredJob{}), + reflect.TypeOf(ChainNode{}), + reflect.TypeOf(ChainRecord{}), + reflect.TypeOf(ChainState{}), + reflect.TypeOf(BatchJob{}), + reflect.TypeOf(BatchJobOutcome("")), + reflect.TypeOf(BatchRecord{}), + reflect.TypeOf(BatchState{}), + reflect.TypeOf(SQLStoreConfig{}), + reflect.TypeOf(RetryPolicy{}), + reflect.TypeOf(SkipWhen{}), + reflect.TypeOf(FailOnError{}), + reflect.TypeOf(RateLimit{}), + reflect.TypeOf(WithoutOverlapping{}), + reflect.TypeOf((*Next)(nil)).Elem(), + reflect.TypeOf((*Middleware)(nil)).Elem(), + reflect.TypeOf((*MiddlewareFunc)(nil)).Elem(), + reflect.TypeOf((*RateLimiter)(nil)).Elem(), + reflect.TypeOf((*Lock)(nil)).Elem(), + reflect.TypeOf((*Locker)(nil)).Elem(), + reflect.TypeOf((*WorkflowStore)(nil)).Elem(), + reflect.TypeOf((*WorkflowOutcomeStore)(nil)).Elem(), + } + for _, modelType := range types { + if got := modelType.PkgPath(); got != "github.com/goforj/queue" { + t.Errorf("%s package = %q, want root queue package", modelType, got) + } + } +} + +// TestNewMessageCopiesPayload verifies both constructor and accessor isolation across adapter boundaries. +func TestNewMessageCopiesPayload(t *testing.T) { + payload := []byte(`{"id":7}`) + message := NewMessage("emails:send", payload) + payload[0] = 'X' + + var decoded struct { + ID int `json:"id"` + } + if err := message.Bind(&decoded); err != nil { + t.Fatalf("bind message: %v", err) + } + if decoded.ID != 7 { + t.Fatalf("bound id = %d, want 7", decoded.ID) + } + + returned := message.PayloadBytes() + returned[0] = 'Y' + if got := string(message.PayloadBytes()); got != `{"id":7}` { + t.Fatalf("message payload = %s, want isolated original", got) + } +} + +// TestMessageWorkflowRoundTrip preserves metadata and payload while crossing the private engine boundary. +func TestMessageWorkflowRoundTrip(t *testing.T) { + message := NewMessage("reports:build", []byte(`{"month":7}`)) + message.SchemaVersion = 1 + message.DispatchID = "dsp_1" + message.JobID = "job_1" + message.ChainID = "chn_1" + message.BatchID = "bat_1" + message.Attempt = 3 + + roundTrip := messageFromWorkflow(messageToWorkflow(message)) + if !reflect.DeepEqual(roundTrip, message) { + t.Fatalf("message round trip = %+v, want %+v", roundTrip, message) + } +} + +// TestStoredJobJSONV1 pins the root-owned model to the established workflow wire representation. +func TestStoredJobJSONV1(t *testing.T) { + encoded, err := json.Marshal(StoredJob{ + Type: "reports:build", + Payload: []byte(`{"month":7}`), + Options: StoredJobOptions{ + Queue: "critical", + Delay: 2 * time.Second, + Timeout: 3 * time.Second, + Retry: 4, + Backoff: 500 * time.Millisecond, + UniqueFor: 30 * time.Second, + }, + }) + if err != nil { + t.Fatalf("marshal stored job: %v", err) + } + want := `{"type":"reports:build","payload":"eyJtb250aCI6N30=","options":{"Queue":"critical","Delay":2000000000,"Timeout":3000000000,"Retry":4,"Backoff":500000000,"UniqueFor":30000000000}}` + if got := string(encoded); got != want { + t.Fatalf("stored job JSON = %s, want %s", got, want) + } +} + +// TestWorkflowMiddlewareAdapterPreservesMessageReplacement proves public middleware remains free to replace message values. +func TestWorkflowMiddlewareAdapterPreservesMessageReplacement(t *testing.T) { + adapter := workflowMiddlewareAdapter{middleware: MiddlewareFunc(func(ctx context.Context, message Message, next Next) error { + replacement := NewMessage("replacement:type", []byte(`{"replacement":true}`)) + replacement.SchemaVersion = message.SchemaVersion + replacement.DispatchID = message.DispatchID + replacement.JobID = message.JobID + replacement.ChainID = message.ChainID + replacement.BatchID = message.BatchID + replacement.Attempt = message.Attempt + 1 + return next(ctx, replacement) + })} + + input := workflow.NewContext(1, "dsp_1", "job_1", "chn_1", "bat_1", 2, "original:type", []byte(`{"original":true}`)) + var received workflow.Context + if err := adapter.Handle(context.Background(), input, func(_ context.Context, message workflow.Context) error { + received = message + return nil + }); err != nil { + t.Fatalf("run middleware adapter: %v", err) + } + + if received.JobType != "replacement:type" || received.Attempt != 3 { + t.Fatalf("received metadata = %+v, want replacement type and incremented attempt", received) + } + if got := string(received.PayloadBytes()); got != `{"replacement":true}` { + t.Fatalf("received payload = %s, want replacement payload", got) + } +} diff --git a/workflow_retry_contract_test.go b/workflow_retry_contract_test.go new file mode 100644 index 0000000..bf5add3 --- /dev/null +++ b/workflow_retry_contract_test.go @@ -0,0 +1,274 @@ +package queue_test + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/goforj/queue" +) + +// retryEventRecorder keeps public retry assertions safe if a backend invokes observers concurrently. +type retryEventRecorder struct { + mu sync.Mutex + events []queue.Event +} + +// Observe records one unified event. +func (r *retryEventRecorder) Observe(_ context.Context, event queue.Event) { + r.mu.Lock() + r.events = append(r.events, event) + r.mu.Unlock() +} + +// snapshot returns an isolated copy so assertions cannot race an observer call. +func (r *retryEventRecorder) snapshot() []queue.Event { + r.mu.Lock() + defer r.mu.Unlock() + return append([]queue.Event(nil), r.events...) +} + +// TestPublicChainWaitsForTerminalAttempt verifies transient delivery failures cannot terminally fail a workflow. +func TestPublicChainWaitsForTerminalAttempt(t *testing.T) { + recorder := &retryEventRecorder{} + q, err := queue.NewSync(queue.WithObserver(recorder)) + if err != nil { + t.Fatalf("new sync queue: %v", err) + } + var attempts []int + q.Register("contract:chain:retry", func(_ context.Context, message queue.Message) error { + attempts = append(attempts, message.Attempt) + if message.Attempt == 0 { + return errors.New("transient") + } + return nil + }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + t.Cleanup(func() { _ = q.Shutdown(context.Background()) }) + + catchCalls := 0 + finallyCalls := 0 + chainID, err := q.Chain(queue.NewJob("contract:chain:retry").Retry(1)). + Catch(func(context.Context, queue.ChainState, error) error { + catchCalls++ + return nil + }). + Finally(func(context.Context, queue.ChainState) error { + finallyCalls++ + return nil + }). + Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch retrying chain: %v", err) + } + state, err := q.FindChain(context.Background(), chainID) + if err != nil { + t.Fatalf("find chain: %v", err) + } + if !state.Completed || state.Failed || catchCalls != 0 || finallyCalls != 1 { + t.Fatalf("chain state/callbacks = %+v catch:%d finally:%d", state, catchCalls, finallyCalls) + } + assertRetryAttempts(t, attempts) + assertTransientWorkflowEvents(t, recorder.snapshot(), queue.EventChainFailed) +} + +// TestPublicBatchWaitsForTerminalAttempt verifies a retrying batch item is counted only after its final outcome. +func TestPublicBatchWaitsForTerminalAttempt(t *testing.T) { + recorder := &retryEventRecorder{} + q, err := queue.NewSync(queue.WithObserver(recorder)) + if err != nil { + t.Fatalf("new sync queue: %v", err) + } + var attempts []int + q.Register("contract:batch:retry", func(_ context.Context, message queue.Message) error { + attempts = append(attempts, message.Attempt) + if message.Attempt == 0 { + return errors.New("transient") + } + return nil + }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + t.Cleanup(func() { _ = q.Shutdown(context.Background()) }) + + catchCalls := 0 + finallyCalls := 0 + batchID, err := q.Batch(queue.NewJob("contract:batch:retry").Retry(1)). + Catch(func(context.Context, queue.BatchState, error) error { + catchCalls++ + return nil + }). + Finally(func(context.Context, queue.BatchState) error { + finallyCalls++ + return nil + }). + Dispatch(context.Background()) + if err != nil { + t.Fatalf("dispatch retrying batch: %v", err) + } + state, err := q.FindBatch(context.Background(), batchID) + if err != nil { + t.Fatalf("find batch: %v", err) + } + if !state.Completed || state.Cancelled || state.Failed != 0 || state.Processed != 1 || catchCalls != 0 || finallyCalls != 1 { + t.Fatalf("batch state/callbacks = %+v catch:%d finally:%d", state, catchCalls, finallyCalls) + } + assertRetryAttempts(t, attempts) + assertTransientWorkflowEvents(t, recorder.snapshot(), queue.EventBatchFailed) +} + +// TestPublicChainFailsOnlyAfterRetryExhaustion verifies terminal state and callbacks commit exactly once. +func TestPublicChainFailsOnlyAfterRetryExhaustion(t *testing.T) { + recorder := &retryEventRecorder{} + q, err := queue.NewSync(queue.WithObserver(recorder)) + if err != nil { + t.Fatalf("new sync queue: %v", err) + } + var attempts []int + q.Register("contract:chain:exhaust", func(_ context.Context, message queue.Message) error { + attempts = append(attempts, message.Attempt) + return errors.New("terminal") + }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + t.Cleanup(func() { _ = q.Shutdown(context.Background()) }) + + catchCalls := 0 + finallyCalls := 0 + chainID, dispatchErr := q.Chain(queue.NewJob("contract:chain:exhaust").Retry(1)). + Catch(func(context.Context, queue.ChainState, error) error { + catchCalls++ + return nil + }). + Finally(func(context.Context, queue.ChainState) error { + finallyCalls++ + return nil + }). + Dispatch(context.Background()) + if dispatchErr == nil { + t.Fatal("exhausted chain dispatch must return the handler error") + } + state, err := q.FindChain(context.Background(), chainID) + if err != nil { + t.Fatalf("find chain: %v", err) + } + if !state.Failed || state.Completed || catchCalls != 1 || finallyCalls != 1 { + t.Fatalf("chain state/callbacks = %+v catch:%d finally:%d", state, catchCalls, finallyCalls) + } + assertRetryAttempts(t, attempts) + events := recorder.snapshot() + if countRetryEvents(events, queue.EventJobFailed) != 1 || countRetryEvents(events, queue.EventChainFailed) != 1 { + t.Fatalf("terminal events must occur once: %+v", events) + } +} + +// TestPublicChainDownstreamFailureDoesNotRetryPredecessor verifies a synchronous +// downstream failure cannot consume the retry budget of an already committed node. +func TestPublicChainDownstreamFailureDoesNotRetryPredecessor(t *testing.T) { + recorder := &retryEventRecorder{} + q, err := queue.NewSync(queue.WithObserver(recorder)) + if err != nil { + t.Fatalf("new sync queue: %v", err) + } + + predecessorCalls := 0 + q.Register("contract:chain:predecessor", func(context.Context, queue.Message) error { + predecessorCalls++ + return nil + }) + downstreamErr := errors.New("downstream terminal failure") + downstreamCalls := 0 + q.Register("contract:chain:downstream", func(context.Context, queue.Message) error { + downstreamCalls++ + return downstreamErr + }) + if err := q.StartWorkers(context.Background()); err != nil { + t.Fatalf("start workers: %v", err) + } + t.Cleanup(func() { _ = q.Shutdown(context.Background()) }) + + chainID, dispatchErr := q.Chain( + queue.NewJob("contract:chain:predecessor").Retry(1), + queue.NewJob("contract:chain:downstream").Retry(0), + ).Dispatch(context.Background()) + if !errors.Is(dispatchErr, downstreamErr) { + t.Fatalf("dispatch error = %v, want downstream error", dispatchErr) + } + if predecessorCalls != 1 || downstreamCalls != 1 { + t.Fatalf("handler calls = predecessor:%d downstream:%d, want 1 each", predecessorCalls, downstreamCalls) + } + + state, err := q.FindChain(context.Background(), chainID) + if err != nil { + t.Fatalf("find chain: %v", err) + } + if !state.Failed || state.Completed { + t.Fatalf("chain state = %+v, want failed and never completed", state) + } + events := recorder.snapshot() + if countRetryEvents(events, queue.EventChainFailed) != 1 || countRetryEvents(events, queue.EventChainCompleted) != 0 { + t.Fatalf("terminal chain events are inconsistent: %+v", events) + } + if countRetryEvents(events, queue.EventChainAdvanced) != 1 { + t.Fatalf("chain advance count is not one: %+v", events) + } + if countRetryJobEvents(events, queue.EventJobSucceeded, "contract:chain:predecessor") != 1 || + countRetryJobEvents(events, queue.EventJobFailed, "contract:chain:predecessor") != 0 || + countRetryJobEvents(events, queue.EventJobSucceeded, "contract:chain:downstream") != 0 || + countRetryJobEvents(events, queue.EventJobFailed, "contract:chain:downstream") != 1 { + t.Fatalf("job outcome events are inconsistent: %+v", events) + } +} + +// assertRetryAttempts verifies messages expose the physical zero-based attempt sequence. +func assertRetryAttempts(t *testing.T, attempts []int) { + t.Helper() + if len(attempts) != 2 || attempts[0] != 0 || attempts[1] != 1 { + t.Fatalf("attempts = %v, want [0 1]", attempts) + } +} + +// assertTransientWorkflowEvents verifies a transient failure stays below the terminal workflow boundary. +func assertTransientWorkflowEvents(t *testing.T, events []queue.Event, terminalKind queue.EventKind) { + t.Helper() + if countRetryEvents(events, queue.EventProcessFailed) != 1 { + t.Fatalf("process failure count is not one: %+v", events) + } + if countRetryEvents(events, queue.EventJobFailed) != 0 || countRetryEvents(events, terminalKind) != 0 { + t.Fatalf("transient attempt emitted terminal workflow facts: %+v", events) + } + jobAttempts := make([]int, 0, 2) + for _, event := range events { + if event.Kind == queue.EventJobStarted { + jobAttempts = append(jobAttempts, event.Attempt) + } + } + assertRetryAttempts(t, jobAttempts) +} + +// countRetryEvents returns the number of matching unified facts. +func countRetryEvents(events []queue.Event, kind queue.EventKind) int { + count := 0 + for _, event := range events { + if event.Kind == kind { + count++ + } + } + return count +} + +// countRetryJobEvents returns the number of matching facts for one logical job type. +func countRetryJobEvents(events []queue.Event, kind queue.EventKind, jobType string) int { + count := 0 + for _, event := range events { + if event.Kind == kind && event.JobType == jobType { + count++ + } + } + return count +} diff --git a/workflow_store.go b/workflow_store.go new file mode 100644 index 0000000..87ac263 --- /dev/null +++ b/workflow_store.go @@ -0,0 +1,286 @@ +package queue + +import ( + "context" + "database/sql" + "errors" + "time" + + "github.com/goforj/queue/internal/workflow" +) + +// SQLStoreConfig configures connection ownership, dialect binding, and schema setup for a SQL workflow store. +// @group Queue +type SQLStoreConfig struct { + DB *sql.DB + DriverName string + DSN string + // AutoMigrate is retained for source compatibility. NewSQLStore keeps + // startup schema creation enabled regardless of the false zero value; use + // NewSQLStoreWithManagedSchema when deployment tooling owns the schema. + AutoMigrate bool +} + +// ErrWorkflowNotFound indicates a workflow state record is not present. +// @group Queue +var ErrWorkflowNotFound = workflow.ErrNotFound + +// NewMemoryStore creates an in-memory workflow state store. It copies chain +// nodes and payload bytes on creation and return so callers retain independent ownership. +// @group Constructors +func NewMemoryStore() WorkflowStore { + return &workflowStoreView{store: workflow.NewMemoryStore()} +} + +// NewSQLStore creates a SQL-backed workflow state store. +// @group Constructors +func NewSQLStore(config SQLStoreConfig) (WorkflowStore, error) { + return wrapWorkflowSQLStore(workflow.NewSQLStore(workflowSQLStoreConfig(config))) +} + +// NewSQLStoreWithManagedSchema creates a SQL-backed workflow state store +// without executing schema DDL. The supplied database must already contain the +// dialect-correct workflow tables, including transition receipts. +// @group Constructors +func NewSQLStoreWithManagedSchema(config SQLStoreConfig) (WorkflowStore, error) { + return wrapWorkflowSQLStore(workflow.NewSQLStoreWithManagedSchema(workflowSQLStoreConfig(config))) +} + +// workflowSQLStoreConfig converts the public connection settings without +// changing the compatibility-preserved config shape. +func workflowSQLStoreConfig(config SQLStoreConfig) workflow.SQLStoreConfig { + return workflow.SQLStoreConfig{ + DB: config.DB, + DriverName: config.DriverName, + DSN: config.DSN, + AutoMigrate: config.AutoMigrate, + } +} + +// wrapWorkflowSQLStore exposes a successfully constructed engine store through +// the root workflow model while preserving constructor errors unchanged. +func wrapWorkflowSQLStore(store workflow.Store, err error) (WorkflowStore, error) { + if err != nil { + return nil, err + } + return &workflowStoreView{store: store}, nil +} + +// workflowStoreProvider identifies built-in root stores whose engine implementation can be reused directly. +type workflowStoreProvider interface { + workflowStore() workflow.Store +} + +// workflowStoreView exposes an internal built-in store through root-owned workflow models. +type workflowStoreView struct { + store workflow.Store +} + +var _ WorkflowStore = (*workflowStoreView)(nil) +var _ WorkflowOutcomeStore = (*workflowStoreView)(nil) + +// workflowStore returns the built-in engine store so Queue construction avoids a redundant adapter layer. +func (s *workflowStoreView) workflowStore() workflow.Store { + return s.store +} + +// CreateChain persists a root-owned chain record through the built-in store. +func (s *workflowStoreView) CreateChain(ctx context.Context, record ChainRecord) error { + return s.store.CreateChain(ctx, chainRecordToWorkflow(record)) +} + +// AdvanceChain commits a chain node and converts any returned successor into the root model. +func (s *workflowStoreView) AdvanceChain(ctx context.Context, chainID string, completedNode string) (*ChainNode, bool, error) { + next, done, err := s.store.AdvanceChain(ctx, chainID, completedNode) + if next == nil { + return nil, done, err + } + converted := chainNodeFromWorkflow(*next) + return &converted, done, err +} + +// FailChain commits a terminal chain failure through the built-in store. +func (s *workflowStoreView) FailChain(ctx context.Context, chainID string, cause error) error { + return s.store.FailChain(ctx, chainID, cause) +} + +// FailChainNode exposes the built-in store's atomic per-node failure ownership. +func (s *workflowStoreView) FailChainNode(ctx context.Context, chainID, nodeID string, cause error) (ChainState, bool, error) { + store, ok := s.store.(interface { + FailChainNode(context.Context, string, string, error) (workflow.ChainState, bool, error) + }) + if !ok { + return ChainState{}, false, errors.New("workflow store does not support atomic chain-node failure") + } + state, owned, err := store.FailChainNode(ctx, chainID, nodeID, cause) + return chainStateFromWorkflow(state), owned, err +} + +// SettleBatchJob exposes the built-in store's first-writer member outcome. +func (s *workflowStoreView) SettleBatchJob(ctx context.Context, batchID, jobID string, outcome BatchJobOutcome, cause error) (BatchState, bool, error) { + store, ok := s.store.(interface { + SettleBatchJob(context.Context, string, string, workflow.BatchJobOutcome, error) (workflow.BatchState, bool, error) + }) + if !ok { + return BatchState{}, false, errors.New("workflow store does not support atomic batch-job outcomes") + } + state, owned, err := store.SettleBatchJob(ctx, batchID, jobID, workflow.BatchJobOutcome(outcome), cause) + return batchStateFromWorkflow(state), owned, err +} + +// GetChain reads and converts current chain state from the built-in store. +func (s *workflowStoreView) GetChain(ctx context.Context, chainID string) (ChainState, error) { + state, err := s.store.GetChain(ctx, chainID) + return chainStateFromWorkflow(state), err +} + +// CreateBatch persists a root-owned batch record through the built-in store. +func (s *workflowStoreView) CreateBatch(ctx context.Context, record BatchRecord) error { + return s.store.CreateBatch(ctx, batchRecordToWorkflow(record)) +} + +// MarkBatchJobStarted records a started member through the built-in store. +func (s *workflowStoreView) MarkBatchJobStarted(ctx context.Context, batchID, jobID string) error { + return s.store.MarkBatchJobStarted(ctx, batchID, jobID) +} + +// MarkBatchJobSucceeded commits a successful member and converts aggregate state. +func (s *workflowStoreView) MarkBatchJobSucceeded(ctx context.Context, batchID, jobID string) (BatchState, bool, error) { + state, done, err := s.store.MarkBatchJobSucceeded(ctx, batchID, jobID) + return batchStateFromWorkflow(state), done, err +} + +// MarkBatchJobFailed commits a failed member and converts aggregate state. +func (s *workflowStoreView) MarkBatchJobFailed(ctx context.Context, batchID, jobID string, cause error) (BatchState, bool, error) { + state, done, err := s.store.MarkBatchJobFailed(ctx, batchID, jobID, cause) + return batchStateFromWorkflow(state), done, err +} + +// CancelBatch commits aggregate cancellation through the built-in store. +func (s *workflowStoreView) CancelBatch(ctx context.Context, batchID string) error { + return s.store.CancelBatch(ctx, batchID) +} + +// GetBatch reads and converts current aggregate state from the built-in store. +func (s *workflowStoreView) GetBatch(ctx context.Context, batchID string) (BatchState, error) { + state, err := s.store.GetBatch(ctx, batchID) + return batchStateFromWorkflow(state), err +} + +// MarkCallbackInvoked claims a callback idempotency key through the built-in store. +func (s *workflowStoreView) MarkCallbackInvoked(ctx context.Context, key string) (bool, error) { + return s.store.MarkCallbackInvoked(ctx, key) +} + +// Prune removes terminal state older than before through the built-in store. +func (s *workflowStoreView) Prune(ctx context.Context, before time.Time) error { + return s.store.Prune(ctx, before) +} + +// rootWorkflowStoreAdapter presents an application-defined root store to the private engine. +type rootWorkflowStoreAdapter struct { + store WorkflowStore +} + +var _ workflow.Store = rootWorkflowStoreAdapter{} + +type rootWorkflowOutcomeStoreAdapter struct { + rootWorkflowStoreAdapter + atomic WorkflowOutcomeStore +} + +// FailChainNode converts an atomic custom-store result back into the engine model. +func (a rootWorkflowOutcomeStoreAdapter) FailChainNode(ctx context.Context, chainID, nodeID string, cause error) (workflow.ChainState, bool, error) { + state, owned, err := a.atomic.FailChainNode(ctx, chainID, nodeID, cause) + return chainStateToWorkflow(state), owned, err +} + +// SettleBatchJob converts an atomic custom-store result back into the engine model. +func (a rootWorkflowOutcomeStoreAdapter) SettleBatchJob(ctx context.Context, batchID, jobID string, outcome workflow.BatchJobOutcome, cause error) (workflow.BatchState, bool, error) { + state, owned, err := a.atomic.SettleBatchJob(ctx, batchID, jobID, BatchJobOutcome(outcome), cause) + return batchStateToWorkflow(state), owned, err +} + +// CreateChain converts the engine record before invoking the application store. +func (a rootWorkflowStoreAdapter) CreateChain(ctx context.Context, record workflow.ChainRecord) error { + return a.store.CreateChain(ctx, chainRecordFromWorkflow(record)) +} + +// AdvanceChain converts the application store's successor back into the engine model. +func (a rootWorkflowStoreAdapter) AdvanceChain(ctx context.Context, chainID string, completedNode string) (*workflow.ChainNode, bool, error) { + next, done, err := a.store.AdvanceChain(ctx, chainID, completedNode) + if next == nil { + return nil, done, err + } + converted := chainNodeToWorkflow(*next) + return &converted, done, err +} + +// FailChain forwards a terminal failure without changing its error chain. +func (a rootWorkflowStoreAdapter) FailChain(ctx context.Context, chainID string, cause error) error { + return a.store.FailChain(ctx, chainID, cause) +} + +// GetChain converts application-owned chain state back into the engine model. +func (a rootWorkflowStoreAdapter) GetChain(ctx context.Context, chainID string) (workflow.ChainState, error) { + state, err := a.store.GetChain(ctx, chainID) + return chainStateToWorkflow(state), err +} + +// CreateBatch converts the engine record before invoking the application store. +func (a rootWorkflowStoreAdapter) CreateBatch(ctx context.Context, record workflow.BatchRecord) error { + return a.store.CreateBatch(ctx, batchRecordFromWorkflow(record)) +} + +// MarkBatchJobStarted forwards a started marker to the application store. +func (a rootWorkflowStoreAdapter) MarkBatchJobStarted(ctx context.Context, batchID, jobID string) error { + return a.store.MarkBatchJobStarted(ctx, batchID, jobID) +} + +// MarkBatchJobSucceeded converts application aggregate state back into the engine model. +func (a rootWorkflowStoreAdapter) MarkBatchJobSucceeded(ctx context.Context, batchID, jobID string) (workflow.BatchState, bool, error) { + state, done, err := a.store.MarkBatchJobSucceeded(ctx, batchID, jobID) + return batchStateToWorkflow(state), done, err +} + +// MarkBatchJobFailed converts application aggregate state back into the engine model. +func (a rootWorkflowStoreAdapter) MarkBatchJobFailed(ctx context.Context, batchID, jobID string, cause error) (workflow.BatchState, bool, error) { + state, done, err := a.store.MarkBatchJobFailed(ctx, batchID, jobID, cause) + return batchStateToWorkflow(state), done, err +} + +// CancelBatch forwards aggregate cancellation to the application store. +func (a rootWorkflowStoreAdapter) CancelBatch(ctx context.Context, batchID string) error { + return a.store.CancelBatch(ctx, batchID) +} + +// GetBatch converts application-owned aggregate state back into the engine model. +func (a rootWorkflowStoreAdapter) GetBatch(ctx context.Context, batchID string) (workflow.BatchState, error) { + state, err := a.store.GetBatch(ctx, batchID) + return batchStateToWorkflow(state), err +} + +// MarkCallbackInvoked forwards callback idempotency claims to the application store. +func (a rootWorkflowStoreAdapter) MarkCallbackInvoked(ctx context.Context, key string) (bool, error) { + return a.store.MarkCallbackInvoked(ctx, key) +} + +// Prune forwards terminal-state retention to the application store. +func (a rootWorkflowStoreAdapter) Prune(ctx context.Context, before time.Time) error { + return a.store.Prune(ctx, before) +} + +// workflowStoreFromRoot unwraps built-ins and adapts application-defined stores exactly once. +func workflowStoreFromRoot(store WorkflowStore) workflow.Store { + if store == nil { + return nil + } + if provider, ok := store.(workflowStoreProvider); ok { + return provider.workflowStore() + } + adapter := rootWorkflowStoreAdapter{store: store} + if atomic, ok := store.(WorkflowOutcomeStore); ok { + return rootWorkflowOutcomeStoreAdapter{rootWorkflowStoreAdapter: adapter, atomic: atomic} + } + return adapter +} diff --git a/workflow_store_adapter_test.go b/workflow_store_adapter_test.go new file mode 100644 index 0000000..fa19bc2 --- /dev/null +++ b/workflow_store_adapter_test.go @@ -0,0 +1,511 @@ +package queue + +import ( + "context" + "errors" + "reflect" + "testing" + "time" + + "github.com/goforj/queue/internal/workflow" +) + +type workflowStoreAdapterContextKey struct{} + +type workflowStoreAdapterSpy struct { + wantContext context.Context + contextsOK bool + + createChainRecord ChainRecord + createChainErr error + + advanceChainID string + advanceCompletedNode string + advanceNext *ChainNode + advanceDone bool + advanceErr error + + failChainID string + failChainCause error + failChainErr error + + getChainID string + getChainState ChainState + getChainErr error + + createBatchRecord BatchRecord + createBatchErr error + + startedBatchID string + startedJobID string + startedErr error + + succeededBatchID string + succeededJobID string + succeededState BatchState + succeededDone bool + succeededErr error + + failedBatchID string + failedJobID string + failedCause error + failedState BatchState + failedDone bool + failedErr error + + cancelBatchID string + cancelErr error + + getBatchID string + getBatchState BatchState + getBatchErr error + + callbackKey string + callbackClaimed bool + callbackErr error + + pruneBefore time.Time + pruneErr error +} + +type workflowOutcomeStoreSpy struct { + *workflowStoreAdapterSpy + failNodeChainID string + failNodeID string + failNodeCause error + failNodeState ChainState + failNodeOwned bool + failNodeErr error + settleBatchID string + settleJobID string + settleOutcome BatchJobOutcome + settleCause error + settleState BatchState + settleOwned bool + settleErr error +} + +// FailChainNode records the additive atomic chain transition. +func (s *workflowOutcomeStoreSpy) FailChainNode(ctx context.Context, chainID, nodeID string, cause error) (ChainState, bool, error) { + s.recordContext(ctx) + s.failNodeChainID = chainID + s.failNodeID = nodeID + s.failNodeCause = cause + return s.failNodeState, s.failNodeOwned, s.failNodeErr +} + +// SettleBatchJob records the additive atomic member transition. +func (s *workflowOutcomeStoreSpy) SettleBatchJob(ctx context.Context, batchID, jobID string, outcome BatchJobOutcome, cause error) (BatchState, bool, error) { + s.recordContext(ctx) + s.settleBatchID = batchID + s.settleJobID = jobID + s.settleOutcome = outcome + s.settleCause = cause + return s.settleState, s.settleOwned, s.settleErr +} + +// recordContext verifies that adapters forward the caller's context without replacement. +func (s *workflowStoreAdapterSpy) recordContext(ctx context.Context) { + s.contextsOK = s.contextsOK && ctx == s.wantContext +} + +// CreateChain records the converted root chain creation model. +func (s *workflowStoreAdapterSpy) CreateChain(ctx context.Context, record ChainRecord) error { + s.recordContext(ctx) + s.createChainRecord = record + return s.createChainErr +} + +// AdvanceChain records transition arguments and returns the configured physical successor. +func (s *workflowStoreAdapterSpy) AdvanceChain(ctx context.Context, chainID string, completedNode string) (*ChainNode, bool, error) { + s.recordContext(ctx) + s.advanceChainID = chainID + s.advanceCompletedNode = completedNode + return s.advanceNext, s.advanceDone, s.advanceErr +} + +// FailChain records the terminal chain failure without wrapping its cause. +func (s *workflowStoreAdapterSpy) FailChain(ctx context.Context, chainID string, cause error) error { + s.recordContext(ctx) + s.failChainID = chainID + s.failChainCause = cause + return s.failChainErr +} + +// GetChain records the lookup and returns configured physical chain state. +func (s *workflowStoreAdapterSpy) GetChain(ctx context.Context, chainID string) (ChainState, error) { + s.recordContext(ctx) + s.getChainID = chainID + return s.getChainState, s.getChainErr +} + +// CreateBatch records the converted root batch creation model. +func (s *workflowStoreAdapterSpy) CreateBatch(ctx context.Context, record BatchRecord) error { + s.recordContext(ctx) + s.createBatchRecord = record + return s.createBatchErr +} + +// MarkBatchJobStarted records the member start transition. +func (s *workflowStoreAdapterSpy) MarkBatchJobStarted(ctx context.Context, batchID, jobID string) error { + s.recordContext(ctx) + s.startedBatchID = batchID + s.startedJobID = jobID + return s.startedErr +} + +// MarkBatchJobSucceeded records the member success transition and returns configured aggregate state. +func (s *workflowStoreAdapterSpy) MarkBatchJobSucceeded(ctx context.Context, batchID, jobID string) (BatchState, bool, error) { + s.recordContext(ctx) + s.succeededBatchID = batchID + s.succeededJobID = jobID + return s.succeededState, s.succeededDone, s.succeededErr +} + +// MarkBatchJobFailed records the member failure transition and returns configured aggregate state. +func (s *workflowStoreAdapterSpy) MarkBatchJobFailed(ctx context.Context, batchID, jobID string, cause error) (BatchState, bool, error) { + s.recordContext(ctx) + s.failedBatchID = batchID + s.failedJobID = jobID + s.failedCause = cause + return s.failedState, s.failedDone, s.failedErr +} + +// CancelBatch records aggregate cancellation. +func (s *workflowStoreAdapterSpy) CancelBatch(ctx context.Context, batchID string) error { + s.recordContext(ctx) + s.cancelBatchID = batchID + return s.cancelErr +} + +// GetBatch records the lookup and returns configured physical aggregate state. +func (s *workflowStoreAdapterSpy) GetBatch(ctx context.Context, batchID string) (BatchState, error) { + s.recordContext(ctx) + s.getBatchID = batchID + return s.getBatchState, s.getBatchErr +} + +// MarkCallbackInvoked records the idempotency claim and returns its configured outcome. +func (s *workflowStoreAdapterSpy) MarkCallbackInvoked(ctx context.Context, key string) (bool, error) { + s.recordContext(ctx) + s.callbackKey = key + return s.callbackClaimed, s.callbackErr +} + +// Prune records the exact retention boundary. +func (s *workflowStoreAdapterSpy) Prune(ctx context.Context, before time.Time) error { + s.recordContext(ctx) + s.pruneBefore = before + return s.pruneErr +} + +// TestRootWorkflowStoreAdapterConvertsEveryMethod pins the complete custom-store boundary in both directions. +func TestRootWorkflowStoreAdapterConvertsEveryMethod(t *testing.T) { + ctx := context.WithValue(context.Background(), workflowStoreAdapterContextKey{}, "adapter-test") + createdAt := time.Date(2026, time.July, 18, 10, 11, 12, 13, time.UTC) + updatedAt := createdAt.Add(time.Minute) + createChainErr := errors.New("create chain failed") + advanceErr := errors.New("advance chain failed") + failCause := errors.New("chain handler failed") + failChainErr := errors.New("fail chain failed") + getChainErr := errors.New("get chain failed") + createBatchErr := errors.New("create batch failed") + startedErr := errors.New("start batch job failed") + succeededErr := errors.New("succeed batch job failed") + memberCause := errors.New("batch member failed") + failedErr := errors.New("fail batch job failed") + cancelErr := errors.New("cancel batch failed") + getBatchErr := errors.New("get batch failed") + callbackErr := errors.New("callback claim failed") + pruneErr := errors.New("prune failed") + + chainState := ChainState{ + ChainID: "chain-state", + DispatchID: "dispatch-state", + Queue: "critical", + Nodes: []ChainNode{{ + NodeID: "node-state", + Job: StoredJob{ + Type: "reports:state", + Payload: []byte(`{"state":true}`), + Options: StoredJobOptions{Queue: "critical", Delay: time.Second, Timeout: 2 * time.Second, Retry: 3, Backoff: 4 * time.Second, UniqueFor: 5 * time.Second}, + }, + }}, + NextIndex: 1, + Completed: false, + Failed: true, + Failure: "state failure", + CreatedAt: createdAt, + UpdatedAt: updatedAt, + } + batchState := BatchState{ + BatchID: "batch-state", + DispatchID: "dispatch-batch-state", + Name: "nightly", + Queue: "bulk", + AllowFailed: true, + Total: 5, + Pending: 2, + Processed: 3, + Failed: 1, + Cancelled: true, + Completed: false, + CreatedAt: createdAt, + UpdatedAt: updatedAt, + } + nextNode := &ChainNode{ + NodeID: "node-next", + Job: StoredJob{ + Type: "reports:next", + Payload: []byte(`{"next":true}`), + Options: StoredJobOptions{Queue: "critical", Retry: 7}, + }, + } + spy := &workflowStoreAdapterSpy{ + wantContext: ctx, + contextsOK: true, + createChainErr: createChainErr, + advanceNext: nextNode, + advanceDone: true, + advanceErr: advanceErr, + failChainErr: failChainErr, + getChainState: chainState, + getChainErr: getChainErr, + createBatchErr: createBatchErr, + startedErr: startedErr, + succeededState: batchState, + succeededDone: true, + succeededErr: succeededErr, + failedState: batchState, + failedDone: false, + failedErr: failedErr, + cancelErr: cancelErr, + getBatchState: batchState, + getBatchErr: getBatchErr, + callbackClaimed: true, + callbackErr: callbackErr, + pruneErr: pruneErr, + } + adapter := rootWorkflowStoreAdapter{store: spy} + + engineChainRecord := workflow.ChainRecord{ + ChainID: "chain-create", + DispatchID: "dispatch-create", + Queue: "critical", + Nodes: []workflow.ChainNode{{ + NodeID: "node-create", + Job: workflow.StoredJob{ + Type: "reports:create", + Payload: []byte(`{"create":true}`), + Options: workflow.JobOptions{Queue: "critical", Delay: time.Second, Timeout: 2 * time.Second, Retry: 3, Backoff: 4 * time.Second, UniqueFor: 5 * time.Second}, + }, + }}, + CreatedAt: createdAt, + } + wantRootChainRecord := ChainRecord{ + ChainID: "chain-create", + DispatchID: "dispatch-create", + Queue: "critical", + Nodes: []ChainNode{{ + NodeID: "node-create", + Job: StoredJob{ + Type: "reports:create", + Payload: []byte(`{"create":true}`), + Options: StoredJobOptions{Queue: "critical", Delay: time.Second, Timeout: 2 * time.Second, Retry: 3, Backoff: 4 * time.Second, UniqueFor: 5 * time.Second}, + }, + }}, + CreatedAt: createdAt, + } + if err := adapter.CreateChain(ctx, engineChainRecord); err != createChainErr { + t.Fatalf("CreateChain error = %v, want exact sentinel", err) + } + if !reflect.DeepEqual(spy.createChainRecord, wantRootChainRecord) { + t.Fatalf("CreateChain record = %+v, want %+v", spy.createChainRecord, wantRootChainRecord) + } + engineChainRecord.Nodes[0].Job.Payload[0] = '!' + if got := string(spy.createChainRecord.Nodes[0].Job.Payload); got != `{"create":true}` { + t.Fatalf("CreateChain payload aliased engine bytes: %q", got) + } + + next, done, err := adapter.AdvanceChain(ctx, "chain-advance", "node-complete") + if err != advanceErr || !done || spy.advanceChainID != "chain-advance" || spy.advanceCompletedNode != "node-complete" { + t.Fatalf("AdvanceChain result = next:%+v done:%t err:%v args:%q/%q", next, done, err, spy.advanceChainID, spy.advanceCompletedNode) + } + wantEngineNext := workflow.ChainNode{ + NodeID: "node-next", + Job: workflow.StoredJob{ + Type: "reports:next", + Payload: []byte(`{"next":true}`), + Options: workflow.JobOptions{Queue: "critical", Retry: 7}, + }, + } + if next == nil || !reflect.DeepEqual(*next, wantEngineNext) { + t.Fatalf("AdvanceChain next = %+v, want %+v", next, wantEngineNext) + } + next.Job.Payload[0] = '?' + if got := string(spy.advanceNext.Job.Payload); got != `{"next":true}` { + t.Fatalf("AdvanceChain output payload aliased physical bytes: %q", got) + } + spy.advanceNext = nil + spy.advanceDone = false + nilNext, nilDone, nilErr := adapter.AdvanceChain(ctx, "chain-nil", "node-nil") + if nilNext != nil || nilDone || nilErr != advanceErr { + t.Fatalf("AdvanceChain nil successor = next:%+v done:%t err:%v", nilNext, nilDone, nilErr) + } + + if err := adapter.FailChain(ctx, "chain-fail", failCause); err != failChainErr || spy.failChainID != "chain-fail" || spy.failChainCause != failCause { + t.Fatalf("FailChain result = err:%v id:%q cause:%v", err, spy.failChainID, spy.failChainCause) + } + gotChainState, err := adapter.GetChain(ctx, "chain-get") + if err != getChainErr || spy.getChainID != "chain-get" { + t.Fatalf("GetChain result = err:%v id:%q", err, spy.getChainID) + } + if gotChainState.ChainID != chainState.ChainID || gotChainState.DispatchID != chainState.DispatchID || gotChainState.Queue != chainState.Queue || gotChainState.NextIndex != chainState.NextIndex || gotChainState.Completed != chainState.Completed || gotChainState.Failed != chainState.Failed || gotChainState.Failure != chainState.Failure || !gotChainState.CreatedAt.Equal(chainState.CreatedAt) || !gotChainState.UpdatedAt.Equal(chainState.UpdatedAt) || len(gotChainState.Nodes) != 1 || gotChainState.Nodes[0].Job.Options.Retry != 3 { + t.Fatalf("GetChain converted state = %+v, want fields from %+v", gotChainState, chainState) + } + gotChainState.Nodes[0].Job.Payload[0] = '#' + if got := string(spy.getChainState.Nodes[0].Job.Payload); got != `{"state":true}` { + t.Fatalf("GetChain payload aliased physical bytes: %q", got) + } + + engineBatchRecord := workflow.BatchRecord{ + BatchID: "batch-create", + DispatchID: "dispatch-batch-create", + Name: "daily", + Queue: "bulk", + AllowFailed: true, + Jobs: []workflow.BatchJob{{ + JobID: "job-create", + Job: workflow.StoredJob{ + Type: "reports:batch", + Payload: []byte(`{"batch":true}`), + Options: workflow.JobOptions{Queue: "bulk", Delay: 6 * time.Second, Timeout: 7 * time.Second, Retry: 8, Backoff: 9 * time.Second, UniqueFor: 10 * time.Second}, + }, + }}, + CreatedAt: createdAt, + } + wantRootBatchRecord := BatchRecord{ + BatchID: "batch-create", + DispatchID: "dispatch-batch-create", + Name: "daily", + Queue: "bulk", + AllowFailed: true, + Jobs: []BatchJob{{ + JobID: "job-create", + Job: StoredJob{ + Type: "reports:batch", + Payload: []byte(`{"batch":true}`), + Options: StoredJobOptions{Queue: "bulk", Delay: 6 * time.Second, Timeout: 7 * time.Second, Retry: 8, Backoff: 9 * time.Second, UniqueFor: 10 * time.Second}, + }, + }}, + CreatedAt: createdAt, + } + if err := adapter.CreateBatch(ctx, engineBatchRecord); err != createBatchErr { + t.Fatalf("CreateBatch error = %v, want exact sentinel", err) + } + if !reflect.DeepEqual(spy.createBatchRecord, wantRootBatchRecord) { + t.Fatalf("CreateBatch record = %+v, want %+v", spy.createBatchRecord, wantRootBatchRecord) + } + engineBatchRecord.Jobs[0].Job.Payload[0] = '!' + if got := string(spy.createBatchRecord.Jobs[0].Job.Payload); got != `{"batch":true}` { + t.Fatalf("CreateBatch payload aliased engine bytes: %q", got) + } + + if err := adapter.MarkBatchJobStarted(ctx, "batch-start", "job-start"); err != startedErr || spy.startedBatchID != "batch-start" || spy.startedJobID != "job-start" { + t.Fatalf("MarkBatchJobStarted result = err:%v args:%q/%q", err, spy.startedBatchID, spy.startedJobID) + } + gotSucceeded, succeededDone, err := adapter.MarkBatchJobSucceeded(ctx, "batch-succeed", "job-succeed") + if err != succeededErr || !succeededDone || spy.succeededBatchID != "batch-succeed" || spy.succeededJobID != "job-succeed" { + t.Fatalf("MarkBatchJobSucceeded result = state:%+v done:%t err:%v args:%q/%q", gotSucceeded, succeededDone, err, spy.succeededBatchID, spy.succeededJobID) + } + assertWorkflowBatchStateMatchesRoot(t, gotSucceeded, batchState) + gotFailed, failedDone, err := adapter.MarkBatchJobFailed(ctx, "batch-fail", "job-fail", memberCause) + if err != failedErr || failedDone || spy.failedBatchID != "batch-fail" || spy.failedJobID != "job-fail" || spy.failedCause != memberCause { + t.Fatalf("MarkBatchJobFailed result = state:%+v done:%t err:%v args:%q/%q cause:%v", gotFailed, failedDone, err, spy.failedBatchID, spy.failedJobID, spy.failedCause) + } + assertWorkflowBatchStateMatchesRoot(t, gotFailed, batchState) + if err := adapter.CancelBatch(ctx, "batch-cancel"); err != cancelErr || spy.cancelBatchID != "batch-cancel" { + t.Fatalf("CancelBatch result = err:%v id:%q", err, spy.cancelBatchID) + } + gotBatch, err := adapter.GetBatch(ctx, "batch-get") + if err != getBatchErr || spy.getBatchID != "batch-get" { + t.Fatalf("GetBatch result = err:%v id:%q", err, spy.getBatchID) + } + assertWorkflowBatchStateMatchesRoot(t, gotBatch, batchState) + + claimed, err := adapter.MarkCallbackInvoked(ctx, "callback-key") + if err != callbackErr || !claimed || spy.callbackKey != "callback-key" { + t.Fatalf("MarkCallbackInvoked result = claimed:%t err:%v key:%q", claimed, err, spy.callbackKey) + } + before := updatedAt.Add(24 * time.Hour) + if err := adapter.Prune(ctx, before); err != pruneErr || !spy.pruneBefore.Equal(before) { + t.Fatalf("Prune result = err:%v before:%v", err, spy.pruneBefore) + } + if !spy.contextsOK { + t.Fatal("one or more custom store adapter methods replaced the caller context") + } +} + +// assertWorkflowBatchStateMatchesRoot verifies every aggregate field crosses the physical model boundary. +func assertWorkflowBatchStateMatchesRoot(t *testing.T, got workflow.BatchState, want BatchState) { + t.Helper() + if got.BatchID != want.BatchID || got.DispatchID != want.DispatchID || got.Name != want.Name || got.Queue != want.Queue || got.AllowFailed != want.AllowFailed || got.Total != want.Total || got.Pending != want.Pending || got.Processed != want.Processed || got.Failed != want.Failed || got.Cancelled != want.Cancelled || got.Completed != want.Completed || !got.CreatedAt.Equal(want.CreatedAt) || !got.UpdatedAt.Equal(want.UpdatedAt) { + t.Fatalf("workflow batch state = %+v, want fields from %+v", got, want) + } +} + +// TestWorkflowStoreFromRootSelectsOneBoundary verifies nil, built-in, and custom stores take their intended routes. +func TestWorkflowStoreFromRootSelectsOneBoundary(t *testing.T) { + if got := workflowStoreFromRoot(nil); got != nil { + t.Fatalf("nil root store adapted as %T, want nil", got) + } + + builtIn := NewMemoryStore() + provider, ok := builtIn.(workflowStoreProvider) + if !ok { + t.Fatalf("built-in store %T does not expose its engine store", builtIn) + } + if got := workflowStoreFromRoot(builtIn); got != provider.workflowStore() { + t.Fatalf("built-in store route = %T, want direct engine store %T", got, provider.workflowStore()) + } + + custom := &workflowStoreAdapterSpy{contextsOK: true} + got := workflowStoreFromRoot(custom) + adapter, ok := got.(rootWorkflowStoreAdapter) + if !ok { + t.Fatalf("custom store route = %T, want rootWorkflowStoreAdapter", got) + } + if adapter.store != custom { + t.Fatalf("custom adapter store = %T, want original %T", adapter.store, custom) + } + + ctx := context.WithValue(context.Background(), workflowStoreAdapterContextKey{}, "outcome-adapter") + chainCause := errors.New("chain outcome failed") + batchCause := errors.New("batch outcome failed") + outcomeErr := errors.New("outcome store failed") + capable := &workflowOutcomeStoreSpy{ + workflowStoreAdapterSpy: &workflowStoreAdapterSpy{wantContext: ctx, contextsOK: true}, + failNodeState: ChainState{ChainID: "chain-outcome", Failed: true}, + failNodeOwned: true, + failNodeErr: outcomeErr, + settleState: BatchState{BatchID: "batch-outcome", Processed: 1}, + settleOwned: false, + settleErr: outcomeErr, + } + adapted := workflowStoreFromRoot(capable) + outcomeAdapter, ok := adapted.(rootWorkflowOutcomeStoreAdapter) + if !ok { + t.Fatalf("capable custom store route = %T, want rootWorkflowOutcomeStoreAdapter", adapted) + } + chainState, owned, err := outcomeAdapter.FailChainNode(ctx, "chain-outcome", "node-outcome", chainCause) + if err != outcomeErr || !owned || chainState.ChainID != "chain-outcome" || capable.failNodeChainID != "chain-outcome" || capable.failNodeID != "node-outcome" || capable.failNodeCause != chainCause { + t.Fatalf("FailChainNode result = state:%+v owned:%t err:%v spy:%+v", chainState, owned, err, capable) + } + batchState, owned, err := outcomeAdapter.SettleBatchJob(ctx, "batch-outcome", "job-outcome", workflow.BatchJobFailed, batchCause) + if err != outcomeErr || owned || batchState.BatchID != "batch-outcome" || capable.settleBatchID != "batch-outcome" || capable.settleJobID != "job-outcome" || capable.settleOutcome != BatchJobFailed || capable.settleCause != batchCause { + t.Fatalf("SettleBatchJob result = state:%+v owned:%t err:%v spy:%+v", batchState, owned, err, capable) + } + if !capable.contextsOK { + t.Fatal("outcome adapter replaced the caller context") + } +} diff --git a/workflow_store_managed_schema_test.go b/workflow_store_managed_schema_test.go new file mode 100644 index 0000000..1e9c86b --- /dev/null +++ b/workflow_store_managed_schema_test.go @@ -0,0 +1,70 @@ +package queue + +import ( + "context" + "database/sql" + "errors" + "path/filepath" + "testing" + + _ "modernc.org/sqlite" +) + +// TestNewSQLStoreWithManagedSchemaDoesNotCreateTables proves the explicit +// caller-managed path performs no startup DDL, even on first use. +func TestNewSQLStoreWithManagedSchemaDoesNotCreateTables(t *testing.T) { + ctx := context.Background() + db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "managed-empty.db")) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + + store, err := NewSQLStoreWithManagedSchema(SQLStoreConfig{DB: db, DriverName: "sqlite"}) + if err != nil { + t.Fatalf("new managed-schema store: %v", err) + } + if _, err := store.GetChain(ctx, "missing"); err == nil || errors.Is(err, ErrWorkflowNotFound) { + t.Fatalf("unprovisioned managed schema error = %v", err) + } + var tableCount int + if err := db.QueryRowContext(ctx, `SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name LIKE 'bus_%'`).Scan(&tableCount); err != nil { + t.Fatalf("count workflow tables: %v", err) + } + if tableCount != 0 { + t.Fatalf("managed-schema constructor created %d workflow tables", tableCount) + } +} + +// TestNewSQLStoreWithManagedSchemaUsesProvisionedTables proves disabling DDL +// changes only schema ownership, not workflow-store behavior. +func TestNewSQLStoreWithManagedSchemaUsesProvisionedTables(t *testing.T) { + ctx := context.Background() + db, err := sql.Open("sqlite", filepath.Join(t.TempDir(), "managed-provisioned.db")) + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + t.Cleanup(func() { _ = db.Close() }) + + bootstrap, err := NewSQLStore(SQLStoreConfig{DB: db, DriverName: "sqlite"}) + if err != nil { + t.Fatalf("new bootstrap store: %v", err) + } + if _, err := bootstrap.GetChain(ctx, "missing"); !errors.Is(err, ErrWorkflowNotFound) { + t.Fatalf("bootstrap schema: %v", err) + } + managed, err := NewSQLStoreWithManagedSchema(SQLStoreConfig{DB: db, DriverName: "sqlite"}) + if err != nil { + t.Fatalf("new managed-schema store: %v", err) + } + if err := managed.CreateChain(ctx, ChainRecord{ChainID: "managed-chain", Nodes: []ChainNode{{NodeID: "managed-node"}}}); err != nil { + t.Fatalf("create chain in provisioned schema: %v", err) + } + state, err := managed.GetChain(ctx, "managed-chain") + if err != nil { + t.Fatalf("get chain from provisioned schema: %v", err) + } + if state.ChainID != "managed-chain" || len(state.Nodes) != 1 || state.Nodes[0].NodeID != "managed-node" { + t.Fatalf("managed-schema chain = %+v", state) + } +} diff --git a/workflow_store_view_test.go b/workflow_store_view_test.go new file mode 100644 index 0000000..b07ff2b --- /dev/null +++ b/workflow_store_view_test.go @@ -0,0 +1,229 @@ +package queue + +import ( + "context" + "errors" + "reflect" + "testing" + "time" +) + +// TestWorkflowStoreViewConvertsEveryMethod pins the complete internal-to-root built-in store boundary. +func TestWorkflowStoreViewConvertsEveryMethod(t *testing.T) { + ctx := context.WithValue(context.Background(), workflowStoreAdapterContextKey{}, "view-test") + createdAt := time.Date(2026, time.July, 18, 12, 13, 14, 15, time.UTC) + updatedAt := createdAt.Add(time.Minute) + storeErr := errors.New("internal store failed") + chainCause := errors.New("chain failed") + memberCause := errors.New("member failed") + + chainState := ChainState{ + ChainID: "chain-state", + DispatchID: "dispatch-chain-state", + Queue: "critical", + Nodes: []ChainNode{{ + NodeID: "node-state", + Job: StoredJob{ + Type: "reports:state", + Payload: []byte(`{"state":true}`), + Options: StoredJobOptions{Queue: "critical", Delay: time.Second, Timeout: 2 * time.Second, Retry: 3, Backoff: 4 * time.Second, UniqueFor: 5 * time.Second}, + }, + }}, + NextIndex: 1, + Completed: false, + Failed: true, + Failure: "state failure", + CreatedAt: createdAt, + UpdatedAt: updatedAt, + } + batchState := BatchState{ + BatchID: "batch-state", + DispatchID: "dispatch-batch-state", + Name: "nightly", + Queue: "bulk", + AllowFailed: true, + Total: 7, + Pending: 3, + Processed: 4, + Failed: 2, + Cancelled: true, + Completed: false, + CreatedAt: createdAt, + UpdatedAt: updatedAt, + } + nextNode := &ChainNode{ + NodeID: "node-next", + Job: StoredJob{ + Type: "reports:next", + Payload: []byte(`{"next":true}`), + Options: StoredJobOptions{Queue: "critical", Retry: 8}, + }, + } + spy := &workflowStoreAdapterSpy{ + wantContext: ctx, + contextsOK: true, + createChainErr: storeErr, + advanceNext: nextNode, + advanceDone: true, + advanceErr: storeErr, + failChainErr: storeErr, + getChainState: chainState, + getChainErr: storeErr, + createBatchErr: storeErr, + startedErr: storeErr, + succeededState: batchState, + succeededDone: true, + succeededErr: storeErr, + failedState: batchState, + failedDone: false, + failedErr: storeErr, + cancelErr: storeErr, + getBatchState: batchState, + getBatchErr: storeErr, + callbackClaimed: true, + callbackErr: storeErr, + pruneErr: storeErr, + } + view := &workflowStoreView{store: rootWorkflowStoreAdapter{store: spy}} + + chainRecord := ChainRecord{ + ChainID: "chain-create", + DispatchID: "dispatch-chain-create", + Queue: "critical", + Nodes: []ChainNode{{ + NodeID: "node-create", + Job: StoredJob{ + Type: "reports:create", + Payload: []byte(`{"create":true}`), + Options: StoredJobOptions{Queue: "critical", Delay: time.Second, Timeout: 2 * time.Second, Retry: 3, Backoff: 4 * time.Second, UniqueFor: 5 * time.Second}, + }, + }}, + CreatedAt: createdAt, + } + wantChainRecord := chainRecord + wantChainRecord.Nodes = append([]ChainNode(nil), chainRecord.Nodes...) + wantChainRecord.Nodes[0].Job.Payload = cloneWorkflowPayload(chainRecord.Nodes[0].Job.Payload) + if err := view.CreateChain(ctx, chainRecord); err != storeErr { + t.Fatalf("CreateChain error = %v, want exact sentinel", err) + } + if !reflect.DeepEqual(spy.createChainRecord, wantChainRecord) { + t.Fatalf("CreateChain record = %+v, want %+v", spy.createChainRecord, wantChainRecord) + } + chainRecord.Nodes[0].Job.Payload[0] = '!' + if got := string(spy.createChainRecord.Nodes[0].Job.Payload); got != `{"create":true}` { + t.Fatalf("CreateChain payload aliased root input: %q", got) + } + + next, done, err := view.AdvanceChain(ctx, "chain-advance", "node-complete") + if err != storeErr || !done || spy.advanceChainID != "chain-advance" || spy.advanceCompletedNode != "node-complete" || next == nil || !reflect.DeepEqual(*next, *nextNode) { + t.Fatalf("AdvanceChain result = next:%+v done:%t err:%v args:%q/%q", next, done, err, spy.advanceChainID, spy.advanceCompletedNode) + } + next.Job.Payload[0] = '?' + if got := string(spy.advanceNext.Job.Payload); got != `{"next":true}` { + t.Fatalf("AdvanceChain payload aliased internal output: %q", got) + } + spy.advanceNext = nil + spy.advanceDone = false + nilNext, nilDone, nilErr := view.AdvanceChain(ctx, "chain-nil", "node-nil") + if nilNext != nil || nilDone || nilErr != storeErr { + t.Fatalf("AdvanceChain nil successor = next:%+v done:%t err:%v", nilNext, nilDone, nilErr) + } + + if err := view.FailChain(ctx, "chain-fail", chainCause); err != storeErr || spy.failChainID != "chain-fail" || spy.failChainCause != chainCause { + t.Fatalf("FailChain result = err:%v id:%q cause:%v", err, spy.failChainID, spy.failChainCause) + } + gotChain, err := view.GetChain(ctx, "chain-get") + if err != storeErr || spy.getChainID != "chain-get" || !reflect.DeepEqual(gotChain, chainState) { + t.Fatalf("GetChain result = state:%+v err:%v id:%q", gotChain, err, spy.getChainID) + } + gotChain.Nodes[0].Job.Payload[0] = '#' + if got := string(spy.getChainState.Nodes[0].Job.Payload); got != `{"state":true}` { + t.Fatalf("GetChain payload aliased internal state: %q", got) + } + + batchRecord := BatchRecord{ + BatchID: "batch-create", + DispatchID: "dispatch-batch-create", + Name: "daily", + Queue: "bulk", + AllowFailed: true, + Jobs: []BatchJob{{ + JobID: "job-create", + Job: StoredJob{ + Type: "reports:batch", + Payload: []byte(`{"batch":true}`), + Options: StoredJobOptions{Queue: "bulk", Delay: 6 * time.Second, Timeout: 7 * time.Second, Retry: 8, Backoff: 9 * time.Second, UniqueFor: 10 * time.Second}, + }, + }}, + CreatedAt: createdAt, + } + wantBatchRecord := batchRecord + wantBatchRecord.Jobs = append([]BatchJob(nil), batchRecord.Jobs...) + wantBatchRecord.Jobs[0].Job.Payload = cloneWorkflowPayload(batchRecord.Jobs[0].Job.Payload) + if err := view.CreateBatch(ctx, batchRecord); err != storeErr { + t.Fatalf("CreateBatch error = %v, want exact sentinel", err) + } + if !reflect.DeepEqual(spy.createBatchRecord, wantBatchRecord) { + t.Fatalf("CreateBatch record = %+v, want %+v", spy.createBatchRecord, wantBatchRecord) + } + batchRecord.Jobs[0].Job.Payload[0] = '!' + if got := string(spy.createBatchRecord.Jobs[0].Job.Payload); got != `{"batch":true}` { + t.Fatalf("CreateBatch payload aliased root input: %q", got) + } + + if err := view.MarkBatchJobStarted(ctx, "batch-start", "job-start"); err != storeErr || spy.startedBatchID != "batch-start" || spy.startedJobID != "job-start" { + t.Fatalf("MarkBatchJobStarted result = err:%v args:%q/%q", err, spy.startedBatchID, spy.startedJobID) + } + gotSucceeded, succeededDone, err := view.MarkBatchJobSucceeded(ctx, "batch-succeed", "job-succeed") + if err != storeErr || !succeededDone || spy.succeededBatchID != "batch-succeed" || spy.succeededJobID != "job-succeed" || !reflect.DeepEqual(gotSucceeded, batchState) { + t.Fatalf("MarkBatchJobSucceeded result = state:%+v done:%t err:%v args:%q/%q", gotSucceeded, succeededDone, err, spy.succeededBatchID, spy.succeededJobID) + } + gotFailed, failedDone, err := view.MarkBatchJobFailed(ctx, "batch-fail", "job-fail", memberCause) + if err != storeErr || failedDone || spy.failedBatchID != "batch-fail" || spy.failedJobID != "job-fail" || spy.failedCause != memberCause || !reflect.DeepEqual(gotFailed, batchState) { + t.Fatalf("MarkBatchJobFailed result = state:%+v done:%t err:%v args:%q/%q cause:%v", gotFailed, failedDone, err, spy.failedBatchID, spy.failedJobID, spy.failedCause) + } + if err := view.CancelBatch(ctx, "batch-cancel"); err != storeErr || spy.cancelBatchID != "batch-cancel" { + t.Fatalf("CancelBatch result = err:%v id:%q", err, spy.cancelBatchID) + } + gotBatch, err := view.GetBatch(ctx, "batch-get") + if err != storeErr || spy.getBatchID != "batch-get" || !reflect.DeepEqual(gotBatch, batchState) { + t.Fatalf("GetBatch result = state:%+v err:%v id:%q", gotBatch, err, spy.getBatchID) + } + + claimed, err := view.MarkCallbackInvoked(ctx, "callback-key") + if err != storeErr || !claimed || spy.callbackKey != "callback-key" { + t.Fatalf("MarkCallbackInvoked result = claimed:%t err:%v key:%q", claimed, err, spy.callbackKey) + } + before := updatedAt.Add(24 * time.Hour) + if err := view.Prune(ctx, before); err != storeErr || !spy.pruneBefore.Equal(before) { + t.Fatalf("Prune result = err:%v before:%v", err, spy.pruneBefore) + } + if !spy.contextsOK { + t.Fatal("one or more built-in store view methods replaced the caller context") + } +} + +// TestWorkflowStoreViewExposesOutcomeCapability proves built-in stores retain +// first-writer arbitration through the root-owned physical model. +func TestWorkflowStoreViewExposesOutcomeCapability(t *testing.T) { + ctx := context.Background() + store := NewMemoryStore() + outcomes, ok := store.(WorkflowOutcomeStore) + if !ok { + t.Fatalf("built-in store %T does not implement WorkflowOutcomeStore", store) + } + if err := store.CreateChain(ctx, ChainRecord{ChainID: "chain-outcome-view", Nodes: []ChainNode{{NodeID: "node-outcome-view"}}}); err != nil { + t.Fatalf("create chain: %v", err) + } + chainState, owned, err := outcomes.FailChainNode(ctx, "chain-outcome-view", "node-outcome-view", errors.New("chain failed")) + if err != nil || !owned || !chainState.Failed || chainState.ChainID != "chain-outcome-view" { + t.Fatalf("chain outcome = state:%+v owned:%t err:%v", chainState, owned, err) + } + if err := store.CreateBatch(ctx, BatchRecord{BatchID: "batch-outcome-view", Jobs: []BatchJob{{JobID: "job-outcome-view"}}}); err != nil { + t.Fatalf("create batch: %v", err) + } + batchState, owned, err := outcomes.SettleBatchJob(ctx, "batch-outcome-view", "job-outcome-view", BatchJobSucceeded, nil) + if err != nil || !owned || !batchState.Completed || batchState.Processed != 1 { + t.Fatalf("batch outcome = state:%+v owned:%t err:%v", batchState, owned, err) + } +}