diff --git a/.github/workflows/test-error-annotation.yaml b/.github/workflows/test-error-annotation.yaml new file mode 100644 index 0000000..5bb4123 --- /dev/null +++ b/.github/workflows/test-error-annotation.yaml @@ -0,0 +1,66 @@ +name: error-annotation +on: + pull_request: + push: +jobs: + entrypoint_error_annotation: + runs-on: ubuntu-latest + name: Entrypoints surface genuine errors as annotations, not success/fail-on + steps: + - name: checkout + uses: actions/checkout@v6 + - name: Exercise every entrypoint with a stubbed oasdiff + run: | + set -eu + + # Stub oasdiff: write $STUB_STDERR to stderr (if set), produce no + # stdout, and exit $STUB_CODE. Lets us drive each entrypoint's + # exit-code handling deterministically — no real spec, no network. + mkdir -p /tmp/bin + cat > /tmp/bin/oasdiff <<'STUB' + #!/bin/sh + [ -n "${STUB_STDERR:-}" ] && printf '%s\n' "$STUB_STDERR" >&2 + exit "${STUB_CODE:-0}" + STUB + chmod +x /tmp/bin/oasdiff + export PATH="/tmp/bin:$PATH" + export GITHUB_OUTPUT=/tmp/gh_output GITHUB_STEP_SUMMARY=/tmp/gh_summary + export GITHUB_REPOSITORY=o/r GITHUB_SHA=deadbeef GITHUB_REF=refs/pull/1/merge + echo '{}' > /tmp/event.json; export GITHUB_EVENT_PATH=/tmp/event.json + + fail() { echo "FAIL: $1" >&2; exit 1; } + + # invoke -> prints the entrypoint's stdout. + # pr-comment gets an empty oasdiff-token so it skips the network POST. + invoke() { + export STUB_CODE="$1" STUB_STDERR="$2"; cmd="$3" + : > "$GITHUB_OUTPUT"; : > "$GITHUB_STEP_SUMMARY" + case "$cmd" in + breaking) sh breaking/entrypoint.sh base.yaml rev.yaml "" "" "" "" "" "" "" "" "" "" "" "" "false" ;; + changelog) sh changelog/entrypoint.sh base.yaml rev.yaml "" "" "" "" "" "" "" "" "" "" "" "" "false" ;; + diff) sh diff/entrypoint.sh base.yaml rev.yaml "" "" "" "" "" "" "" "false" ;; + pr-comment) sh pr-comment/entrypoint.sh base.yaml rev.yaml "" "" "" "" "" "" "false" ;; + validate) sh validate/entrypoint.sh spec.yaml "" "false" ;; + esac + } + + for cmd in breaking changelog diff pr-comment validate; do + # success (exit 0): no ::error:: annotation + out=$(invoke 0 "" "$cmd" 2>/dev/null || true) + echo "$out" | grep -q '::error::' && fail "$cmd: success (exit 0) must not emit ::error::" + + # genuine error (exit 102): ::error:: annotation surfaced + out=$(invoke 102 'Error: failed to load spec from "x.yaml": no such file' "$cmd" 2>/dev/null || true) + echo "$out" | grep -q '::error::' || fail "$cmd: exit 102 must emit a ::error:: annotation" + + # disallowed external ref (exit 123): the allow-external-refs remedy + out=$(invoke 123 'Error: external $ref not allowed: https://evil' "$cmd" 2>/dev/null || true) + echo "$out" | grep -q 'allow-external-refs: true' || fail "$cmd: exit 123 must emit the allow-external-refs remedy" + + # fail-on / changes-found (exit 1, no stderr): no ::error:: annotation + out=$(invoke 1 "" "$cmd" 2>/dev/null || true) + echo "$out" | grep -q '::error::' && fail "$cmd: exit 1 (fail-on) must not emit ::error::" + + echo "ok: $cmd" + done + echo "all error-annotation assertions passed" diff --git a/breaking/entrypoint.sh b/breaking/entrypoint.sh index a1c4bc8..e80c584 100755 --- a/breaking/entrypoint.sh +++ b/breaking/entrypoint.sh @@ -95,6 +95,12 @@ exit_code=0 _err=$(mktemp) breaking_changes=$(oasdiff breaking "$base" "$revision" $flags $fail_on_flag 2>"$_err") || exit_code=$? [ -s "$_err" ] && cat "$_err" >&2 +# Promote a genuine oasdiff failure to a Checks-tab annotation. Exit 0 is +# success and exit 1 is the intended "breaking changes found" / fail-on result; +# only codes >=2 (load/parse/etc.) are real errors worth surfacing here. +if [ "$exit_code" -ge 2 ] && [ -s "$_err" ]; then + echo "::error::$(tr '\n' ' ' < "$_err")" +fi # Exit code 123 = oasdiff refused a disallowed external $ref (stable contract, # not message text). Surface the action-specific remedy. if [ "$exit_code" -eq 123 ]; then diff --git a/changelog/entrypoint.sh b/changelog/entrypoint.sh index e4318f2..0d22c00 100755 --- a/changelog/entrypoint.sh +++ b/changelog/entrypoint.sh @@ -104,6 +104,11 @@ else fi if [ "$exit_code" -ne 0 ]; then [ -s "$_err" ] && cat "$_err" >&2 + # Promote a genuine failure to a Checks-tab annotation. Exit 1 is the + # intended fail-on result (not an error); only codes >=2 are real errors. + if [ "$exit_code" -ge 2 ] && [ -s "$_err" ]; then + echo "::error::$(tr '\n' ' ' < "$_err")" + fi # Exit code 123 = oasdiff refused a disallowed external $ref (stable # contract, not message text). Surface the action-specific remedy. if [ "$exit_code" -eq 123 ]; then diff --git a/diff/entrypoint.sh b/diff/entrypoint.sh index 3e32d15..7b4e183 100755 --- a/diff/entrypoint.sh +++ b/diff/entrypoint.sh @@ -88,6 +88,12 @@ else output=$(oasdiff diff "$base" "$revision" 2>"$_err") || exit_code=$? fi [ -s "$_err" ] && cat "$_err" >&2 +# Promote a genuine oasdiff failure to a Checks-tab annotation. Exit 0 is +# success and exit 1 is the intended fail-on-diff result; only codes >=2 +# (load/parse/etc.) are real errors worth surfacing here. +if [ "$exit_code" -ge 2 ] && [ -s "$_err" ]; then + echo "::error::$(tr '\n' ' ' < "$_err")" +fi # Exit code 123 = oasdiff refused a disallowed external $ref (stable contract, # not message text). Surface the action-specific remedy. if [ "$exit_code" -eq 123 ]; then diff --git a/pr-comment/entrypoint.sh b/pr-comment/entrypoint.sh index cafd5c0..8c40639 100755 --- a/pr-comment/entrypoint.sh +++ b/pr-comment/entrypoint.sh @@ -46,6 +46,11 @@ _err=$(mktemp) changelog=$(oasdiff changelog "$base" "$revision" --format json $flags 2>"$_err") || oasdiff_exit=$? if [ "$oasdiff_exit" -ne 0 ] && [ -z "$changelog" ]; then [ -s "$_err" ] && cat "$_err" >&2 + # Promote a genuine failure to a Checks-tab annotation. Exit 1 is the + # intended fail-on result (not an error); only codes >=2 are real errors. + if [ "$oasdiff_exit" -ge 2 ] && [ -s "$_err" ]; then + echo "::error::$(tr '\n' ' ' < "$_err")" + fi # Exit code 123 = oasdiff refused a disallowed external $ref (stable # contract, not message text). Surface the action-specific remedy. if [ "$oasdiff_exit" -eq 123 ]; then diff --git a/validate/entrypoint.sh b/validate/entrypoint.sh index 6db87f1..331c144 100755 --- a/validate/entrypoint.sh +++ b/validate/entrypoint.sh @@ -35,6 +35,12 @@ exit_code=0 _err=$(mktemp) oasdiff validate $flags --format githubactions "$spec" 2>"$_err" || exit_code=$? [ -s "$_err" ] && cat "$_err" >&2 +# Promote a genuine oasdiff failure to a Checks-tab annotation. Exit 0 is +# success and exit 1 is the intended fail-on result; only codes >=2 +# (load/parse/etc.) are real errors worth surfacing here. +if [ "$exit_code" -ge 2 ] && [ -s "$_err" ]; then + echo "::error::$(tr '\n' ' ' < "$_err")" +fi # Exit code 123 = oasdiff refused a disallowed external $ref (stable contract, # not message text). Surface the action-specific remedy. if [ "$exit_code" -eq 123 ]; then