From 6355de6fa2922a45dbd4b770ba0e58849493b7d5 Mon Sep 17 00:00:00 2001 From: Really Him Date: Tue, 2 Jun 2026 18:23:57 -0400 Subject: [PATCH 1/4] chore(deps): bump wrangler --- package-lock.json | 72 +++++++++++++++++++++++------------------------ package.json | 4 +-- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/package-lock.json b/package-lock.json index 92e19c6..32343c8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -25,7 +25,7 @@ "tsx": "^4.22.4", "typescript": "^6.0.3", "vitest": "^4.1.8", - "wrangler": "^4.96.0" + "wrangler": "^4.97.0" }, "engines": { "node": ">=24" @@ -118,9 +118,9 @@ } }, "node_modules/@cloudflare/workerd-darwin-64": { - "version": "1.20260529.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260529.1.tgz", - "integrity": "sha512-gxh5sXw0CsBxNCNj8uJnrAxqFM7+R8SZI9WIqYMKz6uaPxgg+eTcBDTxjKczMs6bS21FkTEF6ohIzB5+UvxwKw==", + "version": "1.20260601.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-64/-/workerd-darwin-64-1.20260601.1.tgz", + "integrity": "sha512-iXZBVuRbvuVqQ/63wul01hHCv/3R8G5S8zbkjfoHvyPZFynmlKTV59Hk+H8whyGwFAZuB71UJGLr+G5mJKfjWA==", "cpu": [ "x64" ], @@ -135,9 +135,9 @@ } }, "node_modules/@cloudflare/workerd-darwin-arm64": { - "version": "1.20260529.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20260529.1.tgz", - "integrity": "sha512-B8xOwqd8ok8oaWBPhrpmNVSYou6AejFrYf3VzsJF6pg6TEA2tYbdThAGXgtLPQ8d1RD7GXYjVth2dSMg9napDA==", + "version": "1.20260601.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-darwin-arm64/-/workerd-darwin-arm64-1.20260601.1.tgz", + "integrity": "sha512-veGpZQGBw07Twt+Y4z3oyo+/obKHt0iWUwvDV5GOiDAYjC/zW+YGstgVzg4SHq+k1sLH3ElqL2TXx20I5WBv3Q==", "cpu": [ "arm64" ], @@ -152,9 +152,9 @@ } }, "node_modules/@cloudflare/workerd-linux-64": { - "version": "1.20260529.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20260529.1.tgz", - "integrity": "sha512-M1EKzsfoKmmno7MNPkuIc8iOdHLhFnE7ltEYaGGEoOj1MTJfMBK/JkIrhdkzc/06wpyPZPiBfBBmUppbeaMqUg==", + "version": "1.20260601.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-64/-/workerd-linux-64-1.20260601.1.tgz", + "integrity": "sha512-n/9hDz7fPGpYF0J684+Xr5zgjcS2jdmY2Of5m6e+eQ/M9+RfR+UaU8Ee/tkA1dDC0LYQB13hfPafZG66Ff1CsA==", "cpu": [ "x64" ], @@ -169,9 +169,9 @@ } }, "node_modules/@cloudflare/workerd-linux-arm64": { - "version": "1.20260529.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20260529.1.tgz", - "integrity": "sha512-Mn/Qpl1FAHDLtPthw6ti5gsHRj582jJdtK4OMUlW1CN0v+pmmxaav3KSqq7CS6a+5W0o2e8o9fKnjVilBxVVmQ==", + "version": "1.20260601.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-linux-arm64/-/workerd-linux-arm64-1.20260601.1.tgz", + "integrity": "sha512-VHRZZbexATS+n+1j3x/CZaYbIJEye0J3iIHgG0Wp+l+NrZCKQ8qi8Lq1uTV0dLJQ67FuZtJtWdQ95mm9F7Fc+A==", "cpu": [ "arm64" ], @@ -186,9 +186,9 @@ } }, "node_modules/@cloudflare/workerd-windows-64": { - "version": "1.20260529.1", - "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20260529.1.tgz", - "integrity": "sha512-78xgJJeXxkKYumWdKGH1pybUsEjTreSvbJqirW9cth7ZGonqdv5pzAVt+WWcbu0OFcSHrtQFX6zWioPNFp0/xQ==", + "version": "1.20260601.1", + "resolved": "https://registry.npmjs.org/@cloudflare/workerd-windows-64/-/workerd-windows-64-1.20260601.1.tgz", + "integrity": "sha512-ye0C7MFLkeH16iTo8Tcjv2KiFmp23+sZGvUzSQa4xhP0QMe6EoJ+H/4SqqvnZ5nfN54slqKvx2VnXceENWe2CQ==", "cpu": [ "x64" ], @@ -3419,16 +3419,16 @@ } }, "node_modules/miniflare": { - "version": "4.20260529.0", - "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-4.20260529.0.tgz", - "integrity": "sha512-4pj7WZQR/uYqVMa0cpAmmPBKEb0JegSocuystaXCubY455iqWdPUqgVD9R6N28oneWyPiUyAu5N8QpLbK+MU/Q==", + "version": "4.20260601.0", + "resolved": "https://registry.npmjs.org/miniflare/-/miniflare-4.20260601.0.tgz", + "integrity": "sha512-56TFiulSEQu43cYxdXgCiA3U3i+Ls0NoXwJXd6DmpNsx8yl/1Il2T3DQ4CMXjR6yfE7CSvC5MuXaqcSAMREjgw==", "dev": true, "license": "MIT", "dependencies": { "@cspotcode/source-map-support": "0.8.1", - "sharp": "^0.34.5", + "sharp": "0.34.5", "undici": "7.24.8", - "workerd": "1.20260529.1", + "workerd": "1.20260601.1", "ws": "8.20.1", "youch": "4.1.0-beta.10" }, @@ -4350,9 +4350,9 @@ } }, "node_modules/workerd": { - "version": "1.20260529.1", - "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260529.1.tgz", - "integrity": "sha512-G1rurOKEdzCtFE0yUPR9J9mUnPzMU8NdsD7NKM1/oMyCr1j3VEtWJzc5VbhgFQHNBVWrHzCL0JgVPuBirRW31g==", + "version": "1.20260601.1", + "resolved": "https://registry.npmjs.org/workerd/-/workerd-1.20260601.1.tgz", + "integrity": "sha512-Bg4+HF3B8TW0urAv8chiz25HSQ/aJxMBjgheUzu/nB1NQa+CaKGrUPv+Z3bf0np/WxLHYW1kcseVEtzZVPbX4g==", "dev": true, "hasInstallScript": true, "license": "Apache-2.0", @@ -4363,17 +4363,17 @@ "node": ">=16" }, "optionalDependencies": { - "@cloudflare/workerd-darwin-64": "1.20260529.1", - "@cloudflare/workerd-darwin-arm64": "1.20260529.1", - "@cloudflare/workerd-linux-64": "1.20260529.1", - "@cloudflare/workerd-linux-arm64": "1.20260529.1", - "@cloudflare/workerd-windows-64": "1.20260529.1" + "@cloudflare/workerd-darwin-64": "1.20260601.1", + "@cloudflare/workerd-darwin-arm64": "1.20260601.1", + "@cloudflare/workerd-linux-64": "1.20260601.1", + "@cloudflare/workerd-linux-arm64": "1.20260601.1", + "@cloudflare/workerd-windows-64": "1.20260601.1" } }, "node_modules/wrangler": { - "version": "4.96.0", - "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-4.96.0.tgz", - "integrity": "sha512-8WuiMutalyfBB74wwRyy4VKKJEHjQuEnwcvdUav1M5AfQ8VaTYY5ZQnzvVZPOVXap40k5Mntz1LY3SPWpPukTg==", + "version": "4.97.0", + "resolved": "https://registry.npmjs.org/wrangler/-/wrangler-4.97.0.tgz", + "integrity": "sha512-jzW/aNvjerV+4TmwbvwGY6lpcuBk7EFUTonMDNfci45wSmMTj2/OJN+83cc/CeepKdb+6ZjGJw9NRjmcQoxqRg==", "dev": true, "license": "MIT OR Apache-2.0", "dependencies": { @@ -4381,10 +4381,10 @@ "@cloudflare/unenv-preset": "2.16.1", "blake3-wasm": "2.1.5", "esbuild": "0.27.3", - "miniflare": "4.20260529.0", + "miniflare": "4.20260601.0", "path-to-regexp": "6.3.0", "unenv": "2.0.0-rc.24", - "workerd": "1.20260529.1" + "workerd": "1.20260601.1" }, "bin": { "wrangler": "bin/wrangler.js", @@ -4394,10 +4394,10 @@ "node": ">=22.0.0" }, "optionalDependencies": { - "fsevents": "~2.3.2" + "fsevents": "2.3.3" }, "peerDependencies": { - "@cloudflare/workers-types": "^4.20260529.1" + "@cloudflare/workers-types": "^4.20260601.1" }, "peerDependenciesMeta": { "@cloudflare/workers-types": { diff --git a/package.json b/package.json index bf6f824..62cf99f 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "policychecks", "version": "0.2.0", "description": "A badge service that is able to report on a repository's settings, providing a convenient way for maintainers to demonstrate publicly that best practices are enforced as policies at the repo configuration level", - "private": true, + "private": false, "type": "module", "main": "dist/app.js", "engines": { @@ -54,6 +54,6 @@ "tsx": "^4.22.4", "typescript": "^6.0.3", "vitest": "^4.1.8", - "wrangler": "^4.96.0" + "wrangler": "^4.97.0" } } From 86ef2a78aec42de43cf5d5c09304401ec4cce82c Mon Sep 17 00:00:00 2001 From: Really Him Date: Tue, 2 Jun 2026 18:52:39 -0400 Subject: [PATCH 2/4] docs: add policychecks badges to readme --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 0252a32..320ff2e 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # PolicyChecks +[![Immutable releases](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/immutable-releases.svg)](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/immutable-releases/proof.json) [![SHA-pinned actions](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/sha-pinning-required.svg)](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/sha-pinning-required/proof.json) [![Secret scanning](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/secret-scanning-enabled.svg)](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/secret-scanning-enabled/proof.json) [![Dependabot alerts](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/dependabot-alerts-enabled.svg)](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/dependabot-alerts-enabled/proof.json) [![Dependency graph](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/dependency-graph-enabled.svg)](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/dependency-graph-enabled/proof.json) + PolicyChecks is a GitHub App-backed badge service and validation endpoint for repository settings that ordinary public badge services cannot verify. It exposes badge SVG, Shields-compatible JSON, and proof JSON endpoints for repository administration and security checks. This gives maintainers a convenient way to show that a project not only follows best practices, but that these practices are enforced policies at the repository settings level. This fills a modest gap in the badge ecosystem between excellent services like shields.io (which does not have the permissions to report on these facts) and OSSF Scorecard (which does take into account many of these same conditions, but does not expose individual setting-level endpoints). | Check | Claim ID | Passing result | Other results | From 650ecd04dd18964de4cf119e3d2a617f37eed3eb Mon Sep 17 00:00:00 2001 From: Really Him Date: Tue, 2 Jun 2026 19:54:19 -0400 Subject: [PATCH 3/4] fix: align security badges with settings evidence --- docs/claim-semantics.md | 42 +++--- docs/operations.md | 2 +- src/claims/code-security-configuration.ts | 109 +++++++++++++-- src/github/client.ts | 23 ++++ src/server/claim-service.ts | 6 + .../code-security-configuration.test.ts | 129 ++++++++++++------ test/github/api-usage-policy.test.ts | 3 +- test/github/client.test.ts | 11 ++ test/github/installations.test.ts | 1 + test/support/mock-github.ts | 1 + 10 files changed, 252 insertions(+), 75 deletions(-) diff --git a/docs/claim-semantics.md b/docs/claim-semantics.md index 1661521..c2b6109 100644 --- a/docs/claim-semantics.md +++ b/docs/claim-semantics.md @@ -29,6 +29,8 @@ The tables below use these GitHub documentation references as their audit trail: | `immutable-releases-doc` | [Check if immutable releases are enabled for a repository](https://docs.github.com/en/rest/repos/repos#check-if-immutable-releases-are-enabled-for-a-repository) | | `actions-permissions-doc` | [Get GitHub Actions permissions for a repository](https://docs.github.com/en/rest/actions/permissions#get-github-actions-permissions-for-a-repository) | | `branch-rules-doc` | [Get rules for a branch](https://docs.github.com/en/rest/repos/rules#get-rules-for-a-branch) | +| `repository-doc` | [Get a repository](https://docs.github.com/en/rest/repos/repos#get-a-repository) | +| `vulnerability-alerts-doc` | [Check if vulnerability alerts are enabled for a repository](https://docs.github.com/en/rest/repos/repos#check-if-vulnerability-alerts-are-enabled-for-a-repository) | | `code-security-config-doc` | [Get the code security configuration associated with a repository](https://docs.github.com/en/rest/code-security/configurations#get-the-code-security-configuration-associated-with-a-repository) | ## Global Error Mapping @@ -79,43 +81,37 @@ GET /repos/{owner}/{repo}/actions/permissions ## `secret-scanning-enabled` -Claim: the repository's attached code security configuration enables secret scanning. +Claim: the repository's security and analysis settings enable secret scanning. GitHub endpoint: ```http -GET /repos/{owner}/{repo}/code-security-configuration +GET /repos/{owner}/{repo} ``` | GitHub response or value | PolicyChecks status | Proof details | Documentation basis | Judgment | | --- | --- | --- | --- | --- | -| `200 OK` with `status: attached` and `configuration.secret_scanning: enabled` | `pass` | `status`, selected `configuration` metadata, `secret_scanning` | `code-security-config-doc` documents `200 OK` and example fields including `status`, `configuration`, and `secret_scanning`. | Direct evidence that secret scanning is enabled by the attached configuration. | -| `200 OK` with `status: attached` and `configuration.secret_scanning` as a string other than `enabled` | `fail` | `status`, selected `configuration` metadata, `secret_scanning` | `code-security-config-doc` documents `secret_scanning` as the enablement status field. | Direct evidence from an attached configuration that this field is not enabled. | -| `204 No Content` | `unknown` | `status: no_content`, `configuration: null` | `code-security-config-doc` lists `204 No Content` but does not define it as disabled. | Not safe to assert disabled from no content. | -| `200 OK` with missing or non-`attached` `status` | `unknown` | `status`, selected configuration metadata when present | `code-security-config-doc` example shows `status: attached`; other status semantics are not yet mapped. | Not enough documented evidence for enabled or disabled. | -| `200 OK` with attached status but missing/non-string `configuration.secret_scanning` | `unknown` | selected configuration metadata | `code-security-config-doc` documents a string enablement status field; the returned shape does not match. | The service cannot safely interpret the field. | -| `404 Not Found` | `unknown` | error details | `code-security-config-doc` documents `404` as resource not found, not as disabled. | Not safe to assert disabled from this response. | +| `200 OK` with `security_and_analysis.secret_scanning.status: enabled` | `pass` | selected `security_and_analysis.secret_scanning` status | `repository-doc` documents repository metadata as the place to check security and analysis feature status. | Direct evidence that secret scanning is enabled. | +| `200 OK` with `security_and_analysis.secret_scanning.status` as a string other than `enabled` | `fail` | selected `security_and_analysis.secret_scanning` status | `repository-doc` documents the feature status as an enablement field. | Direct evidence that secret scanning is not enabled. | +| `200 OK` with missing or non-string `security_and_analysis.secret_scanning.status` | `unknown` | error details | `repository-doc` documents security and analysis feature status, but the returned shape does not match. | The service cannot safely interpret the response. | +| `404 Not Found` | `unknown` | error details | Repository access is resolved before claim evaluation; a repository metadata `404` is an access or repository identity problem, not a setting value. | Not safe to assert disabled from this response. | ## `dependabot-alerts-enabled` -Claim: the repository's attached code security configuration enables Dependabot alerts. +Claim: repository Dependabot vulnerability alerts are enabled. GitHub endpoint: ```http -GET /repos/{owner}/{repo}/code-security-configuration +GET /repos/{owner}/{repo}/vulnerability-alerts ``` -This claim uses the same endpoint and top-level response rules as `secret-scanning-enabled`, but reads `configuration.dependabot_alerts`. - | GitHub response or value | PolicyChecks status | Proof details | Documentation basis | Judgment | | --- | --- | --- | --- | --- | -| `200 OK` with `status: attached` and `configuration.dependabot_alerts: enabled` | `pass` | `status`, selected `configuration` metadata, `dependabot_alerts` | `code-security-config-doc` documents `dependabot_alerts` as an enablement status field with `enabled`, `disabled`, or `not_set`. | Direct evidence that Dependabot alerts are enabled by the attached configuration. | -| `200 OK` with `status: attached` and `configuration.dependabot_alerts` as a string other than `enabled` | `fail` | `status`, selected `configuration` metadata, `dependabot_alerts` | `code-security-config-doc` documents `dependabot_alerts` as the enablement status field. | Direct evidence from an attached configuration that this field is not enabled. | -| `204 No Content` | `unknown` | `status: no_content`, `configuration: null` | `code-security-config-doc` lists `204 No Content` but does not define it as disabled. | Not safe to assert disabled from no content. | -| `200 OK` with missing or non-`attached` `status` | `unknown` | `status`, selected configuration metadata when present | `code-security-config-doc` example shows `status: attached`; other status semantics are not yet mapped. | Not enough documented evidence for enabled or disabled. | -| `200 OK` with attached status but missing/non-string `configuration.dependabot_alerts` | `unknown` | selected configuration metadata | `code-security-config-doc` documents a string enablement status field; the returned shape does not match. | The service cannot safely interpret the field. | -| `404 Not Found` | `unknown` | error details | `code-security-config-doc` documents `404` as resource not found, not as disabled. | Not safe to assert disabled from this response. | +| `204 No Content` | `pass` | `vulnerability_alerts: enabled` | `vulnerability-alerts-doc` documents `204` as the response when the repository is enabled with vulnerability alerts. | Direct evidence that Dependabot alerts are enabled. | +| `404 Not Found` after repository installation/access has already been verified | `fail` | `vulnerability_alerts: disabled` | `vulnerability-alerts-doc` documents `404` as not enabled for this specific endpoint. | Safe disabled assertion only after repository access has already been verified. | +| `404 Not Found` before repository access is verified | `unknown` | error details | The endpoint-level `404` must be disambiguated from missing repository, missing installation, or missing permission. | Could mean no access rather than disabled. | +| `401`, `403`, rate limit, or other request failure | `unknown` | error details | Endpoint docs list authorization failures separately from setting values. | Not safe to infer disabled from failed access. | ## `dependency-graph-enabled` @@ -127,17 +123,17 @@ GitHub endpoint: GET /repos/{owner}/{repo}/code-security-configuration ``` -This claim uses the same endpoint and top-level response rules as `secret-scanning-enabled`, but reads `configuration.dependency_graph`. - | GitHub response or value | PolicyChecks status | Proof details | Documentation basis | Judgment | | --- | --- | --- | --- | --- | -| `200 OK` with `status: attached` and `configuration.dependency_graph: enabled` | `pass` | `status`, selected `configuration` metadata, `dependency_graph` | `code-security-config-doc` documents `dependency_graph` as an enablement status field with `enabled`, `disabled`, or `not_set`. | Direct evidence that the dependency graph is enabled by the attached configuration. | -| `200 OK` with `status: attached` and `configuration.dependency_graph` as a string other than `enabled` | `fail` | `status`, selected `configuration` metadata, `dependency_graph` | `code-security-config-doc` documents `dependency_graph` as the enablement status field. | Direct evidence from an attached configuration that this field is not enabled. | -| `204 No Content` | `unknown` | `status: no_content`, `configuration: null` | `code-security-config-doc` lists `204 No Content` but does not define it as disabled. | Not safe to assert disabled from no content. | +| `200 OK` with `status: attached` and `configuration.dependency_graph: enabled` | `pass` | `status`, selected `configuration` metadata, `dependency_graph` | `code-security-config-doc` documents `dependency_graph` as the enablement status field for Dependency Graph. | Direct evidence that the attached configuration enables dependency graph. | +| `200 OK` with `status: attached` and `configuration.dependency_graph` as a string other than `enabled` | `fail` | `status`, selected `configuration` metadata, `dependency_graph` | `code-security-config-doc` documents `dependency_graph` as an enablement status field with `enabled`, `disabled`, or `not_set`. | Direct evidence from the attached configuration that this configured policy is not enabled. | +| `204 No Content` | `unknown` | `status: no_content`, `configuration: null` | `code-security-config-doc` lists `204 No Content` but does not define it as disabled. | The repository may satisfy the underlying condition some other way; PolicyChecks only knows no attached configuration was returned. | | `200 OK` with missing or non-`attached` `status` | `unknown` | `status`, selected configuration metadata when present | `code-security-config-doc` example shows `status: attached`; other status semantics are not yet mapped. | Not enough documented evidence for enabled or disabled. | | `200 OK` with attached status but missing/non-string `configuration.dependency_graph` | `unknown` | selected configuration metadata | `code-security-config-doc` documents a string enablement status field; the returned shape does not match. | The service cannot safely interpret the field. | | `404 Not Found` | `unknown` | error details | `code-security-config-doc` documents `404` as resource not found, not as disabled. | Not safe to assert disabled from this response. | +Caveat: this claim does not check automatic dependency submission. GitHub exposes that as `configuration.dependency_graph_autosubmit_action`, a separate field from `configuration.dependency_graph`. + ## Adding A New Claim Before adding a new public badge, document: diff --git a/docs/operations.md b/docs/operations.md index 99a5e67..649ee79 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -26,7 +26,7 @@ Webhook handling must not call GitHub. This is covered by tests in `test/webhook Production GitHub REST usage is restricted by `test/github/api-usage-policy.test.ts`. The allowed request surface is intentionally small and does not include repository enumeration, search, GraphQL, pagination helpers, or mutating repository routes. -Expected cold-cache request cost for one repository is bounded: installation lookup and installation-token creation, repository metadata when needed, immutable releases, Actions permissions, and one code-security-configuration request shared by the three code-security claims. Warm-cache badge/proof requests should make zero GitHub API calls. +Expected cold-cache request cost for one repository is bounded: installation lookup and installation-token creation, repository metadata when needed, immutable releases, Actions permissions, vulnerability-alerts status, and one code-security-configuration request shared by configuration-backed code security claims. Warm-cache badge/proof requests should make zero GitHub API calls. ## Rate-Limit Logs diff --git a/src/claims/code-security-configuration.ts b/src/claims/code-security-configuration.ts index e0f38b9..101f46e 100644 --- a/src/claims/code-security-configuration.ts +++ b/src/claims/code-security-configuration.ts @@ -1,4 +1,4 @@ -import { publicMessage, toPublicClaimError } from "../github/errors.js"; +import { GitHubApiError, publicMessage, toPublicClaimError } from "../github/errors.js"; import { isRecord, makeClaimResult, makeUnknownResult, resultInput } from "./result.js"; import type { ClaimDefinition, ClaimEvaluationInput } from "./types.js"; @@ -14,13 +14,38 @@ interface CodeSecurityClaimOptions { field: CodeSecurityField; } -const endpoint = "GET /repos/{owner}/{repo}/code-security-configuration"; +const codeSecurityConfigurationEndpoint = "GET /repos/{owner}/{repo}/code-security-configuration"; -export const secretScanningEnabledClaim = codeSecurityConfigurationClaim({ +export const secretScanningEnabledClaim: ClaimDefinition = { id: "secret-scanning-enabled", label: "secret scanning", - field: "secret_scanning" -}); + passMessage: "enabled", + failMessage: "disabled", + unknownMessage: "unknown", + source: { + provider: "github", + api: "REST", + endpoint: "GET /repos/{owner}/{repo}", + fields: ["security_and_analysis.secret_scanning.status"] + }, + async evaluate(input: ClaimEvaluationInput) { + try { + const repository = await input.github.getRepository(input.owner, input.repo); + return evaluateRepositorySecurityFeature( + secretScanningEnabledClaim, + input, + repository.security_and_analysis, + "secret_scanning" + ); + } catch (error) { + return makeUnknownResult( + secretScanningEnabledClaim, + resultInput(input), + toPublicClaimError(error) + ); + } + } +}; export const secretScanningPushProtectionEnabledClaim = codeSecurityConfigurationClaim({ id: "secret-scanning-push-protection-enabled", @@ -28,11 +53,40 @@ export const secretScanningPushProtectionEnabledClaim = codeSecurityConfiguratio field: "secret_scanning_push_protection" }); -export const dependabotAlertsEnabledClaim = codeSecurityConfigurationClaim({ +export const dependabotAlertsEnabledClaim: ClaimDefinition = { id: "dependabot-alerts-enabled", label: "Dependabot alerts", - field: "dependabot_alerts" -}); + passMessage: "enabled", + failMessage: "disabled", + unknownMessage: "unknown", + source: { + provider: "github", + api: "REST", + endpoint: "GET /repos/{owner}/{repo}/vulnerability-alerts", + fields: ["HTTP 204", "HTTP 404"] + }, + async evaluate(input: ClaimEvaluationInput) { + try { + const status = await input.github.getVulnerabilityAlertsStatus(input.owner, input.repo); + + return makeClaimResult(dependabotAlertsEnabledClaim, resultInput(input), "pass", true, { + vulnerability_alerts: status + }); + } catch (error) { + if (isEndpointDisabled(error)) { + return makeClaimResult(dependabotAlertsEnabledClaim, resultInput(input), "fail", false, { + vulnerability_alerts: "disabled" + }); + } + + return makeUnknownResult( + dependabotAlertsEnabledClaim, + resultInput(input), + toPublicClaimError(error) + ); + } + } +}; export const dependencyGraphEnabledClaim = codeSecurityConfigurationClaim({ id: "dependency-graph-enabled", @@ -50,7 +104,7 @@ function codeSecurityConfigurationClaim(options: CodeSecurityClaimOptions): Clai source: { provider: "github", api: "REST", - endpoint, + endpoint: codeSecurityConfigurationEndpoint, fields: ["status", `configuration.${options.field}`, "configuration.enforcement"] }, async evaluate(input: ClaimEvaluationInput) { @@ -66,6 +120,43 @@ function codeSecurityConfigurationClaim(options: CodeSecurityClaimOptions): Clai return definition; } +function evaluateRepositorySecurityFeature( + definition: ClaimDefinition, + input: ClaimEvaluationInput, + securityAndAnalysis: unknown, + field: "secret_scanning" +) { + if (!isRecord(securityAndAnalysis)) { + return unexpected(definition, input, { + security_and_analysis: null + }); + } + + const feature = securityAndAnalysis[field]; + + if (!isRecord(feature) || typeof feature.status !== "string") { + return unexpected(definition, input, { + security_and_analysis: { + [field]: null + } + }); + } + + const enabled = feature.status === "enabled"; + + return makeClaimResult(definition, resultInput(input), enabled ? "pass" : "fail", enabled, { + security_and_analysis: { + [field]: { + status: feature.status + } + } + }); +} + +function isEndpointDisabled(error: unknown) { + return error instanceof GitHubApiError && error.status === 404; +} + function evaluateCodeSecurityConfiguration( definition: ClaimDefinition, input: ClaimEvaluationInput, diff --git a/src/github/client.ts b/src/github/client.ts index 55ccd4d..47ec8a7 100644 --- a/src/github/client.ts +++ b/src/github/client.ts @@ -5,6 +5,15 @@ import { toGitHubApiError } from "./errors.js"; export interface GitHubRepository { id: number; default_branch?: string | null; + security_and_analysis?: GitHubRepositorySecurityAndAnalysis; +} + +export interface GitHubRepositorySecurityAndAnalysis { + secret_scanning?: GitHubRepositoryFeatureStatus; +} + +export interface GitHubRepositoryFeatureStatus { + status?: string; } export interface GitHubClient { @@ -12,6 +21,7 @@ export interface GitHubClient { getImmutableReleases(owner: string, repo: string): Promise; getActionsPermissions(owner: string, repo: string): Promise; getCodeSecurityConfiguration(owner: string, repo: string): Promise; + getVulnerabilityAlertsStatus(owner: string, repo: string): Promise<"enabled">; getBranchRules(owner: string, repo: string, branch: string): Promise; } @@ -58,6 +68,19 @@ export class GitHubRestClient implements GitHubClient { } } + async getVulnerabilityAlertsStatus(owner: string, repo: string): Promise<"enabled"> { + try { + await this.githubRequest("GET /repos/{owner}/{repo}/vulnerability-alerts", { + owner, + repo + }); + + return "enabled"; + } catch (error) { + throw toGitHubApiError(error); + } + } + async getBranchRules(owner: string, repo: string, branch: string): Promise { return this.getJson("GET /repos/{owner}/{repo}/rules/branches/{branch}", { owner, diff --git a/src/server/claim-service.ts b/src/server/claim-service.ts index 550a809..537e799 100644 --- a/src/server/claim-service.ts +++ b/src/server/claim-service.ts @@ -133,6 +133,7 @@ function memoizeGitHubClient(github: GitHubClient): GitHubClient { const immutableReleases = new Map>(); const actionsPermissions = new Map>(); const codeSecurityConfigurations = new Map>(); + const vulnerabilityAlerts = new Map>(); const branchRules = new Map>(); return { @@ -156,6 +157,11 @@ function memoizeGitHubClient(github: GitHubClient): GitHubClient { github.getCodeSecurityConfiguration(owner, repo) ); }, + getVulnerabilityAlertsStatus(owner: string, repo: string) { + return memoize(vulnerabilityAlerts, repositoryKey(owner, repo), () => + github.getVulnerabilityAlertsStatus(owner, repo) + ); + }, getBranchRules(owner: string, repo: string, branch: string) { return memoize(branchRules, `${repositoryKey(owner, repo)}:${branch}`, () => github.getBranchRules(owner, repo, branch) diff --git a/test/claims/code-security-configuration.test.ts b/test/claims/code-security-configuration.test.ts index b4a57a6..4fbd125 100644 --- a/test/claims/code-security-configuration.test.ts +++ b/test/claims/code-security-configuration.test.ts @@ -29,7 +29,14 @@ describe("code security configuration claims", () => { const result = await evaluateWithMock( secretScanningEnabledClaim, mockGitHub({ - getCodeSecurityConfiguration: async () => attachedConfiguration + getRepository: async () => ({ + id: 1, + security_and_analysis: { + secret_scanning: { + status: "enabled" + } + } + }) }) ); @@ -37,14 +44,10 @@ describe("code security configuration claims", () => { status: "pass", value: true, details: { - status: "attached", - configuration: { - id: 1325, - target_type: "organization", - name: "recommended settings", - enforcement: "enforced", - updated_at: "2026-06-01T00:00:00Z", - secret_scanning: "enabled" + security_and_analysis: { + secret_scanning: { + status: "enabled" + } } } }); @@ -54,10 +57,12 @@ describe("code security configuration claims", () => { const result = await evaluateWithMock( secretScanningEnabledClaim, mockGitHub({ - getCodeSecurityConfiguration: async () => ({ - status: "attached", - configuration: { - secret_scanning: "disabled" + getRepository: async () => ({ + id: 1, + security_and_analysis: { + secret_scanning: { + status: "disabled" + } } }) }) @@ -67,14 +72,37 @@ describe("code security configuration claims", () => { status: "fail", value: false, details: { - status: "attached", - configuration: { - secret_scanning: "disabled" + security_and_analysis: { + secret_scanning: { + status: "disabled" + } } } }); }); + it("returns unknown when repository security analysis does not include secret scanning", async () => { + const result = await evaluateWithMock( + secretScanningEnabledClaim, + mockGitHub({ + getRepository: async () => ({ + id: 1, + security_and_analysis: {} + }) + }) + ); + + expect(result.status).toBe("unknown"); + expect(result.error).toMatchObject({ + kind: "unexpected_response" + }); + expect(result.details).toEqual({ + security_and_analysis: { + secret_scanning: null + } + }); + }); + it("passes when secret scanning push protection is enabled", async () => { const result = await evaluateWithMock( secretScanningPushProtectionEnabledClaim, @@ -104,7 +132,7 @@ describe("code security configuration claims", () => { const result = await evaluateWithMock( dependabotAlertsEnabledClaim, mockGitHub({ - getCodeSecurityConfiguration: async () => attachedConfiguration + getVulnerabilityAlertsStatus: async () => "enabled" }) ); @@ -112,10 +140,7 @@ describe("code security configuration claims", () => { status: "pass", value: true, details: { - status: "attached", - configuration: { - dependabot_alerts: "enabled" - } + vulnerability_alerts: "enabled" } }); }); @@ -124,12 +149,12 @@ describe("code security configuration claims", () => { const result = await evaluateWithMock( dependabotAlertsEnabledClaim, mockGitHub({ - getCodeSecurityConfiguration: async () => ({ - status: "attached", - configuration: { - dependabot_alerts: "disabled" - } - }) + getVulnerabilityAlertsStatus: async () => { + throw new GitHubApiError("Not Found", { + status: 404, + kind: "not_found" + }); + } }) ); @@ -137,10 +162,7 @@ describe("code security configuration claims", () => { status: "fail", value: false, details: { - status: "attached", - configuration: { - dependabot_alerts: "disabled" - } + vulnerability_alerts: "disabled" } }); }); @@ -165,14 +187,39 @@ describe("code security configuration claims", () => { }); }); + it("fails when the attached configuration does not enable the dependency graph", async () => { + const result = await evaluateWithMock( + dependencyGraphEnabledClaim, + mockGitHub({ + getCodeSecurityConfiguration: async () => ({ + status: "attached", + configuration: { + dependency_graph: "disabled" + } + }) + }) + ); + + expect(result).toMatchObject({ + status: "fail", + value: false, + details: { + status: "attached", + configuration: { + dependency_graph: "disabled" + } + } + }); + }); + it("returns unknown when the repository is not attached to a configuration", async () => { const result = await evaluateWithMock( - secretScanningEnabledClaim, + secretScanningPushProtectionEnabledClaim, mockGitHub({ getCodeSecurityConfiguration: async () => ({ status: "detached", configuration: { - secret_scanning: "enabled" + secret_scanning_push_protection: "enabled" } }) }) @@ -189,7 +236,7 @@ describe("code security configuration claims", () => { it("returns null configuration details when a non-attached response has no object configuration", async () => { const result = await evaluateWithMock( - secretScanningEnabledClaim, + secretScanningPushProtectionEnabledClaim, mockGitHub({ getCodeSecurityConfiguration: async () => ({ status: "detached", @@ -207,7 +254,7 @@ describe("code security configuration claims", () => { it("returns unknown when the code security response is not an object", async () => { const result = await evaluateWithMock( - secretScanningEnabledClaim, + secretScanningPushProtectionEnabledClaim, mockGitHub({ getCodeSecurityConfiguration: async () => "unexpected" }) @@ -222,7 +269,7 @@ describe("code security configuration claims", () => { it("returns unknown when the attached configuration is missing", async () => { const result = await evaluateWithMock( - secretScanningEnabledClaim, + secretScanningPushProtectionEnabledClaim, mockGitHub({ getCodeSecurityConfiguration: async () => ({ status: "attached" @@ -242,7 +289,7 @@ describe("code security configuration claims", () => { it("returns unknown when the configured field is not a string", async () => { const result = await evaluateWithMock( - dependencyGraphEnabledClaim, + secretScanningPushProtectionEnabledClaim, mockGitHub({ getCodeSecurityConfiguration: async () => ({ status: "attached", @@ -252,7 +299,7 @@ describe("code security configuration claims", () => { name: false, enforcement: null, updated_at: {}, - dependency_graph: true + secret_scanning_push_protection: true } }) }) @@ -270,14 +317,14 @@ describe("code security configuration claims", () => { name: null, enforcement: null, updated_at: null, - dependency_graph: null + secret_scanning_push_protection: null } }); }); it("returns unknown when GitHub returns no content", async () => { const result = await evaluateWithMock( - secretScanningEnabledClaim, + secretScanningPushProtectionEnabledClaim, mockGitHub({ getCodeSecurityConfiguration: async () => ({ status: "no_content" @@ -300,7 +347,7 @@ describe("code security configuration claims", () => { const result = await evaluateWithMock( secretScanningEnabledClaim, mockGitHub({ - getCodeSecurityConfiguration: async () => { + getRepository: async () => { throw new GitHubApiError("Forbidden", { status: 403, kind: "forbidden" diff --git a/test/github/api-usage-policy.test.ts b/test/github/api-usage-policy.test.ts index 6f1ae59..f9b797d 100644 --- a/test/github/api-usage-policy.test.ts +++ b/test/github/api-usage-policy.test.ts @@ -18,7 +18,8 @@ const allowedRoutes = new Set([ "GET /repos/{owner}/{repo}/code-security-configuration", "GET /repos/{owner}/{repo}/immutable-releases", "GET /repos/{owner}/{repo}/installation", - "GET /repos/{owner}/{repo}/rules/branches/{branch}" + "GET /repos/{owner}/{repo}/rules/branches/{branch}", + "GET /repos/{owner}/{repo}/vulnerability-alerts" ]); describe("GitHub API usage policy", () => { diff --git a/test/github/client.test.ts b/test/github/client.test.ts index 61f60d9..f71cc73 100644 --- a/test/github/client.test.ts +++ b/test/github/client.test.ts @@ -71,6 +71,17 @@ describe("GitHubRestClient", () => { }); }); + it("getVulnerabilityAlertsStatus treats a successful status response as enabled", async () => { + const request = vi.fn(async () => ({ data: undefined, status: 204 })); + const client = new GitHubRestClient(request as unknown as GitHubRequest); + + await expect(client.getVulnerabilityAlertsStatus("OWNER", "REPO")).resolves.toBe("enabled"); + expect(request).toHaveBeenCalledWith("GET /repos/{owner}/{repo}/vulnerability-alerts", { + owner: "OWNER", + repo: "REPO" + }); + }); + it("getBranchRules requests the branch rules route with pagination", async () => { const { client, request } = clientReturning([]); diff --git a/test/github/installations.test.ts b/test/github/installations.test.ts index 975817e..f6765b5 100644 --- a/test/github/installations.test.ts +++ b/test/github/installations.test.ts @@ -27,6 +27,7 @@ function fakeGitHub( getImmutableReleases: vi.fn(), getActionsPermissions: vi.fn(), getCodeSecurityConfiguration: vi.fn(), + getVulnerabilityAlertsStatus: vi.fn(), getBranchRules: vi.fn() }; } diff --git a/test/support/mock-github.ts b/test/support/mock-github.ts index e94bce3..0240101 100644 --- a/test/support/mock-github.ts +++ b/test/support/mock-github.ts @@ -15,6 +15,7 @@ export function mockGitHub(overrides: Partial): GitHubClient { getImmutableReleases: unmocked, getActionsPermissions: unmocked, getCodeSecurityConfiguration: unmocked, + getVulnerabilityAlertsStatus: unmocked, getBranchRules: unmocked, ...overrides }; From 523d4d56724fca8166411aea51f24af3d96c99ed Mon Sep 17 00:00:00 2001 From: Really Him Date: Tue, 2 Jun 2026 22:03:32 -0400 Subject: [PATCH 4/4] feat: add policy evidence model --- README.md | 18 +-- docs/ADR/0001-policy-evidence-model.md | 84 +++++++++++++ docs/ADR/0002-read-only-policy-surface.md | 126 ++++++++++++++++++++ docs/claim-semantics.md | 16 +-- src/claims/code-security-configuration.ts | 63 ++++++++-- src/claims/immutable-releases.ts | 13 +- src/claims/result.ts | 25 +++- src/claims/sha-pinning-required.ts | 15 ++- src/claims/signed-commits-required.ts | 22 +++- src/claims/types.ts | 16 +++ test/badges.test.ts | 9 +- test/cache.test.ts | 1 + test/claims/signed-commits-required.test.ts | 10 +- test/routes.test.ts | 5 +- test/server/claim-service.test.ts | 1 + test/webhook-routes.test.ts | 1 + 16 files changed, 373 insertions(+), 52 deletions(-) create mode 100644 docs/ADR/0001-policy-evidence-model.md create mode 100644 docs/ADR/0002-read-only-policy-surface.md diff --git a/README.md b/README.md index 320ff2e..068a548 100644 --- a/README.md +++ b/README.md @@ -1,16 +1,16 @@ # PolicyChecks -[![Immutable releases](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/immutable-releases.svg)](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/immutable-releases/proof.json) [![SHA-pinned actions](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/sha-pinning-required.svg)](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/sha-pinning-required/proof.json) [![Secret scanning](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/secret-scanning-enabled.svg)](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/secret-scanning-enabled/proof.json) [![Dependabot alerts](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/dependabot-alerts-enabled.svg)](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/dependabot-alerts-enabled/proof.json) [![Dependency graph](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/dependency-graph-enabled.svg)](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/dependency-graph-enabled/proof.json) +[![Immutable releases](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/immutable-releases.svg)](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/immutable-releases/proof.json) [![SHA pinning](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/sha-pinning-required.svg)](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/sha-pinning-required/proof.json) [![Secret scanning](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/secret-scanning-enabled.svg)](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/secret-scanning-enabled/proof.json) [![Dependabot alerts](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/dependabot-alerts-enabled.svg)](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/dependabot-alerts-enabled/proof.json) [![Dependency graph](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/dependency-graph-enabled.svg)](https://policychecks.reponomics.org/github/reponomics/PolicyChecks/dependency-graph-enabled/proof.json) PolicyChecks is a GitHub App-backed badge service and validation endpoint for repository settings that ordinary public badge services cannot verify. It exposes badge SVG, Shields-compatible JSON, and proof JSON endpoints for repository administration and security checks. This gives maintainers a convenient way to show that a project not only follows best practices, but that these practices are enforced policies at the repository settings level. This fills a modest gap in the badge ecosystem between excellent services like shields.io (which does not have the permissions to report on these facts) and OSSF Scorecard (which does take into account many of these same conditions, but does not expose individual setting-level endpoints). -| Check | Claim ID | Passing result | Other results | -| --- | --- | --- | --- | -| Immutable releases | `immutable-releases` | `enabled` | `disabled` or `unknown` | -| Full SHA-pinned GitHub Actions | `sha-pinning-required` | `required` | `not required` or `unknown` | -| Secret scanning | `secret-scanning-enabled` | `enabled` | `disabled` or `unknown` | -| Dependabot alerts | `dependabot-alerts-enabled` | `enabled` | `disabled` or `unknown` | -| Dependency graph | `dependency-graph-enabled` | `enabled` | `disabled` or `unknown` | +| Check | Claim ID | Passing result | Other results | +| ------------------ | --------------------------- | -------------- | --------------------------- | +| Immutable releases | `immutable-releases` | `enforced` | `not enforced` or `unknown` | +| SHA pinning | `sha-pinning-required` | `enforced` | `not enforced` or `unknown` | +| Secret scanning | `secret-scanning-enabled` | `enforced` | `not enforced` or `unknown` | +| Dependabot alerts | `dependabot-alerts-enabled` | `enforced` | `not enforced` or `unknown` | +| Dependency graph | `dependency-graph-enabled` | `enforced` | `not enforced` or `unknown` | ## Endpoints @@ -28,7 +28,7 @@ Use the SVG endpoint for badges, the Shields-compatible JSON endpoint for badge ```markdown [![Immutable releases](https://policychecks.reponomics.org/github/OWNER/REPO/immutable-releases.svg)](https://policychecks.reponomics.org/github/OWNER/REPO/immutable-releases/proof.json) -[![SHA-pinned actions](https://policychecks.reponomics.org/github/OWNER/REPO/sha-pinning-required.svg)](https://policychecks.reponomics.org/github/OWNER/REPO/sha-pinning-required/proof.json) +[![SHA pinning](https://policychecks.reponomics.org/github/OWNER/REPO/sha-pinning-required.svg)](https://policychecks.reponomics.org/github/OWNER/REPO/sha-pinning-required/proof.json) [![Secret scanning](https://policychecks.reponomics.org/github/OWNER/REPO/secret-scanning-enabled.svg)](https://policychecks.reponomics.org/github/OWNER/REPO/secret-scanning-enabled/proof.json) diff --git a/docs/ADR/0001-policy-evidence-model.md b/docs/ADR/0001-policy-evidence-model.md new file mode 100644 index 0000000..cf2cab5 --- /dev/null +++ b/docs/ADR/0001-policy-evidence-model.md @@ -0,0 +1,84 @@ +# ADR 0001: Policy Evidence Model + +## Status + +Accepted + +## Context + +PolicyChecks badge labels are intentionally short. They name the security posture a maintainer wants to show, not every GitHub API surface that can prove it. + +Some GitHub settings can be reported directly at repository scope. Other settings may be governed by organization or enterprise policy, such as an attached code security configuration. If badge names try to encode every evidence source, they become too qualified to be useful. If proof responses hide the evidence source, users cannot tell why a badge passed, failed, or returned `unknown`. + +PolicyChecks also needs to avoid fuzzy claims. It should report GitHub settings and policy surfaces, not infer posture from repository files, generated artifacts, or indirect feature availability. + +## Decision + +Badge labels remain user-facing and concise: + +- `dependency graph` +- `secret scanning` +- `Dependabot alerts` +- `SHA pinning` + +Badge result messages use policy language: `enforced`, `not enforced`, or `unknown`. GitHub-native values such as `enabled`, `disabled`, `required`, or `not_set` belong in proof details, not in public badge messages. + +Proof JSON carries the qualification. Each proof result records the GitHub evidence source, the scope where the evidence was observed, and whether GitHub reported central enforcement. + +For personal repositories, checks are evaluated at repository scope. + +For organization repositories, checks may be satisfied by repository settings or by organization-managed policy/configuration that applies to the repository. Organization installs should request the permissions needed to inspect those policy surfaces. If an installer cannot approve those permissions, PolicyChecks should report `unknown` rather than making weaker claims under the same badge name. + +Example proof evidence: + +```json +{ + "evidence": { + "scope": "organization", + "source": "attached_code_security_configuration", + "enforcement": "enforced" + } +} +``` + +## Evidence Sources + +| Source | Meaning | Example | +| --- | --- | --- | +| `repository_setting` | GitHub returned a repository-level setting or repository-scoped status endpoint. | `GET /repos/{owner}/{repo}/actions/permissions` | +| `active_branch_rules` | GitHub returned active rules that apply to a repository branch. | `GET /repos/{owner}/{repo}/rules/branches/{branch}` | +| `attached_code_security_configuration` | GitHub returned a code security configuration that manages the repository. | `GET /repos/{owner}/{repo}/code-security-configuration` | +| `unavailable` | PolicyChecks could not obtain interpretable evidence. | Installation, authorization, rate-limit, or unsupported response failures | + +## Evidence Scope + +| Scope | Meaning | +| -------------- | ------------------------------------------------------------------ | +| `repository` | The proof comes from a repository setting/status surface. | +| `organization` | The proof comes from an organization-managed policy/configuration. | +| `enterprise` | The proof comes from an enterprise-managed policy/configuration. | +| `unknown` | GitHub did not provide a usable scope. | + +## Enforcement + +`enforcement` is only included when GitHub exposes enforcement status for the evidence source. + +For repository-local settings, absence of `enforcement` means PolicyChecks verified the setting value but did not verify who can change it. + +For attached code security configurations, `enforcement` is copied from GitHub's configuration response when available. A pass result for a centrally managed configuration means GitHub returned the configured value; enforcement status still needs to be read separately from `evidence.enforcement` and `details.configuration.enforcement`. + +## Consequences + +- README stays a summary; technical semantics live in ADRs and claim documentation. +- Badge names do not need org/repo qualifiers. +- Proof JSON must make the evidence source explicit. +- Claims based on generated artifacts, repository file inspection, or indirect feature availability are out of scope unless a later ADR changes this rule. +- Organization governance checks may require broader permissions than repository-only checks. + +## Claim Design Rules + +1. Prefer direct settings/policy endpoints over file inspection or artifact generation. +2. Keep badge names short when the same posture can be proven by repository or organization evidence. +3. Put evidence source, scope, and enforcement in proof JSON. +4. Return `unknown` when GitHub does not expose the policy surface needed for a confident result. +5. Do not publish claims whose badge label would need to be so qualified that users cannot predict what it means. diff --git a/docs/ADR/0002-read-only-policy-surface.md b/docs/ADR/0002-read-only-policy-surface.md new file mode 100644 index 0000000..925a19c --- /dev/null +++ b/docs/ADR/0002-read-only-policy-surface.md @@ -0,0 +1,126 @@ +# ADR 0002: Read-Only Policy Surface + +## Status + +Accepted + +## Context + +PolicyChecks is a public badge service. The app may need privileged read access to observe repository policy surfaces, but it should not request permissions that allow it to change those policies. + +GitHub exposes policy state through several API families with different permission costs: + +- repository settings endpoints; +- code security configuration endpoints; +- repository metadata; +- repository and branch rules endpoints; +- organization-level policy endpoints. + +The product question is whether PolicyChecks remains useful if it refuses write-policy permissions, especially repository or organization `Administration: Write`. + +## Decision + +PolicyChecks will not request write-policy permissions for badge evaluation. + +Specifically, PolicyChecks should not request repository `Administration: Write` or organization `Administration: Write` merely to strengthen a badge claim. Those permissions allow the app to alter rulesets or other policy surfaces, which is a different trust posture from observing policy state. + +PolicyChecks may request read permissions that expose settings and policy configuration. Repository `Administration: Read` remains acceptable for repository-scoped settings. Organization `Administration: Read` may be acceptable for organization-managed policy surfaces if a later release supports organization installs and documents that scope clearly. Repository `Metadata: Read` is treated as the implicit GitHub App baseline: it appears in endpoint permission tables, but it is not a separate product permission ask once the app is installed. + +Ruleset-derived claims are split into two product categories: + +1. **Active applicable rule claims.** These can be based on GitHub returning active rules that apply to a repository or branch. They are viable under read-only permissions. +2. **Configured-bypass claims.** These require visibility into `bypass_actors`, meaning actors explicitly configured as exceptions inside a ruleset. GitHub does not return `bypass_actors` unless the caller has write access to the ruleset. These claims are out of scope unless GitHub later exposes bypass visibility through read-only permissions. + +## Permission Matrix + +| Product surface | Representative GitHub endpoint | Permission boundary | Lookup shape | Product judgment | +| --- | --- | --- | --- | --- | +| Repository identity and default branch | `GET /repos/{owner}/{repo}` | Implicit repository `Metadata: Read` baseline; public resources may be unauthenticated | One request | Useful support surface, not usually a badge by itself. | +| Repository security metadata, such as `security_and_analysis` | `GET /repos/{owner}/{repo}` | Endpoint is metadata-readable, but GitHub requires repository admin, organization owner, or security manager visibility for the `security_and_analysis` block | One request | Viable for direct security feature checks when the app has sufficient repository visibility. | +| Immutable releases | `GET /repos/{owner}/{repo}/immutable-releases` | Repository `Administration: Read` | One request | Viable. Direct repository setting with documented enabled/not-enabled responses. | +| Actions SHA pinning | `GET /repos/{owner}/{repo}/actions/permissions` | Repository `Administration: Read` | One request | Viable. Direct repository Actions policy field. | +| Dependabot vulnerability alerts | `GET /repos/{owner}/{repo}/vulnerability-alerts` | Repository `Administration: Read` | One request | Viable. Direct repository setting endpoint. | +| Dependabot security updates | `GET /repos/{owner}/{repo}/automated-security-fixes` | Repository `Administration: Read` | One request | Candidate. Direct repository setting endpoint, separate from vulnerability alerts. | +| Code security configuration attached to a repository | `GET /repos/{owner}/{repo}/code-security-configuration` | Repository `Administration: Read`; authenticated caller must be an administrator or security manager for the organization | One request | Viable when a configuration is attached. Gives several code security feature fields and central enforcement status. | +| Code scanning default setup | `GET /repos/{owner}/{repo}/code-scanning/default-setup` | Repository `Administration: Read` | One request | Candidate. Direct default-setup configuration surface; availability depends on code scanning/GHAS eligibility. | +| Active branch rules | `GET /repos/{owner}/{repo}/rules/branches/{branch}` | Implicit repository `Metadata: Read` baseline; public resources may be unauthenticated | One paginated request after default branch is known | Strong candidate. Returns active rules that apply to the branch, including repository- and organization-sourced rules. Does not return disabled or evaluate-mode rules. | +| Repository ruleset summaries | `GET /repos/{owner}/{repo}/rulesets` | Implicit repository `Metadata: Read` baseline; public resources may be unauthenticated | One paginated request | Candidate support surface. Useful for inventory, source scope, and enforcement state; less direct than branch rules for branch-specific claims. | +| Repository ruleset details | `GET /repos/{owner}/{repo}/rulesets/{ruleset_id}` | Implicit repository `Metadata: Read` baseline; public resources may be unauthenticated | One request per ruleset | Candidate when full rule parameters are needed. `bypass_actors` is withheld unless the caller has write access to the ruleset. | +| Organization code security configurations | `GET /orgs/{org}/code-security/configurations/{configuration_id}` and related read endpoints | Organization `Administration: Read`; caller must be organization administrator or security manager | One or paginated org request | Candidate for organization product tier, not required for repository-only badges unless PolicyChecks reports org policy directly. | +| Organization ruleset details with configured bypass actors | `GET /orgs/{org}/rulesets/{ruleset_id}` | Organization `Administration: Write` | One request per ruleset | Out of scope. Write-policy permission is too broad for badge evaluation. | +| Repository ruleset details with configured bypass actors | `GET /repos/{owner}/{repo}/rulesets/{ruleset_id}` plus write access to the ruleset | Repository or org write-policy authority, depending on source ruleset | One request per ruleset | Out of scope for configured-bypass claims unless GitHub adds read-only bypass visibility. | +| Repository file inspection | `GET /repos/{owner}/{repo}/contents/{path}` or git blob/tree APIs | Repository `Contents: Read` for private repositories | One or more repository-content requests | Out of scope for policy badges by default. Contents access supports code/config inference, not direct GitHub policy surfaces. | + +## Read-Only Product Surface + +If PolicyChecks never accepts write-policy permissions, the useful product surface is still substantial: + +- immutable releases; +- Actions SHA pinning; +- Dependabot vulnerability alerts; +- Dependabot security updates; +- secret scanning; +- secret scanning push protection when represented in an attached code security configuration; +- dependency graph; +- dependency graph automatic submission when represented in an attached code security configuration; +- code scanning default setup; +- active signed-commit rules for a branch; +- active pull-request-required rules for a branch; +- active code owner review rules for a branch; +- active stale-review-dismissal rules for a branch; +- active required-status-check rules for a branch; +- active strict status-check rules for a branch; +- active non-fast-forward rules, meaning force pushes are blocked; +- active deployment-required rules; +- active commit message, author email, committer email, branch/tag name, file path, file extension, and file size rules. + +Most of that surface is available through direct settings or policy endpoints. It does not require reading workflow files, package manifests, release artifacts, or repository contents. + +## Bypass And Continuity Boundary + +Ruleset webhooks can tell PolicyChecks that rulesets changed and that cached results should be invalidated. They do not prove who can bypass a rule. + +There are three distinct bypass concepts: + +1. **Policy authority bypass.** Repository administrators, organization owners, security managers, or other policy owners may be able to change a setting, perform an action, and change the setting back. This is always possible for some actor unless PolicyChecks also proves historical continuity and immutable governance. PolicyChecks does not make continuity claims. +2. **Configured bypass actors.** Rulesets may include explicit `bypass_actors` that exempt selected actors from the rule. GitHub's repository ruleset documentation says `bypass_actors` is only returned when the caller has write access to the ruleset. Organization ruleset detail reads require organization `Administration: Write`. +3. **Runtime bypass events.** Some features, such as secret scanning push protection, can produce audit or alert events when a user bypasses protection. Those events are operational history, not current policy configuration. + +PolicyChecks reports observed current policy state. It does not claim that the policy was continuously enabled over time, that no authorized administrator could disable it, or that no configured bypass actors exist unless the proof explicitly says so. + +Therefore PolicyChecks must not publish claims such as "cannot be bypassed," "no bypass actors," or "continuously enforced" under the read-only product boundary. + +For ruleset-backed claims, proof JSON should expose bypass visibility explicitly: + +```json +{ + "details": { + "matching_rule_types": ["required_signatures"], + "bypass_visibility": "unavailable" + } +} +``` + +## Consequences + +- `Contents: Read` is not the next permission to request for policy-surface growth. It enables file inspection, not ruleset truth. +- Repository `Administration: Read` supports the current direct-setting product. +- Implicit repository `Metadata: Read` is enough for many future active-rule claims and should not be described as an additional installer-facing permission ask. +- Organization `Administration: Read` may be needed for a future organization policy tier. +- Write-policy permissions remain out of scope even if they would reveal configured bypass actors. +- Claims that depend on absence of configured bypass actors must remain unpublished or return `unknown` until GitHub exposes bypass visibility through read-only access. +- Claims that depend on historical continuity must remain unpublished unless PolicyChecks adds an audited continuity model. + +## Documentation References + +- [Get a repository](https://docs.github.com/en/rest/repos/repos#get-a-repository) +- [Check if immutable releases are enabled for a repository](https://docs.github.com/en/rest/repos/repos#check-if-immutable-releases-are-enabled-for-a-repository) +- [Get GitHub Actions permissions for a repository](https://docs.github.com/en/rest/actions/permissions#get-github-actions-permissions-for-a-repository) +- [Check if vulnerability alerts are enabled for a repository](https://docs.github.com/en/rest/repos/repos#check-if-vulnerability-alerts-are-enabled-for-a-repository) +- [Get the code security configuration associated with a repository](https://docs.github.com/en/rest/code-security/configurations#get-the-code-security-configuration-associated-with-a-repository) +- [Get a code scanning default setup configuration](https://docs.github.com/en/rest/code-scanning/code-scanning#get-a-code-scanning-default-setup-configuration) +- [Get rules for a branch](https://docs.github.com/en/rest/repos/rules#get-rules-for-a-branch) +- [Get all repository rulesets](https://docs.github.com/en/rest/repos/rules#get-all-repository-rulesets) +- [Get a repository ruleset](https://docs.github.com/en/rest/repos/rules#get-a-repository-ruleset) +- [Get an organization repository ruleset](https://docs.github.com/en/rest/orgs/rules#get-an-organization-repository-ruleset) +- [Permissions required for GitHub Apps](https://docs.github.com/en/rest/authentication/permissions-required-for-github-apps) diff --git a/docs/claim-semantics.md b/docs/claim-semantics.md index c2b6109..6352f1c 100644 --- a/docs/claim-semantics.md +++ b/docs/claim-semantics.md @@ -2,8 +2,8 @@ PolicyChecks uses a cautious three-state result model: -- `pass`: GitHub returned a documented response that directly supports the claim. -- `fail`: GitHub returned a documented response that directly contradicts the claim. +- `pass`: GitHub returned a documented response showing that the policy posture is enforced. +- `fail`: GitHub returned a documented response showing that the policy posture is not enforced. - `unknown`: GitHub access, rate limits, response shape, endpoint semantics, or continuity caveats prevent a confident `pass` or `fail`. `unknown` is not a failure assertion. It means PolicyChecks did not have enough reliable evidence to make the claim either way. @@ -20,6 +20,8 @@ Every proof response includes the requested repository identity: } ``` +Every proof response also includes an `evidence` object. `evidence.source` identifies the GitHub policy surface, `evidence.scope` identifies whether the evidence is repository-, organization-, or enterprise-scoped, and `evidence.enforcement` is included when GitHub reports central enforcement status. The evidence model is defined in [`docs/ADR/0001-policy-evidence-model.md`](ADR/0001-policy-evidence-model.md). + ## Documentation References The tables below use these GitHub documentation references as their audit trail: @@ -147,7 +149,7 @@ Before adding a new public badge, document: ## Unsupported Claim Semantics -Some repository settings require additional context before PolicyChecks can assign an unqualified `pass` or `fail` result. Ruleset-derived checks are the main example: GitHub can report that a rule applies to a branch, but bypass actors and exemption paths affect whether that rule is enforceable in practice. PolicyChecks does not publish ruleset-enforcement claims unless the proof can also account for those bypass conditions. +Some repository settings require additional context before PolicyChecks can assign an unqualified `pass` or `fail` result. Ruleset-derived checks are the main example: GitHub can report that a rule applies to a branch, but configured bypass actors, administrator policy changes, and continuity history are separate concerns. PolicyChecks reports observed current policy state; it does not claim historical continuity or impossibility of administrator override unless a proof explicitly adds that evidence. #### `signed-commits-required` @@ -164,13 +166,13 @@ GET /repos/{owner}/{repo}/rules/branches/{branch} | GitHub response or value | PolicyChecks status | Proof details | Documentation basis | Judgment | | --- | --- | --- | --- | --- | -| Repository metadata includes a default branch, and branch rules response is an array containing `type: required_signatures` | `pass` | `branch`, `matching_rule_types` | `branch-rules-doc` documents that the endpoint returns all active rules applying to the branch. | Direct evidence that an active signed-commit rule applies to the default branch. | -| Repository metadata includes a default branch, and branch rules response is an array without `type: required_signatures` | `fail` | `branch`, `matching_rule_types` | `branch-rules-doc` documents that all active applicable rules are returned and `evaluate`/`disabled` rulesets are excluded. | Safe to say no active applicable signed-commit rule was returned for the default branch. | -| Repository metadata has no usable default branch | `unknown` | `branch: null`, `matching_rule_types: []` | Repository metadata did not provide the branch needed to call `branch-rules-doc` endpoint. | The service does not know which branch to evaluate. | +| Repository metadata includes a default branch, and branch rules response is an array containing `type: required_signatures` | `pass` | `branch`, `matching_rule_types`, `bypass_visibility: unavailable` | `branch-rules-doc` documents that the endpoint returns all active rules applying to the branch. | Direct evidence that an active signed-commit rule applies to the default branch. | +| Repository metadata includes a default branch, and branch rules response is an array without `type: required_signatures` | `fail` | `branch`, `matching_rule_types`, `bypass_visibility: unavailable` | `branch-rules-doc` documents that all active applicable rules are returned and `evaluate`/`disabled` rulesets are excluded. | Safe to say no active applicable signed-commit rule was returned for the default branch. | +| Repository metadata has no usable default branch | `unknown` | `branch: null`, `matching_rule_types: []`, `bypass_visibility: unavailable` | Repository metadata did not provide the branch needed to call `branch-rules-doc` endpoint. | The service does not know which branch to evaluate. | | Branch rules response is not an array | `unknown` | error details | `branch-rules-doc` documents an array response. | The service cannot safely interpret the response. | | Any `404` from repository metadata or branch rules | `unknown` | error details | `branch-rules-doc` does not document `404` as a disabled/not-required setting value. | Not safe to assert disabled from this response. | -Caveat: this claim is ruleset-derived. It says GitHub returned an active applicable `required_signatures` rule for the default branch. It does not yet prove that no bypass actors or exemption paths exist. +Caveat: this claim is ruleset-derived. It says GitHub returned an active applicable `required_signatures` rule for the default branch. It does not prove historical continuity, impossibility of administrator override, or absence of configured bypass actors. #### `secret-scanning-push-protection-enabled` diff --git a/src/claims/code-security-configuration.ts b/src/claims/code-security-configuration.ts index 101f46e..222fd30 100644 --- a/src/claims/code-security-configuration.ts +++ b/src/claims/code-security-configuration.ts @@ -1,6 +1,12 @@ import { GitHubApiError, publicMessage, toPublicClaimError } from "../github/errors.js"; -import { isRecord, makeClaimResult, makeUnknownResult, resultInput } from "./result.js"; -import type { ClaimDefinition, ClaimEvaluationInput } from "./types.js"; +import { + isRecord, + makeClaimResult, + makeUnknownResult, + repositorySettingEvidence, + resultInput +} from "./result.js"; +import type { ClaimDefinition, ClaimEvaluationInput, ClaimEvidence } from "./types.js"; type CodeSecurityField = | "secret_scanning" @@ -19,8 +25,8 @@ const codeSecurityConfigurationEndpoint = "GET /repos/{owner}/{repo}/code-securi export const secretScanningEnabledClaim: ClaimDefinition = { id: "secret-scanning-enabled", label: "secret scanning", - passMessage: "enabled", - failMessage: "disabled", + passMessage: "enforced", + failMessage: "not enforced", unknownMessage: "unknown", source: { provider: "github", @@ -28,6 +34,7 @@ export const secretScanningEnabledClaim: ClaimDefinition = { endpoint: "GET /repos/{owner}/{repo}", fields: ["security_and_analysis.secret_scanning.status"] }, + evidence: repositorySettingEvidence, async evaluate(input: ClaimEvaluationInput) { try { const repository = await input.github.getRepository(input.owner, input.repo); @@ -56,8 +63,8 @@ export const secretScanningPushProtectionEnabledClaim = codeSecurityConfiguratio export const dependabotAlertsEnabledClaim: ClaimDefinition = { id: "dependabot-alerts-enabled", label: "Dependabot alerts", - passMessage: "enabled", - failMessage: "disabled", + passMessage: "enforced", + failMessage: "not enforced", unknownMessage: "unknown", source: { provider: "github", @@ -65,6 +72,7 @@ export const dependabotAlertsEnabledClaim: ClaimDefinition = { endpoint: "GET /repos/{owner}/{repo}/vulnerability-alerts", fields: ["HTTP 204", "HTTP 404"] }, + evidence: repositorySettingEvidence, async evaluate(input: ClaimEvaluationInput) { try { const status = await input.github.getVulnerabilityAlertsStatus(input.owner, input.repo); @@ -98,8 +106,8 @@ function codeSecurityConfigurationClaim(options: CodeSecurityClaimOptions): Clai const definition: ClaimDefinition = { id: options.id, label: options.label, - passMessage: "enabled", - failMessage: "disabled", + passMessage: "enforced", + failMessage: "not enforced", unknownMessage: "unknown", source: { provider: "github", @@ -107,6 +115,10 @@ function codeSecurityConfigurationClaim(options: CodeSecurityClaimOptions): Clai endpoint: codeSecurityConfigurationEndpoint, fields: ["status", `configuration.${options.field}`, "configuration.enforcement"] }, + evidence: { + scope: "unknown", + source: "attached_code_security_configuration" + }, async evaluate(input: ClaimEvaluationInput) { try { const data = await input.github.getCodeSecurityConfiguration(input.owner, input.repo); @@ -218,10 +230,18 @@ function evaluateCodeSecurityConfiguration( const enabled = value === "enabled"; - return makeClaimResult(definition, resultInput(input), enabled ? "pass" : "fail", enabled, { - status, - configuration: configurationDetails(configuration, field) - }); + return makeClaimResult( + definition, + resultInput(input), + enabled ? "pass" : "fail", + enabled, + { + status, + configuration: configurationDetails(configuration, field) + }, + undefined, + codeSecurityConfigurationEvidence(configuration) + ); } function unexpected( @@ -254,3 +274,22 @@ function configurationDetails(configuration: unknown, field: CodeSecurityField) [field]: typeof configuration[field] === "string" ? configuration[field] : null }; } + +function codeSecurityConfigurationEvidence(configuration: Record): ClaimEvidence { + const enforcement = + typeof configuration.enforcement === "string" ? configuration.enforcement : undefined; + + return { + scope: codeSecurityConfigurationScope(configuration.target_type), + source: "attached_code_security_configuration", + ...(enforcement !== undefined ? { enforcement } : {}) + }; +} + +function codeSecurityConfigurationScope(value: unknown): ClaimEvidence["scope"] { + if (value === "organization" || value === "enterprise") { + return value; + } + + return "unknown"; +} diff --git a/src/claims/immutable-releases.ts b/src/claims/immutable-releases.ts index 278ff5c..021bd5a 100644 --- a/src/claims/immutable-releases.ts +++ b/src/claims/immutable-releases.ts @@ -1,12 +1,18 @@ import { GitHubApiError, publicMessage, toPublicClaimError } from "../github/errors.js"; -import { isRecord, makeClaimResult, makeUnknownResult, resultInput } from "./result.js"; +import { + isRecord, + makeClaimResult, + makeUnknownResult, + repositorySettingEvidence, + resultInput +} from "./result.js"; import type { ClaimDefinition, ClaimEvaluationInput } from "./types.js"; export const immutableReleasesClaim: ClaimDefinition = { id: "immutable-releases", label: "immutable releases", - passMessage: "enabled", - failMessage: "disabled", + passMessage: "enforced", + failMessage: "not enforced", unknownMessage: "unknown", source: { provider: "github", @@ -14,6 +20,7 @@ export const immutableReleasesClaim: ClaimDefinition = { endpoint: "GET /repos/{owner}/{repo}/immutable-releases", fields: ["enabled", "enforced_by_owner"] }, + evidence: repositorySettingEvidence, async evaluate(input: ClaimEvaluationInput) { try { const data = await input.github.getImmutableReleases(input.owner, input.repo); diff --git a/src/claims/result.ts b/src/claims/result.ts index dd5787d..c55c5d8 100644 --- a/src/claims/result.ts +++ b/src/claims/result.ts @@ -1,5 +1,6 @@ import type { ClaimDefinition, + ClaimEvidence, ClaimError, ClaimEvaluationInput, ClaimResult, @@ -22,7 +23,8 @@ export function makeClaimResult( status: ClaimStatus, value: boolean | null, details: Record, - error?: ClaimError + error?: ClaimError, + evidence: ClaimEvidence = definition.evidence ?? unavailableEvidence ): ClaimResult { return { claim: definition.id, @@ -32,6 +34,7 @@ export function makeClaimResult( status, value, source: definition.source, + evidence, checked_at: checkedAt(input.now), details, ...(error ? { error } : {}) @@ -50,9 +53,10 @@ export function makeUnknownResult( definition: ClaimDefinition, input: ResultInput, error: ClaimError, - details: Record = {} + details: Record = {}, + evidence: ClaimEvidence = unavailableEvidence ): ClaimResult { - return makeClaimResult(definition, input, "unknown", null, details, error); + return makeClaimResult(definition, input, "unknown", null, details, error, evidence); } export function resultInput(input: ClaimEvaluationInput): ResultInput { @@ -66,3 +70,18 @@ export function resultInput(input: ClaimEvaluationInput): ResultInput { export function isRecord(value: unknown): value is Record { return typeof value === "object" && value !== null && !Array.isArray(value); } + +export const repositorySettingEvidence: ClaimEvidence = { + scope: "repository", + source: "repository_setting" +}; + +export const activeBranchRulesEvidence: ClaimEvidence = { + scope: "unknown", + source: "active_branch_rules" +}; + +export const unavailableEvidence: ClaimEvidence = { + scope: "unknown", + source: "unavailable" +}; diff --git a/src/claims/sha-pinning-required.ts b/src/claims/sha-pinning-required.ts index 0134346..0ab5142 100644 --- a/src/claims/sha-pinning-required.ts +++ b/src/claims/sha-pinning-required.ts @@ -1,12 +1,18 @@ import { publicMessage, toPublicClaimError } from "../github/errors.js"; -import { isRecord, makeClaimResult, makeUnknownResult, resultInput } from "./result.js"; +import { + isRecord, + makeClaimResult, + makeUnknownResult, + repositorySettingEvidence, + resultInput +} from "./result.js"; import type { ClaimDefinition, ClaimEvaluationInput } from "./types.js"; export const shaPinningRequiredClaim: ClaimDefinition = { id: "sha-pinning-required", - label: "SHA-pinned actions", - passMessage: "required", - failMessage: "not required", + label: "SHA pinning", + passMessage: "enforced", + failMessage: "not enforced", unknownMessage: "unknown", source: { provider: "github", @@ -14,6 +20,7 @@ export const shaPinningRequiredClaim: ClaimDefinition = { endpoint: "GET /repos/{owner}/{repo}/actions/permissions", fields: ["sha_pinning_required"] }, + evidence: repositorySettingEvidence, async evaluate(input: ClaimEvaluationInput) { try { const data = await input.github.getActionsPermissions(input.owner, input.repo); diff --git a/src/claims/signed-commits-required.ts b/src/claims/signed-commits-required.ts index d56dfdc..317b74b 100644 --- a/src/claims/signed-commits-required.ts +++ b/src/claims/signed-commits-required.ts @@ -1,12 +1,18 @@ import { publicMessage, toPublicClaimError } from "../github/errors.js"; -import { isRecord, makeClaimResult, makeUnknownResult, resultInput } from "./result.js"; +import { + activeBranchRulesEvidence, + isRecord, + makeClaimResult, + makeUnknownResult, + resultInput +} from "./result.js"; import type { ClaimDefinition, ClaimEvaluationInput } from "./types.js"; export const signedCommitsRequiredClaim: ClaimDefinition = { id: "signed-commits-required", label: "signed commits", - passMessage: "required", - failMessage: "not required", + passMessage: "enforced", + failMessage: "not enforced", unknownMessage: "unknown", source: { provider: "github", @@ -14,6 +20,7 @@ export const signedCommitsRequiredClaim: ClaimDefinition = { endpoint: "GET /repos/{owner}/{repo}/rules/branches/{branch}", fields: ["type"] }, + evidence: activeBranchRulesEvidence, async evaluate(input: ClaimEvaluationInput) { try { const repository = await input.github.getRepository(input.owner, input.repo); @@ -29,7 +36,8 @@ export const signedCommitsRequiredClaim: ClaimDefinition = { }, { branch: null, - matching_rule_types: [] + matching_rule_types: [], + bypass_visibility: "unavailable" } ); } @@ -46,7 +54,8 @@ export const signedCommitsRequiredClaim: ClaimDefinition = { }, { branch, - matching_rule_types: [] + matching_rule_types: [], + bypass_visibility: "unavailable" } ); } @@ -64,7 +73,8 @@ export const signedCommitsRequiredClaim: ClaimDefinition = { required, { branch, - matching_rule_types: matchingRuleTypes + matching_rule_types: matchingRuleTypes, + bypass_visibility: "unavailable" } ); } catch (error) { diff --git a/src/claims/types.ts b/src/claims/types.ts index c4d81eb..37edf79 100644 --- a/src/claims/types.ts +++ b/src/claims/types.ts @@ -22,6 +22,20 @@ export interface ClaimSource { fields: string[]; } +export type ClaimEvidenceScope = "repository" | "organization" | "enterprise" | "unknown"; + +export type ClaimEvidenceSource = + | "repository_setting" + | "active_branch_rules" + | "attached_code_security_configuration" + | "unavailable"; + +export interface ClaimEvidence { + scope: ClaimEvidenceScope; + source: ClaimEvidenceSource; + enforcement?: string; +} + export interface ClaimRepositoryIdentity { owner: string; repo: string; @@ -36,6 +50,7 @@ export interface ClaimResult { status: ClaimStatus; value: boolean | null; source: ClaimSource; + evidence: ClaimEvidence; checked_at: string; details: Record; error?: ClaimError; @@ -58,5 +73,6 @@ export interface ClaimDefinition { failMessage: string; unknownMessage: string; source: ClaimSource; + evidence?: ClaimEvidence; evaluate(input: ClaimEvaluationInput): Promise; } diff --git a/test/badges.test.ts b/test/badges.test.ts index 866897c..c64b5e6 100644 --- a/test/badges.test.ts +++ b/test/badges.test.ts @@ -9,8 +9,8 @@ describe("badge renderers", () => { it("renders Shields-compatible JSON", () => { expect(toShieldsJson(shaPinningRequiredClaim, result("pass"))).toEqual({ schemaVersion: 1, - label: "SHA-pinned actions", - message: "required", + label: "SHA pinning", + message: "enforced", color: "brightgreen" }); @@ -24,7 +24,7 @@ describe("badge renderers", () => { const definition = { ...shaPinningRequiredClaim, label: "SHA ", - failMessage: "not & required" + failMessage: "not & enforced" }; const svg = renderBadgeSvg(definition, { ...result("fail"), @@ -34,7 +34,7 @@ describe("badge renderers", () => { }); expect(svg).toContain("SHA <actions>"); - expect(svg).toContain("not & required"); + expect(svg).toContain("not & enforced"); expect(svg).not.toContain("