diff --git a/docs/policy-rfc-engine.md b/docs/policy-rfc-engine.md new file mode 100644 index 0000000000..8da1bbd04d --- /dev/null +++ b/docs/policy-rfc-engine.md @@ -0,0 +1,67 @@ +# Policy RFC Engine + +The Policy RFC Engine turns repeated ClawSweeper review and repair patterns into structured, reviewable policy proposals. It is an additive, manual tool: it reads durable local records and writes generated documentation/state under `results/policy-rfc/`. It does not mutate GitHub, dispatch repairs, close issues, change labels, or alter scheduler behavior. + +## Usage + +Build the project, then run: + +```sh +pnpm run policy-rfc -- --target-repo openclaw/openclaw --min-occurrences 5 +``` + +Useful options: + +- `--target-repo`: repository profile to scan, such as `openclaw/openclaw`. +- `--records-root`: local durable record root. Defaults to `records`. +- `--output-root`: generated proposal root. Defaults to `results/policy-rfc`. +- `--min-occurrences`: minimum repeated observations before an RFC is emitted. Defaults to `5`. + +## What It Reads + +The collector scans existing markdown and JSON records below `records//`. It tolerates missing directories, unreadable files, older markdown shapes, and malformed partial records by skipping what it cannot safely parse. + +The first version extracts repeated examples of: + +- file conflict types +- labels +- repair markers +- review verdict markers +- safe-close reasons +- automerge repair causes + +## What It Writes + +For each eligible pattern, the engine writes: + +- `results/policy-rfc//.md` +- `results/policy-rfc//.json` + +Markdown RFCs contain: + +- Title +- Status: Draft +- Summary +- Observed Pattern +- Evidence +- Proposed Policy +- Safety Constraints +- Non-Goals +- Rollout Plan +- Metrics +- Reversion Plan + +JSON proposals include the stable machine-readable fields needed for review automation or later dashboards: `id`, `title`, `status`, `pattern_type`, `evidence_items`, `confidence_score`, `proposed_conditions`, `proposed_action`, `safety_constraints`, `created_at`, and `source_records`. + +## Proposal-Only Boundary + +The engine intentionally stops at documentation/state. A generated RFC is evidence that a pattern may deserve a formal policy; it is not an executable rule. Any accepted proposal must be implemented separately, reviewed as normal code, and routed through ClawSweeper's existing conservative apply paths. + +This keeps the feature out of hot scheduler paths: + +- no GitHub mutation +- no automatic policy execution +- no changes to close/apply/automerge logic +- no extra review shard work +- no live GitHub scans in the scheduler critical path + diff --git a/package.json b/package.json index 2ea2f65bad..9f06e9646c 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,7 @@ "audit": "node dist/clawsweeper.js audit", "reconcile": "node dist/clawsweeper.js reconcile", "status": "node dist/clawsweeper.js status", + "policy-rfc": "node dist/policy-rfc/index.js", "commit-review": "node dist/commit-sweeper.js", "commit-reports": "node dist/commit-sweeper.js reports", "repair:validate": "node dist/repair/validate-all.js", diff --git a/src/policy-rfc/collector.ts b/src/policy-rfc/collector.ts new file mode 100644 index 0000000000..053a8dc6e6 --- /dev/null +++ b/src/policy-rfc/collector.ts @@ -0,0 +1,329 @@ +import { existsSync, readFileSync, readdirSync, statSync } from "node:fs"; +import { join, relative, sep } from "node:path"; + +import { repositoryProfileFor, repositoryProfileForSlug } from "../repository-profiles.js"; +import type { + PatternCollectorOptions, + PolicyPatternObservation, + PolicyPatternType, +} from "./types.js"; + +interface RecordCandidate { + absolutePath: string; + relativePath: string; + repoSlug: string; +} + +export function collectPolicyPatterns( + options: PatternCollectorOptions, +): PolicyPatternObservation[] { + const repoSlugs = targetRepoSlugs(options.targetRepo); + const candidates = recordCandidates(options.recordsRoot, repoSlugs); + const observations: PolicyPatternObservation[] = []; + + for (const candidate of candidates) { + const text = safeRead(candidate.absolutePath); + if (!text) continue; + observations.push(...observationsFromRecord(candidate, text)); + } + + return observations.sort(compareObservation); +} + +function targetRepoSlugs(targetRepo: string | undefined): Set | undefined { + if (!targetRepo) return undefined; + return new Set([repositoryProfileFor(targetRepo).slug]); +} + +function recordCandidates( + recordsRoot: string, + repoSlugs: Set | undefined, +): RecordCandidate[] { + if (!existsSync(recordsRoot)) return []; + const candidates: RecordCandidate[] = []; + for (const repoSlug of safeReadDir(recordsRoot).sort()) { + if (repoSlugs && !repoSlugs.has(repoSlug)) continue; + const repoRoot = join(recordsRoot, repoSlug); + if (!safeIsDirectory(repoRoot)) continue; + for (const absolutePath of walkFiles(repoRoot)) { + if (!absolutePath.endsWith(".md") && !absolutePath.endsWith(".json")) continue; + candidates.push({ + absolutePath, + relativePath: normalizePath(relative(recordsRoot, absolutePath)), + repoSlug, + }); + } + } + return candidates.sort((left, right) => left.relativePath.localeCompare(right.relativePath)); +} + +function walkFiles(root: string): string[] { + const files: string[] = []; + for (const name of safeReadDir(root).sort()) { + const fullPath = join(root, name); + if (safeIsDirectory(fullPath)) files.push(...walkFiles(fullPath)); + else files.push(fullPath); + } + return files; +} + +function observationsFromRecord( + candidate: RecordCandidate, + text: string, +): PolicyPatternObservation[] { + const repo = repoForSlug(candidate.repoSlug); + const item = itemFromPath(candidate.relativePath); + const observedAt = firstDate([ + frontMatterValue(text, "reviewed_at"), + frontMatterValue(text, "updated_at"), + frontMatterValue(text, "created_at"), + jsonStringValue(text, "reviewedAt"), + jsonStringValue(text, "updatedAt"), + jsonStringValue(text, "createdAt"), + ]); + const successfulOutcome = hasSuccessfulOutcome(text); + const observations: PolicyPatternObservation[] = []; + + for (const label of frontMatterStringArray(text, "labels")) { + observations.push( + observation(candidate, repo, item, observedAt, successfulOutcome, "label", label), + ); + } + for (const verdict of uniqueMatches(text, /clawsweeper-verdict:([a-z0-9_-]+)/gi)) { + observations.push( + observation(candidate, repo, item, observedAt, successfulOutcome, "review_verdict", verdict), + ); + } + for (const reason of [ + frontMatterValue(text, "close_reason"), + frontMatterValue(text, "closeReason"), + labeledLineValue(text, "close reason"), + labeledLineValue(text, "safe close reason"), + ]) { + if (reason) { + observations.push( + observation( + candidate, + repo, + item, + observedAt, + successfulOutcome, + "safe_close_reason", + reason, + ), + ); + } + } + for (const marker of [ + ...uniqueMatches(text, /clawsweeper-repair:([a-z0-9_-]+)/gi), + ...jsonStringValues(text, "repair_marker"), + ...jsonStringValues(text, "repairMarker"), + ]) { + observations.push( + observation(candidate, repo, item, observedAt, successfulOutcome, "repair_marker", marker), + ); + } + for (const cause of [ + ...jsonStringValues(text, "automerge_repair_cause"), + ...jsonStringValues(text, "automergeRepairCause"), + labeledLineValue(text, "automerge repair cause"), + ]) { + if (cause) { + observations.push( + observation( + candidate, + repo, + item, + observedAt, + successfulOutcome, + "automerge_repair_cause", + cause, + ), + ); + } + } + for (const conflictType of [ + ...jsonStringValues(text, "conflict_type"), + ...jsonStringValues(text, "conflictType"), + labeledLineValue(text, "conflict type"), + labeledLineValue(text, "file conflict type"), + ]) { + if (conflictType) { + observations.push( + observation( + candidate, + repo, + item, + observedAt, + successfulOutcome, + "file_conflict_type", + conflictType, + ), + ); + } + } + + return dedupeObservations(observations); +} + +function observation( + candidate: RecordCandidate, + repo: string, + item: string, + observedAt: string | undefined, + successfulOutcome: boolean, + patternType: PolicyPatternType, + rawValue: string, +): PolicyPatternObservation { + const value = normalizeValue(rawValue); + return { + patternType, + value, + repo, + item, + sourceRecord: `records/${candidate.relativePath}`, + observedAt, + successfulOutcome, + }; +} + +function dedupeObservations(observations: PolicyPatternObservation[]): PolicyPatternObservation[] { + const seen = new Set(); + return observations.filter((candidate) => { + if (!candidate.value) return false; + const key = `${candidate.patternType}\0${candidate.value}\0${candidate.sourceRecord}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} + +function repoForSlug(slug: string): string { + return repositoryProfileForSlug(slug)?.targetRepo ?? slug.replace("-", "/"); +} + +function itemFromPath(relativePath: string): string { + const match = relativePath.match(/\/items\/([^/.]+)\./); + return match?.[1] ? `#${match[1]}` : relativePath; +} + +function safeRead(filePath: string): string | null { + try { + return readFileSync(filePath, "utf8"); + } catch { + return null; + } +} + +function safeReadDir(dirPath: string): string[] { + try { + return readdirSync(dirPath); + } catch { + return []; + } +} + +function safeIsDirectory(path: string): boolean { + try { + return statSync(path).isDirectory(); + } catch { + return false; + } +} + +function normalizePath(path: string): string { + return path.split(sep).join("/"); +} + +function frontMatterStringArray(markdown: string, key: string): string[] { + const raw = frontMatterValue(markdown, key); + if (!raw) return []; + if (raw.startsWith("[")) { + try { + const parsed: unknown = JSON.parse(raw); + if (Array.isArray(parsed)) + return parsed.filter((value): value is string => typeof value === "string"); + } catch { + return []; + } + } + return raw + .split(",") + .map((value) => normalizeValue(value)) + .filter(Boolean); +} + +function frontMatterValue(markdown: string, key: string): string | undefined { + const frontMatter = markdown.match(/^---\r?\n([\s\S]*?)\r?\n---/); + if (!frontMatter?.[1]) return undefined; + const lines = frontMatter[1].split(/\r?\n/); + const direct = lines.find((line) => line.toLowerCase().startsWith(`${key.toLowerCase()}:`)); + if (!direct) return undefined; + return stripQuotes(direct.slice(direct.indexOf(":") + 1).trim()); +} + +function labeledLineValue(text: string, label: string): string | undefined { + const escaped = label.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const match = text.match( + new RegExp(`(?:^|\\n)\\s*(?:[-*]\\s*)?(?:\\*\\*)?${escaped}(?:\\*\\*)?\\s*:\\s*([^\\n]+)`, "i"), + ); + return match?.[1] ? normalizeValue(match[1]) : undefined; +} + +function jsonStringValue(text: string, key: string): string | undefined { + return jsonStringValues(text, key)[0]; +} + +function jsonStringValues(text: string, key: string): string[] { + const values: string[] = []; + const escaped = key.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const pattern = new RegExp(`"${escaped}"\\s*:\\s*"([^"]+)"`, "gi"); + for (const match of text.matchAll(pattern)) { + if (match[1]) values.push(normalizeValue(match[1])); + } + return values; +} + +function uniqueMatches(text: string, pattern: RegExp): string[] { + return [ + ...new Set([...text.matchAll(pattern)].map((match) => normalizeValue(match[1] ?? ""))), + ].filter(Boolean); +} + +function firstDate(values: Array): string | undefined { + for (const value of values) { + if (!value) continue; + const parsed = new Date(value); + if (!Number.isNaN(parsed.valueOf())) return parsed.toISOString(); + } + return undefined; +} + +function hasSuccessfulOutcome(text: string): boolean { + return /\b(applied|merged|closed|success|succeeded|pass)\b/i.test(text); +} + +function normalizeValue(value: string): string { + return stripQuotes(value) + .replace(//g, "") + .replaceAll("`", "") + .trim() + .toLowerCase() + .replace(/\s+/g, " ") + .slice(0, 120); +} + +function stripQuotes(value: string): string { + return value.replace(/^["']|["']$/g, ""); +} + +function compareObservation( + left: PolicyPatternObservation, + right: PolicyPatternObservation, +): number { + return ( + left.patternType.localeCompare(right.patternType) || + left.value.localeCompare(right.value) || + left.sourceRecord.localeCompare(right.sourceRecord) + ); +} diff --git a/src/policy-rfc/index.ts b/src/policy-rfc/index.ts new file mode 100644 index 0000000000..43dc61f763 --- /dev/null +++ b/src/policy-rfc/index.ts @@ -0,0 +1,73 @@ +#!/usr/bin/env node +import { mkdirSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { numberArg, parseArgs, stringArg } from "../clawsweeper-args.js"; +import { repositoryProfileFor } from "../repository-profiles.js"; +import { sortStable } from "../stable-json.js"; +import { collectPolicyPatterns } from "./collector.js"; +import { scorePolicyPatterns } from "./scorer.js"; +import { synthesizePolicyProposal } from "./synthesizer.js"; + +export { collectPolicyPatterns } from "./collector.js"; +export { scorePolicyPatterns } from "./scorer.js"; +export { synthesizePolicyProposal } from "./synthesizer.js"; +export type * from "./types.js"; + +interface RunPolicyRfcOptions { + recordsRoot: string; + outputRoot: string; + targetRepo: string; + minOccurrences: number; + createdAt?: string | undefined; +} + +export function runPolicyRfc(options: RunPolicyRfcOptions): { + proposals: number; + outputDir: string; +} { + const profile = repositoryProfileFor(options.targetRepo); + const outputDir = join(options.outputRoot, profile.slug); + const observations = collectPolicyPatterns({ + recordsRoot: options.recordsRoot, + targetRepo: options.targetRepo, + }); + const scored = scorePolicyPatterns(observations, { + minOccurrences: options.minOccurrences, + }); + + mkdirSync(outputDir, { recursive: true }); + for (const pattern of scored) { + const proposal = synthesizePolicyProposal(pattern, { createdAt: options.createdAt }); + writeFileSync(join(outputDir, `${proposal.id}.md`), proposal.markdown); + writeFileSync( + join(outputDir, `${proposal.id}.json`), + `${JSON.stringify(sortStable(proposal.json), null, 2)}\n`, + ); + } + + return { proposals: scored.length, outputDir }; +} + +function main(): void { + const args = parseArgs(process.argv.slice(2)); + const targetRepo = stringArg(args.target_repo, "openclaw/openclaw"); + const recordsRoot = resolve(stringArg(args.records_root, "records")); + const outputRoot = resolve(stringArg(args.output_root, "results/policy-rfc")); + const minOccurrences = numberArg(args.min_occurrences, 5); + const createdAt = typeof args.created_at === "string" ? args.created_at : undefined; + const result = runPolicyRfc({ + recordsRoot, + outputRoot, + targetRepo, + minOccurrences, + createdAt, + }); + console.log(`Policy RFC proposals written: ${result.proposals}`); + console.log(`Output directory: ${result.outputDir}`); +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) { + main(); +} diff --git a/src/policy-rfc/scorer.ts b/src/policy-rfc/scorer.ts new file mode 100644 index 0000000000..d1fef528b0 --- /dev/null +++ b/src/policy-rfc/scorer.ts @@ -0,0 +1,189 @@ +import { createHash } from "node:crypto"; + +import type { + PatternScorerOptions, + PolicyPatternObservation, + PolicyPatternType, + ScoredPolicyPattern, +} from "./types.js"; + +export function scorePolicyPatterns( + observations: readonly PolicyPatternObservation[], + options: PatternScorerOptions, +): ScoredPolicyPattern[] { + const minDistinctItems = options.minDistinctItems ?? Math.min(2, options.minOccurrences); + const minDistinctRepos = options.minDistinctRepos ?? 1; + const groups = new Map(); + + for (const observation of observations) { + const key = `${observation.patternType}\0${observation.value}`; + const group = groups.get(key) ?? []; + group.push(observation); + groups.set(key, group); + } + + const scored: ScoredPolicyPattern[] = []; + for (const group of groups.values()) { + const sortedGroup = [...group].sort(compareObservation); + const first = sortedGroup[0]; + if (!first) continue; + const distinctItems = sortedUnique(sortedGroup.map((item) => `${item.repo}${item.item}`)); + const distinctRepos = sortedUnique(sortedGroup.map((item) => item.repo)); + if (sortedGroup.length < options.minOccurrences) continue; + if (distinctItems.length < minDistinctItems) continue; + if (distinctRepos.length < minDistinctRepos) continue; + + const successfulOutcomes = sortedGroup.filter((item) => item.successfulOutcome).length; + const latestObservedAt = latestDate(sortedGroup); + const confidenceScore = confidence({ + occurrenceCount: sortedGroup.length, + distinctItems: distinctItems.length, + distinctRepos: distinctRepos.length, + successfulOutcomes, + latestObservedAt, + now: options.now ?? new Date(), + }); + + scored.push({ + id: policyPatternId(first.patternType, first.value), + patternType: first.patternType, + value: first.value, + title: policyTitle(first.patternType, first.value), + confidenceScore, + occurrenceCount: sortedGroup.length, + distinctItems, + distinctRepos, + successfulOutcomes, + latestObservedAt, + evidenceItems: sortedGroup.slice(0, 20), + proposedConditions: proposedConditions(first.patternType, first.value), + proposedAction: proposedAction(first.patternType), + safetyConstraints: safetyConstraints(first.patternType), + sourceRecords: sortedUnique(sortedGroup.map((item) => item.sourceRecord)), + }); + } + + return scored.sort( + (left, right) => + right.confidenceScore - left.confidenceScore || + right.occurrenceCount - left.occurrenceCount || + left.id.localeCompare(right.id), + ); +} + +function confidence(options: { + occurrenceCount: number; + distinctItems: number; + distinctRepos: number; + successfulOutcomes: number; + latestObservedAt?: string | undefined; + now: Date; +}): number { + const occurrence = Math.min(options.occurrenceCount / 10, 1) * 0.35; + const itemSpread = Math.min(options.distinctItems / 5, 1) * 0.25; + const repoSpread = Math.min(options.distinctRepos / 2, 1) * 0.15; + const success = + Math.min(options.successfulOutcomes / Math.max(options.occurrenceCount, 1), 1) * 0.15; + const recentness = recencyScore(options.latestObservedAt, options.now) * 0.1; + return Number((occurrence + itemSpread + repoSpread + success + recentness).toFixed(3)); +} + +function recencyScore(latestObservedAt: string | undefined, now: Date): number { + if (!latestObservedAt) return 0.3; + const latest = new Date(latestObservedAt); + if (Number.isNaN(latest.valueOf())) return 0.3; + const ageDays = Math.max(0, (now.valueOf() - latest.valueOf()) / 86_400_000); + if (ageDays <= 30) return 1; + if (ageDays <= 90) return 0.7; + if (ageDays <= 180) return 0.4; + return 0.2; +} + +function policyPatternId(patternType: PolicyPatternType, value: string): string { + const slug = `${patternType}-${value}` + .replace(/[^a-z0-9]+/g, "-") + .replace(/^-|-$/g, "") + .slice(0, 72); + const hash = createHash("sha256").update(`${patternType}:${value}`).digest("hex").slice(0, 8); + return `policy-rfc-${slug}-${hash}`; +} + +function policyTitle(patternType: PolicyPatternType, value: string): string { + return `Policy RFC: ${labelForPatternType(patternType)} - ${value}`; +} + +function labelForPatternType(patternType: PolicyPatternType): string { + return patternType + .split("_") + .map((part) => part[0]?.toUpperCase() + part.slice(1)) + .join(" "); +} + +function proposedConditions(patternType: PolicyPatternType, value: string): string[] { + return [ + `The observed ${labelForPatternType(patternType).toLowerCase()} is "${value}".`, + "At least the configured minimum number of distinct records show the same pattern.", + "The source record has durable ClawSweeper evidence and is not a malformed or partial record.", + ]; +} + +function proposedAction(patternType: PolicyPatternType): string { + switch (patternType) { + case "label": + return "Document a candidate review heuristic for this repeated label; do not mutate labels automatically."; + case "review_verdict": + return "Document the repeated review verdict as a candidate triage heuristic for maintainer review."; + case "safe_close_reason": + return "Document a candidate close-policy clarification; keep all close actions on existing apply paths."; + case "automerge_repair_cause": + return "Document a candidate automerge repair precondition; do not change automerge behavior automatically."; + case "file_conflict_type": + return "Document a candidate conflict-handling policy for repair planning only."; + case "repair_marker": + return "Document a candidate repair policy based on repeated repair markers."; + } +} + +function safetyConstraints(patternType: PolicyPatternType): string[] { + const base = [ + "Proposal-only: this RFC must not execute policy automatically.", + "No GitHub mutation is allowed from the Policy RFC Engine.", + "Existing scheduler review, apply, close, and automerge paths remain unchanged.", + ]; + if (patternType === "safe_close_reason") { + return [ + ...base, + "Any close behavior must continue to require existing repository close rules.", + ]; + } + if (patternType === "automerge_repair_cause") { + return [ + ...base, + "Automerge eligibility must continue to be decided by existing automerge guards.", + ]; + } + return base; +} + +function latestDate(observations: readonly PolicyPatternObservation[]): string | undefined { + const dates = observations + .map((item) => item.observedAt) + .filter((item): item is string => Boolean(item)) + .sort(); + return dates.at(-1); +} + +function sortedUnique(values: readonly string[]): string[] { + return [...new Set(values)].sort(); +} + +function compareObservation( + left: PolicyPatternObservation, + right: PolicyPatternObservation, +): number { + return ( + left.repo.localeCompare(right.repo) || + left.item.localeCompare(right.item) || + left.sourceRecord.localeCompare(right.sourceRecord) + ); +} diff --git a/src/policy-rfc/synthesizer.ts b/src/policy-rfc/synthesizer.ts new file mode 100644 index 0000000000..7eed004c2f --- /dev/null +++ b/src/policy-rfc/synthesizer.ts @@ -0,0 +1,121 @@ +import type { + PolicyProposalJson, + ScoredPolicyPattern, + SynthesizeOptions, + SynthesizedPolicyProposal, +} from "./types.js"; + +export function synthesizePolicyProposal( + pattern: ScoredPolicyPattern, + options: SynthesizeOptions = {}, +): SynthesizedPolicyProposal { + const createdAt = options.createdAt ?? new Date().toISOString(); + const json = policyProposalJson(pattern, createdAt); + return { + id: pattern.id, + markdown: policyProposalMarkdown(pattern, createdAt), + json, + }; +} + +export function policyProposalJson( + pattern: ScoredPolicyPattern, + createdAt: string, +): PolicyProposalJson { + return { + id: pattern.id, + title: pattern.title, + status: "Draft", + pattern_type: pattern.patternType, + evidence_items: pattern.evidenceItems.map((item) => { + const evidence = { + repo: item.repo, + item: item.item, + source_record: item.sourceRecord, + }; + return { + ...evidence, + ...(item.observedAt ? { observed_at: item.observedAt } : {}), + ...(item.detail ? { detail: item.detail } : {}), + }; + }), + confidence_score: pattern.confidenceScore, + proposed_conditions: pattern.proposedConditions, + proposed_action: pattern.proposedAction, + safety_constraints: pattern.safetyConstraints, + created_at: createdAt, + source_records: pattern.sourceRecords, + }; +} + +export function policyProposalMarkdown(pattern: ScoredPolicyPattern, createdAt: string): string { + return [ + `# ${pattern.title}`, + "", + "Status: Draft", + "", + "## Summary", + "", + `ClawSweeper observed the repeated \`${pattern.patternType}\` pattern \`${pattern.value}\` across ${pattern.occurrenceCount} records. This RFC proposes documenting a conservative policy candidate for maintainer review only.`, + "", + "## Observed Pattern", + "", + `- Pattern type: \`${pattern.patternType}\``, + `- Pattern value: \`${pattern.value}\``, + `- Occurrences: ${pattern.occurrenceCount}`, + `- Distinct items: ${pattern.distinctItems.length}`, + `- Distinct repositories: ${pattern.distinctRepos.length}`, + `- Successful repair/apply outcomes observed: ${pattern.successfulOutcomes}`, + `- Latest observation: ${pattern.latestObservedAt ?? "unknown"}`, + `- Confidence score: ${pattern.confidenceScore.toFixed(3)}`, + "", + "## Evidence", + "", + ...evidenceLines(pattern), + "", + "## Proposed Policy", + "", + pattern.proposedAction, + "", + "Proposed conditions:", + "", + ...pattern.proposedConditions.map((condition) => `- ${condition}`), + "", + "## Safety Constraints", + "", + ...pattern.safetyConstraints.map((constraint) => `- ${constraint}`), + "", + "## Non-Goals", + "", + "- Do not auto-apply this policy.", + "- Do not mutate GitHub state from this proposal.", + "- Do not alter scheduler cadence, review shards, close behavior, apply behavior, or automerge behavior.", + "", + "## Rollout Plan", + "", + "1. Review this RFC with maintainers.", + "2. If accepted, convert the proposal into an explicit policy change in a separate pull request.", + "3. Ship any executable behavior behind existing conservative apply paths and tests.", + "", + "## Metrics", + "", + "- Number of future records matching the proposed conditions.", + "- False-positive rate found during maintainer review.", + "- Number of accepted, revised, or rejected proposals for this pattern.", + "", + "## Reversion Plan", + "", + "Archive or delete the generated RFC and JSON proposal. Since this engine is proposal-only, no runtime policy behavior needs to be reverted.", + "", + `Generated by ClawSweeper Policy RFC Engine at ${createdAt}.`, + "", + ].join("\n"); +} + +function evidenceLines(pattern: ScoredPolicyPattern): string[] { + if (!pattern.evidenceItems.length) return ["No evidence items were retained."]; + return pattern.evidenceItems.map((item) => { + const observed = item.observedAt ? ` observed ${item.observedAt}` : ""; + return `- ${item.repo} ${item.item}${observed}: \`${item.sourceRecord}\``; + }); +} diff --git a/src/policy-rfc/types.ts b/src/policy-rfc/types.ts new file mode 100644 index 0000000000..cb1299d596 --- /dev/null +++ b/src/policy-rfc/types.ts @@ -0,0 +1,78 @@ +export type PolicyPatternType = + | "file_conflict_type" + | "label" + | "repair_marker" + | "review_verdict" + | "safe_close_reason" + | "automerge_repair_cause"; + +export interface PolicyPatternObservation { + patternType: PolicyPatternType; + value: string; + repo: string; + item: string; + sourceRecord: string; + observedAt?: string | undefined; + successfulOutcome: boolean; + detail?: string | undefined; +} + +export interface PatternCollectorOptions { + recordsRoot: string; + targetRepo?: string | undefined; +} + +export interface ScoredPolicyPattern { + id: string; + patternType: PolicyPatternType; + value: string; + title: string; + confidenceScore: number; + occurrenceCount: number; + distinctItems: string[]; + distinctRepos: string[]; + successfulOutcomes: number; + latestObservedAt?: string | undefined; + evidenceItems: PolicyPatternObservation[]; + proposedConditions: string[]; + proposedAction: string; + safetyConstraints: string[]; + sourceRecords: string[]; +} + +export interface PatternScorerOptions { + minOccurrences: number; + minDistinctItems?: number | undefined; + minDistinctRepos?: number | undefined; + now?: Date | undefined; +} + +export interface PolicyProposalJson { + id: string; + title: string; + status: "Draft"; + pattern_type: PolicyPatternType; + evidence_items: Array<{ + repo: string; + item: string; + source_record: string; + observed_at?: string | undefined; + detail?: string | undefined; + }>; + confidence_score: number; + proposed_conditions: string[]; + proposed_action: string; + safety_constraints: string[]; + created_at: string; + source_records: string[]; +} + +export interface SynthesizedPolicyProposal { + id: string; + markdown: string; + json: PolicyProposalJson; +} + +export interface SynthesizeOptions { + createdAt?: string | undefined; +} diff --git a/test/policy-rfc.test.ts b/test/policy-rfc.test.ts new file mode 100644 index 0000000000..a25089794c --- /dev/null +++ b/test/policy-rfc.test.ts @@ -0,0 +1,160 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { + collectPolicyPatterns, + scorePolicyPatterns, + synthesizePolicyProposal, +} from "../dist/policy-rfc/index.js"; + +function writeRecord(root: string, item: number, body: string): void { + const dir = join(root, "openclaw-openclaw", "items"); + mkdirSync(dir, { recursive: true }); + writeFileSync(join(dir, `${item}.md`), body); +} + +function withPolicyFixture(run: (recordsRoot: string) => void): void { + const recordsRoot = mkdtempSync(join(tmpdir(), "clawsweeper-policy-rfc-")); + try { + writeRecord( + recordsRoot, + 1, + `--- +labels: ["clawsweeper:autofix", "bug"] +reviewed_at: 2026-05-01T00:00:00.000Z +close_reason: implemented_on_main +--- + +Automerge repair cause: flaky validation +Conflict type: generated lockfile + +Result: applied +`, + ); + writeRecord( + recordsRoot, + 2, + `--- +labels: ["bug"] +reviewed_at: 2026-05-02T00:00:00.000Z +--- + + +Result: applied +`, + ); + writeRecord( + recordsRoot, + 3, + `--- +labels: ["bug"] +reviewed_at: 2026-05-03T00:00:00.000Z +--- + + +Result: applied +`, + ); + writeRecord(recordsRoot, 4, "{ this is malformed but should not crash"); + run(recordsRoot); + } finally { + rmSync(recordsRoot, { recursive: true, force: true }); + } +} + +function collectFixture(recordsRoot: string) { + return collectPolicyPatterns({ + recordsRoot, + targetRepo: "openclaw/openclaw", + }); +} + +test("collector extracts repeated patterns from durable records", () => { + withPolicyFixture((recordsRoot) => { + const observations = collectFixture(recordsRoot); + + assert.ok( + observations.some( + (item) => item.patternType === "repair_marker" && item.value === "validation-fix", + ), + ); + assert.ok( + observations.some( + (item) => item.patternType === "review_verdict" && item.value === "needs-changes", + ), + ); + assert.ok( + observations.some( + (item) => item.patternType === "safe_close_reason" && item.value === "implemented_on_main", + ), + ); + }); +}); + +test("collector tolerates missing and malformed records", () => { + withPolicyFixture((recordsRoot) => { + assert.doesNotThrow(() => collectFixture(recordsRoot)); + assert.deepEqual( + collectPolicyPatterns({ + recordsRoot: join(recordsRoot, "does-not-exist"), + targetRepo: "openclaw/openclaw", + }), + [], + ); + }); +}); + +test("scorer rejects low-frequency patterns", () => { + withPolicyFixture((recordsRoot) => { + const rejected = scorePolicyPatterns(collectFixture(recordsRoot), { + minOccurrences: 4, + now: new Date("2026-05-04T00:00:00.000Z"), + }); + + assert.equal( + rejected.some((item) => item.patternType === "repair_marker"), + false, + ); + }); +}); + +test("scorer accepts patterns above the configured threshold", () => { + withPolicyFixture((recordsRoot) => { + const accepted = scorePolicyPatterns(collectFixture(recordsRoot), { + minOccurrences: 3, + now: new Date("2026-05-04T00:00:00.000Z"), + }); + const repairPattern = accepted.find((item) => item.patternType === "repair_marker"); + + assert.ok(repairPattern); + assert.equal(repairPattern.occurrenceCount, 3); + assert.equal(repairPattern.distinctItems.length, 3); + assert.equal(repairPattern.successfulOutcomes, 3); + }); +}); + +test("synthesizer produces stable markdown and proposal JSON", () => { + withPolicyFixture((recordsRoot) => { + const accepted = scorePolicyPatterns(collectFixture(recordsRoot), { + minOccurrences: 3, + now: new Date("2026-05-04T00:00:00.000Z"), + }); + const repairPattern = accepted.find((item) => item.patternType === "repair_marker"); + assert.ok(repairPattern); + + const proposal = synthesizePolicyProposal(repairPattern, { + createdAt: "2026-05-04T00:00:00.000Z", + }); + + assert.match(proposal.markdown, /^# Policy RFC: Repair Marker - validation-fix/); + assert.match(proposal.markdown, /Status: Draft/); + assert.match(proposal.markdown, /## Safety Constraints/); + assert.equal(proposal.json.status, "Draft"); + assert.equal(proposal.json.pattern_type, "repair_marker"); + assert.equal(proposal.json.evidence_items.length, 3); + assert.equal(proposal.json.created_at, "2026-05-04T00:00:00.000Z"); + }); +});