feat(autonomy): add budget guard, compensation registry, agent-jobs D…

.github/workflows/autonomy-ops.yml

@@ -0,0 +1,135 @@
+# autonomy-ops.yml
+# NEW workflow — does NOT replace or conflict with: ci.yml
+# Provides manual-dispatch and weekly scheduled gate for full-site autonomous SEO runs.
+# Routine per-client module jobs continue to be driven by the internal BullMQ scheduler.
+
+name: SEO Bot — Autonomous Full-Site Ops
+
+on:
+  schedule:
+    - cron: '0 3 * * 1'   # Weekly Monday 03:00 UTC — off-peak
+  workflow_dispatch:
+    inputs:
+      scope:
+        description: 'Analysis scope: full | delta | single-url'
+        required: false
+        default: 'delta'
+      cost_cap_usd:
+        description: 'Hard cost cap for this run (USD)'
+        required: false
+        default: '2.00'
+      target_url:
+        description: 'Target URL (only used when scope=single-url)'
+        required: false
+        default: ''
+      client_id:
+        description: 'Client ID to scope this run (leave empty for all clients)'
+        required: false
+        default: ''
+
+permissions:
+  contents: write
+  pull-requests: write
+  issues: write
+
+jobs:
+  gate:
+    name: Pre-flight gates
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Type check
+        run: npm run build   # tsc — fail on type errors
+
+      - name: Lint
+        run: npm run lint
+
+      - name: Verify all
+        run: npm run verify:all
+        env:
+          DATABASE_URL: ${{ secrets.DATABASE_URL }}
+          REDIS_URL: ${{ secrets.REDIS_URL }}
+
+  run-seo-ops:
+    name: Run SEO Bot autonomous ops
+    needs: gate
+    runs-on: ubuntu-latest
+    timeout-minutes: 120
+    env:
+      DATABASE_URL: ${{ secrets.DATABASE_URL }}
+      REDIS_URL: ${{ secrets.REDIS_URL }}
+      OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+      SERPAPI_KEY: ${{ secrets.SERPAPI_KEY }}
+      DATAFORSEO_LOGIN: ${{ secrets.DATAFORSEO_LOGIN }}
+      DATAFORSEO_PASSWORD: ${{ secrets.DATAFORSEO_PASSWORD }}
+      POSTHOG_PROJECT_API_KEY: ${{ secrets.POSTHOG_PROJECT_API_KEY }}
+      COST_CAP_USD: ${{ inputs.cost_cap_usd || '2.00' }}
+      ANALYSIS_SCOPE: ${{ inputs.scope || 'delta' }}
+      TARGET_URL: ${{ inputs.target_url || '' }}
+      CLIENT_ID: ${{ inputs.client_id || '' }}
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '22'
+          cache: 'npm'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Restore crawl cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/seo-bot
+          key: seo-bot-crawl-${{ github.run_number }}
+          restore-keys: seo-bot-crawl-
+
+      - name: Run autonomous SEO ops
+        run: node dist/scripts/autonomous-run.js
+        # This script:
+        #   1. Reads COST_CAP_USD, ANALYSIS_SCOPE, TARGET_URL, CLIENT_ID from env
+        #   2. Creates an agent_jobs row in Postgres with idempotency_key = hash(scope+date+client)
+        #   3. Dispatches a BullMQ job with the parameters
+        #   4. Polls job completion with a 110-minute timeout
+        #   5. Writes outputs/seo-report-{run_id}.json on success
+
+      - name: Create recommendations PR
+        if: success()
+        run: |
+          BRANCH="bot/seo-${{ github.run_id }}"
+          git config user.email "seo-bot@quantum-l9.io"
+          git config user.name "SEO Bot"
+          git checkout -b "$BRANCH"
+          if git diff --quiet; then
+            echo "No file changes to commit"
+          else
+            git add outputs/
+            git commit -m "chore(seo-bot): recommendations run ${{ github.run_id }}"
+            git push origin "$BRANCH"
+            gh pr create \
+              --title "SEO Bot: recommendations ${{ github.run_id }}" \
+              --body-file outputs/pr-body.md \
+              --base main --head "$BRANCH" \
+              --label "seo-bot,automated" || true
+          fi
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upload SEO report artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: seo-report-${{ github.run_id }}
+          path: outputs/
+          if-no-files-found: ignore

adr/0005-autonomy-runtime-controls.md

@@ -0,0 +1,61 @@
+# ADR-0005: Autonomy Runtime Controls
+
+**Status:** Proposed  
+**Date:** 2026-07-15  
+**Supersedes:** None  
+**Related:** ADR-0001 (BullMQ), ADR-0002 (multi-tenancy), ADR-0003 (token budget)
+
+## Context
+
+SEO Bot operates as a 24/7 multi-tenant service on a Hetzner VPS with BullMQ + Redis and Postgres.
+Existing job scheduling (BullMQ) and token budgeting (TokenBudget per scheduler job) cover per-run constraints well.
+
+Gaps identified in the L9 nuclear architecture brief:
+1. No USD-level cross-run spend tracking or cap enforcement.
+2. No registered compensation/rollback path for external SEO API writes or outreach sends.
+3. No GitHub Actions workflow for autonomous full-site ops (only `ci.yml` for PR validation exists).
+4. No `agent_jobs`, `budget_violations`, or `compensation_log` tables for runtime evidence.
+
+## Decision
+
+Add the following **without replacing or modifying any existing files**:
+
+| New file | Purpose |
+|---|---|
+| `.github/workflows/autonomy-ops.yml` | Manual/scheduled full-site ops workflow. Does not replace `ci.yml`. |
+| `src/core/budget-guard.ts` | `AgentBudgetGuard` class — USD-level admission, reserve, reconcile, enforce. |
+| `src/core/compensation.ts` | `CompensationRegistry` — saga rollback for external mutations. |
+| `src/core/schema-additions.sql` | Additive DDL for `agent_jobs`, `budget_violations`, `compensation_log`. |
+
+## What does NOT change
+
+- `ci.yml` — untouched.
+- `src/core/scheduler.ts` — untouched. Existing `TokenBudget` per-job controls remain primary.
+- `src/core/config.ts` — untouched.
+- `src/core/database/schema.ts` — the SQL DDL is provided separately; Drizzle migration is a follow-on task.
+- `AGENTS.md` locked decisions (BullMQ, multi-tenancy, Pino logging, Drizzle, no Puppeteer/Playwright).
+- Existing ADRs 0001–0004.
+
+## Integration path
+
+1. Apply `src/core/schema-additions.sql` to the Postgres database.
+2. Add `AgentBudgetGuard` calls to new or high-cost job handlers where USD tracking is needed.
+3. Add `CompensationRegistry` registration before external writes in outreach, SEO provider, and future live-site mutation steps.
+4. Reflect new tables into `schema.ts` and generate a Drizzle migration (follow-on PR).
+5. The `autonomy-ops.yml` workflow fires `node dist/scripts/autonomous-run.js` — this script must be created in a follow-on PR.
+
+## Budget mode thresholds
+
+| Pressure | Mode | Agent action |
+|---|---|---|
+| < 70% | `normal` | Continue |
+| 70–85% | `cheaper_model` | Route to fast tier |
+| 85–95% | `narrow_scope` | Reduce page/module scope |
+| 95–100% | `require_approval` | Pause and notify |
+| > 100% | `stop` | Throw `BudgetExceededError` |
+
+## Consequences
+
+- Every external SEO API write or outreach send that registers a compensation action becomes reversible.
+- USD spend is visible per-run and per-client in Postgres for billing, auditing, and anomaly detection.
+- The new `autonomy-ops.yml` workflow gives CI/CD entry point for full-site ops without touching the existing `ci.yml`.

src/core/budget-guard.ts

@@ -0,0 +1,119 @@
+/**
+ * AgentBudgetGuard — four-move runtime budget control loop for SEO Bot.
+ *
+ * Aligns with the existing TokenBudget policy in src/core/scheduler.ts.
+ * Per-client budget caps (maxFastTokensPerRun, maxStrategicTokensPerRun) remain
+ * the primary per-job controls — this guard adds USD-level tracking for cross-client
+ * aggregate spend and GitHub Actions workflow-level cost caps.
+ *
+ * Moves: Open (admission) → Reserve → Reconcile → Enforce
+ */
+
+import { getConfig } from './config.js';
+import { logger } from './logger.js';
+
+export class BudgetExceededError extends Error {}
+export class AdmissionRejectedError extends Error {}
+
+export type BudgetMode = 'normal' | 'cheaper_model' | 'narrow_scope' | 'require_approval' | 'stop';
+
+export interface BudgetEnforcement {
+  jobId: string;
+  clientId?: string;
+  mode: BudgetMode;
+  actualUsd: number;
+  reservedUsd: number;
+  remainingUsd: number;
+  forecastUsd: number;
+}
+
+export class AgentBudgetGuard {
+  private capUsd: number;
+  private actualUsd = 0;
+  private reservedUsd = 0;
+  private forecastUsd = 0;
+  private mode: BudgetMode = 'normal';
+
+  constructor(
+    public readonly jobId: string,
+    capUsd: number,
+    public readonly clientId?: string,
+  ) {
+    this.capUsd = capUsd;
+  }
+
+  /** MOVE 1 — Admission: verify forecast is feasible before enqueueing. */
+  open(initialForecastUsd = 0): void {
+    this.forecastUsd = initialForecastUsd;
+    if (this.forecastUsd > this.capUsd) {
+      throw new AdmissionRejectedError(
+        `Admission rejected for job=${this.jobId} client=${this.clientId}: forecast $${this.forecastUsd.toFixed(4)} > cap $${this.capUsd.toFixed(4)}`,
+      );
+    }
+    logger.info({ jobId: this.jobId, clientId: this.clientId, capUsd: this.capUsd }, 'budget_guard:opened');
+  }
+
+  /** MOVE 2 — Reserve: lock budget before each LLM call or API call with token cost. */
+  reserve(estimatedUsd: number): void {
+    const remaining = this.capUsd - this.actualUsd - this.reservedUsd;
+    if (estimatedUsd > remaining) {
+      this._updateMode();
+      const remainingAfterMode = this.capUsd - this.actualUsd - this.reservedUsd;
+      if (estimatedUsd > remainingAfterMode) {
+        throw new BudgetExceededError(
+          `Reservation denied for job=${this.jobId}: need $${estimatedUsd.toFixed(4)}, remaining $${remainingAfterMode.toFixed(4)}, mode=${this.mode}`,
+        );
+      }
+    }
+    this.reservedUsd += estimatedUsd;
+    this.forecastUsd = this.actualUsd + this.reservedUsd;
+  }
+
+  /** MOVE 3 — Reconcile: record actual spend after each step. */
+  reconcile(actualUsd: number, nextEstimateUsd = 0): void {
+    this.actualUsd += actualUsd;
+    this.reservedUsd = Math.max(0, this.reservedUsd - actualUsd);
+    this.forecastUsd = this.actualUsd + this.reservedUsd + nextEstimateUsd;
+    logger.debug({ jobId: this.jobId, actualUsd: this.actualUsd, forecastUsd: this.forecastUsd }, 'budget_guard:reconciled');
+    if (this.actualUsd > this.capUsd) {
+      throw new BudgetExceededError(
+        `Budget cap $${this.capUsd.toFixed(4)} exceeded for job=${this.jobId}: actual=$${this.actualUsd.toFixed(4)}`,
+      );
+    }
+    if (this.forecastUsd > this.capUsd) this._updateMode();
+  }
+
+  /** MOVE 4 — Enforce: return state snapshot; throws if cap is fully exhausted. */
+  enforce(): BudgetEnforcement {
+    const remaining = this.capUsd - this.actualUsd;
+    if (remaining <= 0) {
+      this.mode = 'stop';
+      throw new BudgetExceededError(
+        `Cap exhausted for job=${this.jobId}: actual=$${this.actualUsd.toFixed(4)}, cap=$${this.capUsd.toFixed(4)}`,
+      );
+    }
+    return {
+      jobId: this.jobId,
+      clientId: this.clientId,
+      mode: this.mode,
+      actualUsd: this.actualUsd,
+      reservedUsd: this.reservedUsd,
+      remainingUsd: remaining,
+      forecastUsd: this.forecastUsd,
+    };
+  }
+
+  get currentMode(): BudgetMode {
+    return this.mode;
+  }
+
+  private _updateMode(): void {
+    const pressure = this.capUsd === 0 ? 1 : (this.actualUsd + this.reservedUsd) / this.capUsd;
+    if (pressure < 0.70) this.mode = 'normal';
+    else if (pressure < 0.85) this.mode = 'cheaper_model';
+    else if (pressure < 0.95) this.mode = 'narrow_scope';
+    else if (pressure < 1.00) this.mode = 'require_approval';
+    else this.mode = 'stop';
+    logger.warn({ jobId: this.jobId, mode: this.mode, pressure: pressure.toFixed(3) }, 'budget_guard:mode_changed');
+  }
+}

src/core/compensation.ts

@@ -0,0 +1,64 @@
+/**
+ * CompensationRegistry — saga-pattern rollback for external mutations in SEO Bot.
+ *
+ * Covers: external SEO provider writes, outreach API calls, Postgres state writes,
+ * client-snippet pushes, and any future live-site mutations.
+ *
+ * Rule: register compensation BEFORE executing the mutating step.
+ * On failure, call compensate() to reverse registered actions in LIFO order.
+ */
+
+import { logger } from './logger.js';
+
+export interface CompensationEntry {
+  stepId: string;
+  clientId?: string;
+  action: () => Promise<void>;
+  registeredAt: Date;
+}
+
+export class CompensationRegistry {
+  private readonly entries: CompensationEntry[] = [];
+
+  constructor(
+    public readonly jobId: string,
+    public readonly clientId?: string,
+  ) {}
+
+  /**
+   * Register a compensation action for a step about to mutate external state.
+   * Always call this BEFORE the mutation, not after.
+   */
+  register(stepId: string, action: () => Promise<void>): void {
+    this.entries.push({ stepId, clientId: this.clientId, action, registeredAt: new Date() });
+    logger.debug({ jobId: this.jobId, stepId, clientId: this.clientId }, 'compensation:registered');
+  }
+
+  /**
+   * Execute all compensations in reverse order (LIFO).
+   * Errors in individual compensations are collected but do not abort others.
+   */
+  async compensate(): Promise<{ stepId: string; error?: string }[]> {
+    const results: { stepId: string; error?: string }[] = [];
+    for (const entry of [...this.entries].reverse()) {
+      try {
+        await entry.action();
+        results.push({ stepId: entry.stepId });
+        logger.info({ jobId: this.jobId, stepId: entry.stepId }, 'compensation:ok');
+      } catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        results.push({ stepId: entry.stepId, error: message });
+        logger.error({ jobId: this.jobId, stepId: entry.stepId, message }, 'compensation:failed');
+      }
+    }
+    return results;
+  }
+
+  clear(): void {
+    this.entries.length = 0;
+  }
+
+  get size(): number {
+    return this.entries.length;
+  }
+}