diff --git a/.github/workflows/autonomy-ops.yml b/.github/workflows/autonomy-ops.yml new file mode 100644 index 0000000..98bef72 --- /dev/null +++ b/.github/workflows/autonomy-ops.yml @@ -0,0 +1,135 @@ +# autonomy-ops.yml +# NEW workflow — does NOT replace or conflict with: ci.yml +# Provides manual-dispatch and weekly scheduled gate for full-site autonomous SEO runs. +# Routine per-client module jobs continue to be driven by the internal BullMQ scheduler. + +name: SEO Bot — Autonomous Full-Site Ops + +on: + schedule: + - cron: '0 3 * * 1' # Weekly Monday 03:00 UTC — off-peak + workflow_dispatch: + inputs: + scope: + description: 'Analysis scope: full | delta | single-url' + required: false + default: 'delta' + cost_cap_usd: + description: 'Hard cost cap for this run (USD)' + required: false + default: '2.00' + target_url: + description: 'Target URL (only used when scope=single-url)' + required: false + default: '' + client_id: + description: 'Client ID to scope this run (leave empty for all clients)' + required: false + default: '' + +permissions: + contents: write + pull-requests: write + issues: write + +jobs: + gate: + name: Pre-flight gates + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Type check + run: npm run build # tsc — fail on type errors + + - name: Lint + run: npm run lint + + - name: Verify all + run: npm run verify:all + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + REDIS_URL: ${{ secrets.REDIS_URL }} + + run-seo-ops: + name: Run SEO Bot autonomous ops + needs: gate + runs-on: ubuntu-latest + timeout-minutes: 120 + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + REDIS_URL: ${{ secrets.REDIS_URL }} + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + SERPAPI_KEY: ${{ secrets.SERPAPI_KEY }} + DATAFORSEO_LOGIN: ${{ secrets.DATAFORSEO_LOGIN }} + DATAFORSEO_PASSWORD: ${{ secrets.DATAFORSEO_PASSWORD }} + POSTHOG_PROJECT_API_KEY: ${{ secrets.POSTHOG_PROJECT_API_KEY }} + COST_CAP_USD: ${{ inputs.cost_cap_usd || '2.00' }} + ANALYSIS_SCOPE: ${{ inputs.scope || 'delta' }} + TARGET_URL: ${{ inputs.target_url || '' }} + CLIENT_ID: ${{ inputs.client_id || '' }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: '22' + cache: 'npm' + + - name: Install dependencies + run: npm ci + + - name: Restore crawl cache + uses: actions/cache@v4 + with: + path: ~/.cache/seo-bot + key: seo-bot-crawl-${{ github.run_number }} + restore-keys: seo-bot-crawl- + + - name: Run autonomous SEO ops + run: node dist/scripts/autonomous-run.js + # This script: + # 1. Reads COST_CAP_USD, ANALYSIS_SCOPE, TARGET_URL, CLIENT_ID from env + # 2. Creates an agent_jobs row in Postgres with idempotency_key = hash(scope+date+client) + # 3. Dispatches a BullMQ job with the parameters + # 4. Polls job completion with a 110-minute timeout + # 5. Writes outputs/seo-report-{run_id}.json on success + + - name: Create recommendations PR + if: success() + run: | + BRANCH="bot/seo-${{ github.run_id }}" + git config user.email "seo-bot@quantum-l9.io" + git config user.name "SEO Bot" + git checkout -b "$BRANCH" + if git diff --quiet; then + echo "No file changes to commit" + else + git add outputs/ + git commit -m "chore(seo-bot): recommendations run ${{ github.run_id }}" + git push origin "$BRANCH" + gh pr create \ + --title "SEO Bot: recommendations ${{ github.run_id }}" \ + --body-file outputs/pr-body.md \ + --base main --head "$BRANCH" \ + --label "seo-bot,automated" || true + fi + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Upload SEO report artifact + if: always() + uses: actions/upload-artifact@v4 + with: + name: seo-report-${{ github.run_id }} + path: outputs/ + if-no-files-found: ignore diff --git a/adr/0005-autonomy-runtime-controls.md b/adr/0005-autonomy-runtime-controls.md new file mode 100644 index 0000000..2d649c8 --- /dev/null +++ b/adr/0005-autonomy-runtime-controls.md @@ -0,0 +1,61 @@ +# ADR-0005: Autonomy Runtime Controls + +**Status:** Proposed +**Date:** 2026-07-15 +**Supersedes:** None +**Related:** ADR-0001 (BullMQ), ADR-0002 (multi-tenancy), ADR-0003 (token budget) + +## Context + +SEO Bot operates as a 24/7 multi-tenant service on a Hetzner VPS with BullMQ + Redis and Postgres. +Existing job scheduling (BullMQ) and token budgeting (TokenBudget per scheduler job) cover per-run constraints well. + +Gaps identified in the L9 nuclear architecture brief: +1. No USD-level cross-run spend tracking or cap enforcement. +2. No registered compensation/rollback path for external SEO API writes or outreach sends. +3. No GitHub Actions workflow for autonomous full-site ops (only `ci.yml` for PR validation exists). +4. No `agent_jobs`, `budget_violations`, or `compensation_log` tables for runtime evidence. + +## Decision + +Add the following **without replacing or modifying any existing files**: + +| New file | Purpose | +|---|---| +| `.github/workflows/autonomy-ops.yml` | Manual/scheduled full-site ops workflow. Does not replace `ci.yml`. | +| `src/core/budget-guard.ts` | `AgentBudgetGuard` class — USD-level admission, reserve, reconcile, enforce. | +| `src/core/compensation.ts` | `CompensationRegistry` — saga rollback for external mutations. | +| `src/core/schema-additions.sql` | Additive DDL for `agent_jobs`, `budget_violations`, `compensation_log`. | + +## What does NOT change + +- `ci.yml` — untouched. +- `src/core/scheduler.ts` — untouched. Existing `TokenBudget` per-job controls remain primary. +- `src/core/config.ts` — untouched. +- `src/core/database/schema.ts` — the SQL DDL is provided separately; Drizzle migration is a follow-on task. +- `AGENTS.md` locked decisions (BullMQ, multi-tenancy, Pino logging, Drizzle, no Puppeteer/Playwright). +- Existing ADRs 0001–0004. + +## Integration path + +1. Apply `src/core/schema-additions.sql` to the Postgres database. +2. Add `AgentBudgetGuard` calls to new or high-cost job handlers where USD tracking is needed. +3. Add `CompensationRegistry` registration before external writes in outreach, SEO provider, and future live-site mutation steps. +4. Reflect new tables into `schema.ts` and generate a Drizzle migration (follow-on PR). +5. The `autonomy-ops.yml` workflow fires `node dist/scripts/autonomous-run.js` — this script must be created in a follow-on PR. + +## Budget mode thresholds + +| Pressure | Mode | Agent action | +|---|---|---| +| < 70% | `normal` | Continue | +| 70–85% | `cheaper_model` | Route to fast tier | +| 85–95% | `narrow_scope` | Reduce page/module scope | +| 95–100% | `require_approval` | Pause and notify | +| > 100% | `stop` | Throw `BudgetExceededError` | + +## Consequences + +- Every external SEO API write or outreach send that registers a compensation action becomes reversible. +- USD spend is visible per-run and per-client in Postgres for billing, auditing, and anomaly detection. +- The new `autonomy-ops.yml` workflow gives CI/CD entry point for full-site ops without touching the existing `ci.yml`. diff --git a/src/core/budget-guard.ts b/src/core/budget-guard.ts new file mode 100644 index 0000000..a74908b --- /dev/null +++ b/src/core/budget-guard.ts @@ -0,0 +1,119 @@ +/** + * AgentBudgetGuard — four-move runtime budget control loop for SEO Bot. + * + * Aligns with the existing TokenBudget policy in src/core/scheduler.ts. + * Per-client budget caps (maxFastTokensPerRun, maxStrategicTokensPerRun) remain + * the primary per-job controls — this guard adds USD-level tracking for cross-client + * aggregate spend and GitHub Actions workflow-level cost caps. + * + * Moves: Open (admission) → Reserve → Reconcile → Enforce + */ + +import { getConfig } from './config.js'; +import { logger } from './logger.js'; + +export class BudgetExceededError extends Error {} +export class AdmissionRejectedError extends Error {} + +export type BudgetMode = 'normal' | 'cheaper_model' | 'narrow_scope' | 'require_approval' | 'stop'; + +export interface BudgetEnforcement { + jobId: string; + clientId?: string; + mode: BudgetMode; + actualUsd: number; + reservedUsd: number; + remainingUsd: number; + forecastUsd: number; +} + +export class AgentBudgetGuard { + private capUsd: number; + private actualUsd = 0; + private reservedUsd = 0; + private forecastUsd = 0; + private mode: BudgetMode = 'normal'; + + constructor( + public readonly jobId: string, + capUsd: number, + public readonly clientId?: string, + ) { + this.capUsd = capUsd; + } + + /** MOVE 1 — Admission: verify forecast is feasible before enqueueing. */ + open(initialForecastUsd = 0): void { + this.forecastUsd = initialForecastUsd; + if (this.forecastUsd > this.capUsd) { + throw new AdmissionRejectedError( + `Admission rejected for job=${this.jobId} client=${this.clientId}: forecast $${this.forecastUsd.toFixed(4)} > cap $${this.capUsd.toFixed(4)}`, + ); + } + logger.info({ jobId: this.jobId, clientId: this.clientId, capUsd: this.capUsd }, 'budget_guard:opened'); + } + + /** MOVE 2 — Reserve: lock budget before each LLM call or API call with token cost. */ + reserve(estimatedUsd: number): void { + const remaining = this.capUsd - this.actualUsd - this.reservedUsd; + if (estimatedUsd > remaining) { + this._updateMode(); + const remainingAfterMode = this.capUsd - this.actualUsd - this.reservedUsd; + if (estimatedUsd > remainingAfterMode) { + throw new BudgetExceededError( + `Reservation denied for job=${this.jobId}: need $${estimatedUsd.toFixed(4)}, remaining $${remainingAfterMode.toFixed(4)}, mode=${this.mode}`, + ); + } + } + this.reservedUsd += estimatedUsd; + this.forecastUsd = this.actualUsd + this.reservedUsd; + } + + /** MOVE 3 — Reconcile: record actual spend after each step. */ + reconcile(actualUsd: number, nextEstimateUsd = 0): void { + this.actualUsd += actualUsd; + this.reservedUsd = Math.max(0, this.reservedUsd - actualUsd); + this.forecastUsd = this.actualUsd + this.reservedUsd + nextEstimateUsd; + logger.debug({ jobId: this.jobId, actualUsd: this.actualUsd, forecastUsd: this.forecastUsd }, 'budget_guard:reconciled'); + if (this.actualUsd > this.capUsd) { + throw new BudgetExceededError( + `Budget cap $${this.capUsd.toFixed(4)} exceeded for job=${this.jobId}: actual=$${this.actualUsd.toFixed(4)}`, + ); + } + if (this.forecastUsd > this.capUsd) this._updateMode(); + } + + /** MOVE 4 — Enforce: return state snapshot; throws if cap is fully exhausted. */ + enforce(): BudgetEnforcement { + const remaining = this.capUsd - this.actualUsd; + if (remaining <= 0) { + this.mode = 'stop'; + throw new BudgetExceededError( + `Cap exhausted for job=${this.jobId}: actual=$${this.actualUsd.toFixed(4)}, cap=$${this.capUsd.toFixed(4)}`, + ); + } + return { + jobId: this.jobId, + clientId: this.clientId, + mode: this.mode, + actualUsd: this.actualUsd, + reservedUsd: this.reservedUsd, + remainingUsd: remaining, + forecastUsd: this.forecastUsd, + }; + } + + get currentMode(): BudgetMode { + return this.mode; + } + + private _updateMode(): void { + const pressure = this.capUsd === 0 ? 1 : (this.actualUsd + this.reservedUsd) / this.capUsd; + if (pressure < 0.70) this.mode = 'normal'; + else if (pressure < 0.85) this.mode = 'cheaper_model'; + else if (pressure < 0.95) this.mode = 'narrow_scope'; + else if (pressure < 1.00) this.mode = 'require_approval'; + else this.mode = 'stop'; + logger.warn({ jobId: this.jobId, mode: this.mode, pressure: pressure.toFixed(3) }, 'budget_guard:mode_changed'); + } +} diff --git a/src/core/compensation.ts b/src/core/compensation.ts new file mode 100644 index 0000000..55e63b9 --- /dev/null +++ b/src/core/compensation.ts @@ -0,0 +1,64 @@ +/** + * CompensationRegistry — saga-pattern rollback for external mutations in SEO Bot. + * + * Covers: external SEO provider writes, outreach API calls, Postgres state writes, + * client-snippet pushes, and any future live-site mutations. + * + * Rule: register compensation BEFORE executing the mutating step. + * On failure, call compensate() to reverse registered actions in LIFO order. + */ + +import { logger } from './logger.js'; + +export interface CompensationEntry { + stepId: string; + clientId?: string; + action: () => Promise; + registeredAt: Date; +} + +export class CompensationRegistry { + private readonly entries: CompensationEntry[] = []; + + constructor( + public readonly jobId: string, + public readonly clientId?: string, + ) {} + + /** + * Register a compensation action for a step about to mutate external state. + * Always call this BEFORE the mutation, not after. + */ + register(stepId: string, action: () => Promise): void { + this.entries.push({ stepId, clientId: this.clientId, action, registeredAt: new Date() }); + logger.debug({ jobId: this.jobId, stepId, clientId: this.clientId }, 'compensation:registered'); + } + + /** + * Execute all compensations in reverse order (LIFO). + * Errors in individual compensations are collected but do not abort others. + */ + async compensate(): Promise<{ stepId: string; error?: string }[]> { + const results: { stepId: string; error?: string }[] = []; + for (const entry of [...this.entries].reverse()) { + try { + await entry.action(); + results.push({ stepId: entry.stepId }); + logger.info({ jobId: this.jobId, stepId: entry.stepId }, 'compensation:ok'); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + results.push({ stepId: entry.stepId, error: message }); + logger.error({ jobId: this.jobId, stepId: entry.stepId, message }, 'compensation:failed'); + } + } + return results; + } + + clear(): void { + this.entries.length = 0; + } + + get size(): number { + return this.entries.length; + } +} diff --git a/src/core/schema-additions.sql b/src/core/schema-additions.sql new file mode 100644 index 0000000..24addd9 --- /dev/null +++ b/src/core/schema-additions.sql @@ -0,0 +1,82 @@ +-- schema-additions.sql +-- Additive tables for SEO Bot runtime accounting. +-- Does NOT modify existing Drizzle schema.ts or existing migration files. +-- Apply via: psql $DATABASE_URL < src/core/schema-additions.sql +-- Then reflect into Drizzle by adding the table definitions to schema.ts and generating a migration. + +-- ─── Agent jobs (cross-run accounting) ─────────────────────────────────────────── + +CREATE TABLE IF NOT EXISTS agent_jobs ( + job_id UUID PRIMARY KEY DEFAULT gen_random_uuid(), + repo VARCHAR(100) NOT NULL DEFAULT 'seo-bot', + client_id VARCHAR(255), -- NULL = all clients + trigger_type VARCHAR(50) NOT NULL, -- 'cron' | 'api' | 'dispatch' | 'manual' + trigger_payload JSONB, + status VARCHAR(20) NOT NULL DEFAULT 'queued', -- queued | running | success | failed | suspended + assigned_worker VARCHAR(100), + attempt_count INT NOT NULL DEFAULT 0, + max_attempts INT NOT NULL DEFAULT 3, + cost_usd NUMERIC(10,6) NOT NULL DEFAULT 0, + cost_cap_usd NUMERIC(10,6) NOT NULL DEFAULT 2.00, + created_at TIMESTAMPTZ NOT NULL DEFAULT now(), + started_at TIMESTAMPTZ, + completed_at TIMESTAMPTZ, + error_message TEXT, + result_artifact JSONB, + idempotency_key VARCHAR(255) UNIQUE +); + +CREATE INDEX IF NOT EXISTS idx_seo_agent_jobs_status_created + ON agent_jobs (status, created_at) + WHERE status IN ('queued', 'running'); + +CREATE INDEX IF NOT EXISTS idx_seo_agent_jobs_client_id + ON agent_jobs (client_id, created_at DESC); + +-- ─── Budget violations ──────────────────────────────────────────────────────── + +CREATE TABLE IF NOT EXISTS budget_violations ( + id BIGSERIAL PRIMARY KEY, + job_id UUID NOT NULL REFERENCES agent_jobs(job_id), + repo VARCHAR(100) NOT NULL DEFAULT 'seo-bot', + client_id VARCHAR(255), + cost_usd NUMERIC(10,6) NOT NULL, + cost_cap_usd NUMERIC(10,6) NOT NULL, + overage_usd NUMERIC(10,6) NOT NULL, + created_at TIMESTAMPTZ NOT NULL DEFAULT now() +); + +CREATE OR REPLACE FUNCTION log_seo_budget_violation() +RETURNS TRIGGER +LANGUAGE plpgsql +AS $$ +BEGIN + IF NEW.cost_usd IS NOT NULL + AND NEW.cost_cap_usd IS NOT NULL + AND NEW.cost_usd > NEW.cost_cap_usd + AND (OLD.cost_usd IS NULL OR OLD.cost_usd <= OLD.cost_cap_usd) + THEN + INSERT INTO budget_violations (job_id, repo, client_id, cost_usd, cost_cap_usd, overage_usd) + VALUES (NEW.job_id, 'seo-bot', NEW.client_id, NEW.cost_usd, NEW.cost_cap_usd, NEW.cost_usd - NEW.cost_cap_usd); + END IF; + RETURN NEW; +END; +$$; + +DROP TRIGGER IF EXISTS trg_seo_budget_violation ON agent_jobs; +CREATE TRIGGER trg_seo_budget_violation + AFTER INSERT OR UPDATE OF cost_usd, cost_cap_usd ON agent_jobs + FOR EACH ROW EXECUTE FUNCTION log_seo_budget_violation(); + +-- ─── Compensation log ───────────────────────────────────────────────────────── + +CREATE TABLE IF NOT EXISTS compensation_log ( + id BIGSERIAL PRIMARY KEY, + job_id UUID, + client_id VARCHAR(255), + step_id VARCHAR(255) NOT NULL, + action_type VARCHAR(100) NOT NULL, -- e.g. 'seo-provider-rollback', 'outreach-cancel' + status VARCHAR(20) NOT NULL, -- 'success' | 'failed' + error_message TEXT, + compensated_at TIMESTAMPTZ NOT NULL DEFAULT now() +);