From faebe5c52ed7acc2bbd5fd6941339f0e8ab837ae Mon Sep 17 00:00:00 2001 From: t Date: Fri, 6 Nov 2026 21:52:14 -0500 Subject: [PATCH 1/4] Add apply_sql, capture_env_rows and diff_env_rows task steps apply_sql runs SQL in a deployed env's service database in one transaction, then deletes the changelog rows it wrote and advances the service sequences, so a run can start from a base universe plus exactly the declared SQL. capture_env_rows and diff_env_rows compare two states of an env row by row and record extra, missing and changed rows. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/task_step/registry.py | 5 + .../task_step/task_steps/apply_sql.py | 188 ++++++++++++ .../task_step/task_steps/env_rows.py | 284 ++++++++++++++++++ .../task_step/task_steps/service_db_sql.py | 97 ++++++ .../task_step/test_service_db_sql_postgres.py | 238 +++++++++++++++ tst/unit/task_step/test_apply_sql.py | 181 +++++++++++ 6 files changed, 993 insertions(+) create mode 100644 src/agent_env/task_step/task_steps/apply_sql.py create mode 100644 src/agent_env/task_step/task_steps/env_rows.py create mode 100644 src/agent_env/task_step/task_steps/service_db_sql.py create mode 100644 tst/integration/task_step/test_service_db_sql_postgres.py create mode 100644 tst/unit/task_step/test_apply_sql.py diff --git a/src/agent_env/task_step/registry.py b/src/agent_env/task_step/registry.py index 1b9a3f298..5802beecb 100644 --- a/src/agent_env/task_step/registry.py +++ b/src/agent_env/task_step/registry.py @@ -49,6 +49,8 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: from agent_env.task_step.task_steps.deploy_human_agent import DeployHumanAgentTaskStep from agent_env.task_step.task_steps.deploy_sandbox import DeploySandboxTaskStep from agent_env.task_step.task_steps.reset_env import ResetEnvTaskStep + from agent_env.task_step.task_steps.apply_sql import ApplySqlTaskStep + from agent_env.task_step.task_steps.env_rows import CaptureEnvRowsTaskStep, DiffEnvRowsTaskStep from agent_env.task_step.task_steps.teardown_sandboxes import TeardownSandboxesTaskStep from agent_env.task_step.task_steps.verifiers.env_outcome_verifier import EnvOutcomeVerifierTaskStep from agent_env.task_step.task_steps.load_artifact import LoadArtifactTaskStep @@ -114,6 +116,9 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: _get_type(DeployHumanAgentTaskStep): DeployHumanAgentTaskStep, _get_type(DeploySandboxTaskStep): DeploySandboxTaskStep, _get_type(ResetEnvTaskStep): ResetEnvTaskStep, + _get_type(ApplySqlTaskStep): ApplySqlTaskStep, + _get_type(CaptureEnvRowsTaskStep): CaptureEnvRowsTaskStep, + _get_type(DiffEnvRowsTaskStep): DiffEnvRowsTaskStep, _get_type(TeardownSandboxesTaskStep): TeardownSandboxesTaskStep, _get_type(EnvOutcomeVerifierTaskStep): EnvOutcomeVerifierTaskStep, _get_type(LoadArtifactTaskStep): LoadArtifactTaskStep, diff --git a/src/agent_env/task_step/task_steps/apply_sql.py b/src/agent_env/task_step/task_steps/apply_sql.py new file mode 100644 index 000000000..38efaf2f0 --- /dev/null +++ b/src/agent_env/task_step/task_steps/apply_sql.py @@ -0,0 +1,188 @@ +"""Apply SQL to a deployed env's service database, as part of building the state a run starts from. + +Typical use: ``deploy_env -> load_artifact (a base universe) -> apply_sql -> deploy_agent -> prompt_agent``, so the +agent starts from the base universe plus exactly the declared SQL, and nothing else. + +The SQL runs as one psql script in a single transaction (``psql -1``) and stops at its first error, which fails the +step with psql's message (it names the statement's line). After it: + +- the env's changelog rows the SQL wrote are deleted, so the run starts with a changelog that holds only what + happens after this step, as a freshly loaded universe does; nothing the SQL did shows up as an agent's change; +- every serial / identity sequence in the services' schemas is advanced past its column's largest value, so an + insert with explicit ids doesn't make the services' own later inserts collide; +- ``context.metadata["applied_sql"][]`` records the SQL's sha256 and size, the rows it wrote per table and + operation (read from the changelog before it was cleared), and any volatile functions (``now()``, ``random()``, + ...) it calls: those give a different result on every apply, so a rebuild can't reproduce their rows exactly. + +Blank SQL is a no-op that still records the sha256 of the empty string, so a task built from an empty field runs on +the base universe alone. +""" +from __future__ import annotations + +import hashlib +import json +import logging +import re +from typing import Optional + +from agent_env.entity.refs import EntityRef +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_step import TaskStep +from agent_env.task_step.task_steps.service_db_sql import ServiceDbSqlError, service_db_host + +logger = logging.getLogger(__name__) + +SUMMARY_MARKER = "__agentenv_apply_sql_summary__" + +# Calls whose result differs on every apply. +_VOLATILE = re.compile( + r"\b(now|clock_timestamp|statement_timestamp|transaction_timestamp|timeofday|random|gen_random_uuid|" + r"uuid_generate_v[14])\s*\(|\b(current_timestamp|current_date|current_time|localtimestamp|localtime)\b", + re.IGNORECASE, +) + +# Before the SQL: remember where the changelog ends. +_PREAMBLE = """\ +CREATE TEMP TABLE _agentenv_apply_sql_mark AS SELECT coalesce(max(id), 0) AS pre FROM public._changelog; +""" + +# After the SQL: summarize and clear its changelog rows, then advance every service sequence past its column's max. +_EPILOGUE = f"""\ +; +\\echo {SUMMARY_MARKER} +SELECT coalesce(jsonb_agg(s ORDER BY s.schema_name, s.table_name, s.operation), '[]'::jsonb) +FROM (SELECT schema_name, table_name, operation, count(*) AS rows FROM public._changelog + WHERE id > (SELECT pre FROM _agentenv_apply_sql_mark) GROUP BY 1, 2, 3) s; +DELETE FROM public._changelog WHERE id > (SELECT pre FROM _agentenv_apply_sql_mark); +DO $agentenv_sync$ +DECLARE r record; mx bigint; cur bigint; called boolean; +BEGIN + FOR r IN + SELECT n.nspname AS sch, c.relname AS tbl, a.attname AS col, + pg_get_serial_sequence(format('%I.%I', n.nspname, c.relname), a.attname) AS seq + FROM pg_attribute a + JOIN pg_class c ON c.oid = a.attrelid + JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind IN ('r', 'p') AND a.attnum > 0 AND NOT a.attisdropped + AND n.nspname NOT IN ('pg_catalog', 'information_schema', 'agent_env_internal') + AND n.nspname NOT LIKE 'pg\\_%' + AND pg_get_serial_sequence(format('%I.%I', n.nspname, c.relname), a.attname) IS NOT NULL + LOOP + EXECUTE format('SELECT max(%I)::bigint FROM %I.%I', r.col, r.sch, r.tbl) INTO mx; + EXECUTE format('SELECT last_value, is_called FROM %s', r.seq) INTO cur, called; + IF mx IS NOT NULL AND (mx > cur OR (mx = cur AND NOT called)) THEN + PERFORM setval(r.seq, mx, true); + END IF; + END LOOP; +END +$agentenv_sync$; +""" + + +def volatile_functions(sql: str) -> list[str]: + """The volatile calls ``sql`` makes, lower-cased and de-duplicated, in order of first use.""" + seen: dict[str, None] = {} + for m in _VOLATILE.finditer(sql): + seen.setdefault((m.group(1) or m.group(2)).lower(), None) + return list(seen) + + +def build_script(sql: str) -> str: + """The psql script ``apply_sql`` runs: the preamble, the SQL, then the epilogue.""" + return _PREAMBLE + sql.rstrip() + "\n" + _EPILOGUE + + +_PSQL_LINE = re.compile(r"psql::(\d+):") + + +def sql_relative_error(message: str) -> str: + """psql's ``psql::N:`` positions, which count the preamble's lines, as lines of the SQL itself.""" + offset = _PREAMBLE.count("\n") + + def line(m: re.Match) -> str: + n = int(m.group(1)) - offset + return f"line {n} of the SQL:" if n >= 1 else m.group(0) + + return _PSQL_LINE.sub(line, message) + + +def parse_summary(stdout: str) -> list[dict]: + """The per-table write counts the epilogue prints after its marker; [] when absent or unparsable.""" + _, found, rest = stdout.partition(SUMMARY_MARKER) + if not found: + return [] + for line in rest.splitlines(): + line = line.strip() + if line.startswith("["): + try: + return json.loads(line) + except ValueError: + return [] + return [] + + +class ApplySqlTaskStep(TaskStep): + """Run SQL against a deployed env's service database, then clear the changelog rows it wrote.""" + + type = "apply_sql" + entity_refs = (EntityRef.env("env_id"),) + + def __init__( + self, + id: str, + version: Optional[int], + env_id: str, + sql: str = "", + depends_on: Optional[list] = None, + fail_task_on_error: bool = True, + ): + if not env_id: + raise ValueError("apply_sql requires an env_id (the env whose service database the SQL runs in)") + if not isinstance(sql, str): + raise ValueError("apply_sql: `sql` must be a string") + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + self.env_id = env_id + self.sql = sql + + def to_dict(self) -> dict: + base = super().to_dict() + base["env_id"] = self.env_id + base["sql"] = self.sql + return base + + @classmethod + def from_dict(cls, data: dict) -> "ApplySqlTaskStep": + if not data.get("env_id"): + raise ValueError("apply_sql step requires 'env_id'") + sql = data.get("sql") + return cls(**cls._base_from_dict(data), env_id=data["env_id"], sql="" if sql is None else sql) + + async def execute(self, context: TaskStepContext) -> TaskStepContext: + sql = self.sql or "" + record: dict = { + "env_id": self.env_id, + "sql_sha256": hashlib.sha256(sql.encode("utf-8")).hexdigest(), + "sql_bytes": len(sql.encode("utf-8")), + "volatile_functions": volatile_functions(sql), + "changes": [], + "applied": False, + } + context.metadata.setdefault("applied_sql", {})[self.id] = record + if not sql.strip(): + logger.info("apply_sql '%s': no SQL; env '%s' keeps the state it was loaded with", self.id, self.env_id) + return context + if record["volatile_functions"]: + logger.warning("apply_sql '%s': the SQL calls %s, which give a different result on every apply", + self.id, ", ".join(record["volatile_functions"])) + host = await service_db_host(context, self.env_id, "apply_sql") + what = f"apply_sql '{self.id}' on env '{self.env_id}'" + try: + out = await host.run_sql_file(what, build_script(sql), "-1", "-A", "-t") + except ServiceDbSqlError as e: + raise ServiceDbSqlError(what, e.exit_code, sql_relative_error(e.message)) from None + record["changes"] = parse_summary(out) + record["applied"] = True + logger.info("apply_sql '%s': applied %d bytes to env '%s': %s", self.id, record["sql_bytes"], self.env_id, + ", ".join(f"{c['operation']} {c['rows']} {c['schema_name']}.{c['table_name']}" + for c in record["changes"]) or "no rows written") + return context diff --git a/src/agent_env/task_step/task_steps/env_rows.py b/src/agent_env/task_step/task_steps/env_rows.py new file mode 100644 index 000000000..cfd8e0022 --- /dev/null +++ b/src/agent_env/task_step/task_steps/env_rows.py @@ -0,0 +1,284 @@ +"""Row-level capture and diff of a deployed env's service database. + +``capture_env_rows`` writes every row of every service table to a file on the env's VM host, one line per row: +``. TAB TAB TAB ``. The key is the row's primary key as a JSON +array, or ``row:`` for a table without one. ``ignore_columns`` names columns left out of every row (and its md5), +e.g. timestamps a rebuild can't reproduce. + +``diff_env_rows`` compares two captures of the same env (say: a base universe plus declared SQL, and the universe a run +actually started from), row by row, never by SQL text, and records ``context.metadata["verifications"][]``: + + {"type": "env_rows_diff", "match": bool, "expected": , "actual": , + "counts": {"extra": n, "missing": n, "changed": n}, + "by_table": {".
": {"extra": n, "missing": n, "changed": n}}, + "examples": {"extra": [{"table", "key", "row"}], "missing": [...], "changed": [{"table", "key", "expected", + "actual"}]}} + +``extra`` rows are in ``actual`` only, ``missing`` rows in ``expected`` only, ``changed`` rows share a key but differ. +With ``fail_on_mismatch`` a mismatch also fails the step. + +Typical task: ``deploy_env -> load_artifact(base) -> apply_sql -> capture_env_rows(expected) -> +load_artifact(run universe) -> capture_env_rows(actual) -> diff_env_rows``. +""" +from __future__ import annotations + +import json +import logging +import re +import shlex +from collections import defaultdict +from typing import Optional + +from agent_env.entity.refs import EntityRef +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_step import TaskStep +from agent_env.task_step.task_steps.service_db_sql import WORK_DIR, ServiceDbHost, service_db_host + +logger = logging.getLogger(__name__) + +_NAME = re.compile(r"^[A-Za-z0-9_-]{1,64}$") + +# Emits one COPY per service table and runs each (\gexec); every COPY writes that table's rows as capture lines. +_CAPTURE_SQL = """\ +SELECT format( + 'COPY (SELECT %L, %s, md5((to_jsonb(t) - %L::text[])::text), (to_jsonb(t) - %L::text[])::text FROM %I.%I t) TO STDOUT', + n.nspname || '.' || c.relname, + coalesce( + (SELECT 'jsonb_build_array(' || string_agg(format('t.%I', a.attname), ', ' ORDER BY k.ord) || ')::text' + FROM pg_index i + CROSS JOIN LATERAL unnest(i.indkey::int2[]) WITH ORDINALITY AS k(attnum, ord) + JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = k.attnum + WHERE i.indrelid = c.oid AND i.indisprimary), + format('''row:'' || md5((to_jsonb(t) - %L::text[])::text)', :'ignore')), + :'ignore', :'ignore', n.nspname, c.relname) +FROM pg_class c +JOIN pg_namespace n ON n.oid = c.relnamespace +WHERE c.relkind IN ('r', 'p') + AND NOT c.relispartition + AND n.nspname NOT IN ('pg_catalog', 'information_schema', 'agent_env_internal') + AND n.nspname NOT LIKE 'pg\\_%' + AND NOT (n.nspname = 'public' AND c.relname = '_changelog') +ORDER BY n.nspname, c.relname +\\gexec +""" + + +def _check_name(value: str, field: str, step: str) -> str: + if not isinstance(value, str) or not _NAME.match(value): + raise ValueError(f"{step}: `{field}` must be 1-64 letters, digits, '_' or '-'; got {value!r}") + return value + + +def capture_path(name: str) -> str: + return f"{WORK_DIR}/captures/{name}.tsv" + + +def pg_text_array(values: list[str]) -> str: + """``values`` as a PostgreSQL text-array literal, e.g. ``{"a","b"}``.""" + return "{" + ",".join('"' + v.replace("\\", "\\\\").replace('"', '\\"') + '"' for v in values) + "}" + + +def parse_capture_index(text: str) -> dict[str, dict[str, list[str]]]: + """``
TAB TAB `` lines as {table: {key: [md5, ...]}} (a key repeats in a table without a primary + key).""" + index: dict[str, dict[str, list[str]]] = defaultdict(lambda: defaultdict(list)) + for line in text.splitlines(): + if not line: + continue + parts = line.split("\t") + if len(parts) < 3: + continue + index[parts[0]][parts[1]].append(parts[2]) + return index + + +def diff_indexes(expected: dict, actual: dict) -> dict[str, dict[str, list[str]]]: + """{"extra" | "missing" | "changed": [(table, key), ...]} between two capture indexes, each list sorted.""" + out: dict[str, list] = {"extra": [], "missing": [], "changed": []} + for table in sorted(set(expected) | set(actual)): + exp, act = expected.get(table, {}), actual.get(table, {}) + for key in sorted(set(exp) | set(act)): + e, a = sorted(exp.get(key, [])), sorted(act.get(key, [])) + if e == a: + continue + if not e: + out["extra"].extend([(table, key)] * len(a)) + elif not a: + out["missing"].extend([(table, key)] * len(e)) + elif len(e) == len(a): + out["changed"].append((table, key)) + else: # same key, a different number of copies: a table without a primary key + kind = "extra" if len(a) > len(e) else "missing" + out[kind].extend([(table, key)] * abs(len(a) - len(e))) + return out + + +def _unescape_copy(value: str) -> str: + """Undo COPY text-format escaping (``\\t``, ``\\n``, ``\\\\``, ...) in one field.""" + return re.sub(r"\\(.)", lambda m: {"t": "\t", "n": "\n", "r": "\r", "b": "\b", "f": "\f", "v": "\v"}.get( + m.group(1), m.group(1)), value) + + +def _row(text: Optional[str]): + if text is None: + return None + raw = _unescape_copy(text) + try: + return json.loads(raw) + except ValueError: + return raw + + +class CaptureEnvRowsTaskStep(TaskStep): + """Capture every row of a deployed env's service tables to a named file on its VM host.""" + + type = "capture_env_rows" + entity_refs = (EntityRef.env("env_id"),) + + def __init__(self, id: str, version: Optional[int], env_id: str, capture: str, + ignore_columns: Optional[list[str]] = None, depends_on: Optional[list] = None, + fail_task_on_error: bool = True): + if not env_id: + raise ValueError("capture_env_rows requires an env_id") + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + self.env_id = env_id + self.capture = _check_name(capture, "capture", "capture_env_rows") + self.ignore_columns = [str(c) for c in (ignore_columns or [])] + + def to_dict(self) -> dict: + base = super().to_dict() + base.update(env_id=self.env_id, capture=self.capture, ignore_columns=self.ignore_columns) + return base + + @classmethod + def from_dict(cls, data: dict) -> "CaptureEnvRowsTaskStep": + if not data.get("env_id"): + raise ValueError("capture_env_rows step requires 'env_id'") + return cls(**cls._base_from_dict(data), env_id=data["env_id"], capture=data.get("capture", ""), + ignore_columns=data.get("ignore_columns")) + + async def execute(self, context: TaskStepContext) -> TaskStepContext: + host = await service_db_host(context, self.env_id, "capture_env_rows") + path = capture_path(self.capture) + sql_path = f"{WORK_DIR}/capture-{self.capture}.sql" + await host.write(sql_path, _CAPTURE_SQL) + ignore = pg_text_array(self.ignore_columns) + cmd = (f"mkdir -p {shlex.quote(WORK_DIR)}/captures && " + f"{host.psql('-A', '-t', '-v', shlex.quote('ignore=' + ignore), '-f', '-')} " + f"< {shlex.quote(sql_path)} > {shlex.quote(path)}") + code, _, err = await host.run(cmd) + await host.run(f"rm -f {shlex.quote(sql_path)}") + if code != 0: + from agent_env.task_step.task_steps.service_db_sql import ServiceDbSqlError, _psql_error + + raise ServiceDbSqlError(f"capture_env_rows '{self.id}' on env '{self.env_id}'", code, _psql_error(err, "")) + code, counts, err = await host.run(f"cut -f1 {shlex.quote(path)} | sort | uniq -c") + tables = {} + for line in counts.splitlines(): + n, _, table = line.strip().partition(" ") + if table: + tables[table] = int(n) + context.metadata.setdefault("env_row_captures", {})[self.capture] = { + "env_id": self.env_id, "path": path, "rows": sum(tables.values()), "tables": tables, + "ignore_columns": self.ignore_columns, + } + logger.info("capture_env_rows '%s': %d rows in %d tables of env '%s' -> %s", self.id, sum(tables.values()), + len(tables), self.env_id, path) + return context + + +class DiffEnvRowsTaskStep(TaskStep): + """Diff two row captures of an env; record extra, missing and changed rows as a verification.""" + + type = "diff_env_rows" + entity_refs = (EntityRef.env("env_id"),) + + def __init__(self, id: str, version: Optional[int], env_id: str, expected: str, actual: str, + max_examples: int = 50, fail_on_mismatch: bool = False, depends_on: Optional[list] = None, + fail_task_on_error: bool = True): + if not env_id: + raise ValueError("diff_env_rows requires an env_id") + super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) + self.env_id = env_id + self.expected = _check_name(expected, "expected", "diff_env_rows") + self.actual = _check_name(actual, "actual", "diff_env_rows") + if self.expected == self.actual: + raise ValueError("diff_env_rows: `expected` and `actual` must name different captures") + self.max_examples = max(0, int(max_examples)) + self.fail_on_mismatch = bool(fail_on_mismatch) + + def to_dict(self) -> dict: + base = super().to_dict() + base.update(env_id=self.env_id, expected=self.expected, actual=self.actual, max_examples=self.max_examples, + fail_on_mismatch=self.fail_on_mismatch) + return base + + @classmethod + def from_dict(cls, data: dict) -> "DiffEnvRowsTaskStep": + if not data.get("env_id"): + raise ValueError("diff_env_rows step requires 'env_id'") + return cls(**cls._base_from_dict(data), env_id=data["env_id"], expected=data.get("expected", ""), + actual=data.get("actual", ""), max_examples=data.get("max_examples", 50), + fail_on_mismatch=data.get("fail_on_mismatch", False)) + + async def _index(self, host: ServiceDbHost, name: str) -> dict: + path = capture_path(name) + code, out, err = await host.run(f"test -f {shlex.quote(path)} && cut -f1-3 {shlex.quote(path)}") + if code != 0: + raise RuntimeError(f"diff_env_rows '{self.id}': no capture '{name}' on env '{self.env_id}'; run a " + f"capture_env_rows step with capture='{name}' first ({err.strip()[-200:]})") + return parse_capture_index(out) + + async def _rows(self, host: ServiceDbHost, name: str, keys: list[tuple[str, str]]) -> dict[tuple[str, str], str]: + """The row JSON of each (table, key) in capture ``name``.""" + if not keys: + return {} + keys_path = f"{WORK_DIR}/diff-{self.id}-{name}.keys" + await host.write(keys_path, "".join(f"{t}\t{k}\n" for t, k in keys)) + script = ("awk -F'\\t' 'NR==FNR{want[$0]=1; next} (($1 FS $2) in want){print $1 FS $2 FS $4}' " + f"{shlex.quote(keys_path)} {shlex.quote(capture_path(name))}; rm -f {shlex.quote(keys_path)}") + _, out, _ = await host.run(script) + rows: dict[tuple[str, str], str] = {} + for line in out.splitlines(): + parts = line.split("\t", 2) + if len(parts) == 3: + rows.setdefault((parts[0], parts[1]), parts[2]) + return rows + + async def execute(self, context: TaskStepContext) -> TaskStepContext: + host = await service_db_host(context, self.env_id, "diff_env_rows") + expected, actual = await self._index(host, self.expected), await self._index(host, self.actual) + diff = diff_indexes(expected, actual) + by_table: dict[str, dict[str, int]] = {} + for kind, items in diff.items(): + for table, _ in items: + by_table.setdefault(table, {"extra": 0, "missing": 0, "changed": 0})[kind] += 1 + n = self.max_examples + extra, missing, changed = diff["extra"][:n], diff["missing"][:n], diff["changed"][:n] + exp_rows = await self._rows(host, self.expected, sorted(set(missing + changed))) + act_rows = await self._rows(host, self.actual, sorted(set(extra + changed))) + result = { + "type": "env_rows_diff", + "match": not any(diff.values()), + "expected": self.expected, + "actual": self.actual, + "counts": {k: len(v) for k, v in diff.items()}, + "by_table": by_table, + "examples": { + "extra": [{"table": t, "key": k, "row": _row(act_rows.get((t, k)))} for t, k in extra], + "missing": [{"table": t, "key": k, "row": _row(exp_rows.get((t, k)))} for t, k in missing], + "changed": [{"table": t, "key": k, "expected": _row(exp_rows.get((t, k))), + "actual": _row(act_rows.get((t, k)))} for t, k in changed], + }, + } + context.metadata.setdefault("verifications", {})[self.id] = result + summary = ", ".join(f"{v} {k}" for k, v in result["counts"].items()) + if result["match"]: + logger.info("diff_env_rows '%s': '%s' and '%s' match row for row", self.id, self.expected, self.actual) + else: + logger.warning("diff_env_rows '%s': '%s' vs '%s' differ: %s (tables: %s)", self.id, self.expected, + self.actual, summary, ", ".join(sorted(by_table))) + if self.fail_on_mismatch: + raise RuntimeError(f"diff_env_rows '{self.id}': the env's rows differ from '{self.expected}': {summary}" + f" in {', '.join(sorted(by_table))}") + return context diff --git a/src/agent_env/task_step/task_steps/service_db_sql.py b/src/agent_env/task_step/task_steps/service_db_sql.py new file mode 100644 index 000000000..104d5db8d --- /dev/null +++ b/src/agent_env/task_step/task_steps/service_db_sql.py @@ -0,0 +1,97 @@ +"""Run SQL in a deployed env's service database, the PostgreSQL every MCP service of a gateway env keeps its state in. + +The database runs as the ``servicedb`` compose service on the env's VM host, so SQL reaches it with +``docker compose exec servicedb psql`` from that host, as the env snapshot store does. Envs whose state lives +elsewhere (a container-mode or external store) are refused with a message saying so. +""" +from __future__ import annotations + +import logging +import shlex +import uuid +from dataclasses import dataclass +from typing import Any + +from agent_env.env.env import DeployedSandboxEnv, require_sandbox +from agent_env.task_step.context import TaskStepContext + +logger = logging.getLogger(__name__) + +# The directory on the env's VM host that holds the SQL files and row captures these steps write. +WORK_DIR = "/tmp/agentenv-service-db-sql" + + +class ServiceDbSqlError(RuntimeError): + """psql exited non-zero; ``message`` is its error text, e.g. ``psql::3: ERROR: relation "x" does not + exist``.""" + + def __init__(self, what: str, exit_code: int, message: str): + self.exit_code = exit_code + self.message = message + super().__init__(f"{what} failed (psql exit {exit_code}): {message}") + + +@dataclass +class ServiceDbHost: + """The env's VM host, and the psql command line that reaches its service database from there.""" + + sandbox: Any + env_id: str + + def psql(self, *flags: str) -> str: + from agent_env.env.envs.service_db import DATABASE_SERVICE_NAME, DB_NAME, DB_USER + from agent_env.providers.env_providers.constants import DOCKER_COMPOSE_PATH + + # -X: no psqlrc. -q: no command tags. ON_ERROR_STOP: the first error ends the script with exit 3. + return " ".join( + ["docker", "compose", "-f", shlex.quote(DOCKER_COMPOSE_PATH), "exec", "-T", DATABASE_SERVICE_NAME, + "psql", "-X", "-q", "-v", "ON_ERROR_STOP=1", "-U", DB_USER, "-d", DB_NAME, *flags] + ) + + async def run(self, script: str) -> tuple[int, str, str]: + """Run a bash script on the host as root, returning (exit code, stdout, stderr).""" + return await self.sandbox.exec_with_output("sudo", "bash", "-c", script) + + async def write(self, path: str, text: str) -> None: + await self.sandbox.write_host_file(text.encode("utf-8"), path) + + async def run_sql_file(self, what: str, sql: str, *flags: str) -> str: + """Run ``sql`` as one psql script fed on stdin; return its stdout, raise ServiceDbSqlError on a failure.""" + path = f"{WORK_DIR}/{uuid.uuid4().hex}.sql" + await self.write(path, sql) + try: + code, out, err = await self.run(f"{self.psql(*flags, '-f', '-')} < {shlex.quote(path)}") + finally: + await self.run(f"rm -f {shlex.quote(path)}") + if code != 0: + raise ServiceDbSqlError(what, code, _psql_error(err, out)) + return out + + +def _psql_error(stderr: str, stdout: str) -> str: + """psql's error lines, else the tail of what it printed. A missing ``servicedb`` service is named plainly.""" + text = stderr.strip() or stdout.strip() + if "no such service" in text or "is not running" in text: + return ("this env has no running 'servicedb' service on its VM host, so its state is not in a local " + f"service database these steps can reach ({text[-300:]})") + lines = [line for line in text.splitlines() if "ERROR" in line or "FATAL" in line or line.startswith(("LINE", "DETAIL", "HINT", "CONTEXT"))] + return ("\n".join(lines) if lines else text)[-2000:] + + +async def service_db_host(context: TaskStepContext, env_id: str, what: str) -> ServiceDbHost: + """The VM host of ``env_id``'s deploy in this run.""" + from agent_env.providers.sandbox_providers.sandbox import VmSandbox + from agent_env.providers.sandbox_providers.sandbox_provider import build_sandbox_provider, get_env_sandbox_provider + + deployed = next((d for d in context.deployed_envs if d.env_id == env_id), None) + if deployed is None: + raise RuntimeError(f"{what}: env '{env_id}' is not deployed in this run; put a deploy_env step for it first") + deployed: DeployedSandboxEnv = require_sandbox(deployed, what) + if not deployed.sandbox_id: + raise RuntimeError(f"{what}: env '{env_id}' has no sandbox_id") + provider = build_sandbox_provider(deployed.sandbox_type) if deployed.sandbox_type else get_env_sandbox_provider() + sandbox = await provider.get_sandbox(deployed.sandbox_id) + if not isinstance(sandbox, VmSandbox): + raise RuntimeError(f"{what}: env '{env_id}' runs in a container sandbox; its service database is only " + "reachable on a VM-hosted env") + return ServiceDbHost(sandbox=sandbox, env_id=env_id) diff --git a/tst/integration/task_step/test_service_db_sql_postgres.py b/tst/integration/task_step/test_service_db_sql_postgres.py new file mode 100644 index 000000000..667987313 --- /dev/null +++ b/tst/integration/task_step/test_service_db_sql_postgres.py @@ -0,0 +1,238 @@ +"""apply_sql, capture_env_rows and diff_env_rows against a real PostgreSQL 16 holding a service schema set up the way a +gateway env's service database is: the shared changelog objects from the state provider's init script and changelog +triggers on the service's tables. + +The steps reach the database through ``ServiceDbHost``; here its psql runs through ``docker exec`` into a local +postgres container and its shell commands run on this machine, so everything but the env's VM is real: the scripts, +psql's single transaction and ``\\gexec``, the triggers, and the ``cut`` / ``awk`` the diff reads captures with. +""" +import asyncio +import os +import shutil +import subprocess +import uuid + +import pytest + +from agent_env.providers.env_state.env_state_provider import DatabaseStateProvider +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.task_steps import apply_sql as apply_sql_module +from agent_env.task_step.task_steps import env_rows as env_rows_module +from agent_env.task_step.task_steps import service_db_sql +from agent_env.task_step.task_steps.apply_sql import ApplySqlTaskStep +from agent_env.task_step.task_steps.env_rows import CaptureEnvRowsTaskStep, DiffEnvRowsTaskStep +from agent_env.task_step.task_steps.service_db_sql import ServiceDbHost, ServiceDbSqlError +from tst.util.capabilities import missing_capability_reason + +pytestmark = pytest.mark.integration + +_IMAGE = "public.ecr.aws/docker/library/postgres:16-alpine" + +_BASE = """\ +CREATE TABLE slack.channels (id serial PRIMARY KEY, name text NOT NULL); +CREATE TABLE slack.messages (id serial PRIMARY KEY, channel_id int REFERENCES slack.channels(id), body text, + posted_at timestamptz NOT NULL DEFAULT '2026-02-14T09:00:00Z'); +CREATE TABLE slack.reactions (message_id int, emoji text); -- no primary key +INSERT INTO slack.channels (name) VALUES ('general'), ('bugs'); +INSERT INTO slack.messages (channel_id, body) VALUES (1, 'hello'), (2, 'login is broken'); +INSERT INTO slack.reactions VALUES (2, 'eyes'); +SELECT _install_changelog_triggers('slack'); +""" + + +def _docker_ok() -> bool: + if not shutil.which("docker"): + return False + return subprocess.run(["docker", "info"], capture_output=True).returncode == 0 + + +class _LocalHost(ServiceDbHost): + """psql through ``docker exec`` into the test container; shell commands on this machine.""" + + def __init__(self, container: str, work_dir: str): + super().__init__(sandbox=None, env_id="test-env") + self.container, self.work_dir = container, work_dir + + def psql(self, *flags: str) -> str: + import shlex + + return " ".join(["docker", "exec", "-i", self.container, "psql", "-X", "-q", "-v", "ON_ERROR_STOP=1", + "-U", "agentenv", "-d", "agentenv", *(shlex.quote(f) if " " in f else f for f in flags)]) + + async def run(self, script: str): + script = script.replace(service_db_sql.WORK_DIR, self.work_dir) + proc = await asyncio.create_subprocess_exec("bash", "-c", script, stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE) + out, err = await proc.communicate() + return proc.returncode, out.decode(), err.decode() + + async def write(self, path: str, text: str) -> None: + path = path.replace(service_db_sql.WORK_DIR, self.work_dir) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w") as f: + f.write(text) + + +@pytest.fixture(scope="module") +def pg(): + if not _docker_ok(): + pytest.skip(missing_capability_reason("docker_daemon")) + name = f"agentenv-sql-test-{uuid.uuid4().hex[:8]}" + subprocess.run(["docker", "run", "-d", "--rm", "--name", name, "-e", "POSTGRES_USER=agentenv", + "-e", "POSTGRES_PASSWORD=agentenv", "-e", "POSTGRES_DB=agentenv", _IMAGE], + check=True, capture_output=True) + try: + for _ in range(60): + if subprocess.run(["docker", "exec", name, "pg_isready", "-U", "agentenv", "-d", "agentenv"], + capture_output=True).returncode == 0: + # pg_isready answers during the image's init-time restart; a query proves the final server. + if subprocess.run(["docker", "exec", name, "psql", "-U", "agentenv", "-d", "agentenv", "-c", + "SELECT 1"], capture_output=True).returncode == 0: + break + subprocess.run(["sleep", "1"]) + else: + pytest.fail("postgres did not start") + yield name + finally: + subprocess.run(["docker", "rm", "-f", name], capture_output=True) + + +def _sql(container: str, sql: str) -> str: + res = subprocess.run(["docker", "exec", "-i", container, "psql", "-X", "-q", "-A", "-t", "-v", "ON_ERROR_STOP=1", + "-U", "agentenv", "-d", "agentenv", "-f", "-"], input=sql, capture_output=True, text=True) + assert res.returncode == 0, res.stderr + return res.stdout.strip() + + +@pytest.fixture +def base(pg, tmp_path, monkeypatch): + """A fresh service database holding the base universe, with an empty changelog, as a loaded universe leaves it.""" + _sql(pg, "DROP SCHEMA IF EXISTS slack CASCADE; DROP TABLE IF EXISTS public._changelog CASCADE;") + _sql(pg, DatabaseStateProvider.get_init_script(["slack"])) + _sql(pg, _BASE) + _sql(pg, "DELETE FROM public._changelog;") + host = _LocalHost(pg, str(tmp_path)) + + async def fake(context, env_id, what): + return host + + monkeypatch.setattr(apply_sql_module, "service_db_host", fake) + monkeypatch.setattr(env_rows_module, "service_db_host", fake) + return pg + + +_INJECTION = """\ +-- a CB's injection, as pasted: explicit ids, a quote, a tab and a newline in a value, a ':' that psql must not read +INSERT INTO slack.channels (id, name) VALUES (7, 'zombie-bugs'); +INSERT INTO slack.messages (id, channel_id, body) VALUES + (10, 7, 'Login 500s again: see ENG-1'), + (11, 7, E'it''s back\tafter the\nhotfix :ticket'); +UPDATE slack.messages SET body = 'login is broken (prod)' WHERE id = 2 +""" + + +@pytest.mark.asyncio +async def test_apply_sql_applies_clears_its_changelog_and_syncs_sequences(base): + ctx = TaskStepContext() + await ApplySqlTaskStep(id="inject", version=1, env_id="test-env", sql=_INJECTION).execute(ctx) + + assert _sql(base, "SELECT count(*) FROM slack.messages") == "4" + assert _sql(base, "SELECT body FROM slack.messages WHERE id = 11") == "it's back\tafter the\nhotfix :ticket" + # Nothing the injection wrote is left in the changelog for the run to attribute to an agent ... + assert _sql(base, "SELECT count(*) FROM public._changelog") == "0" + # ... but the step recorded what it wrote, per table and operation. + assert {(c["table_name"], c["operation"], c["rows"]) for c in ctx.metadata["applied_sql"]["inject"]["changes"]} \ + == {("channels", "INSERT", 1), ("messages", "INSERT", 2), ("messages", "UPDATE", 1)} + # A service's next insert gets a fresh id instead of colliding with the injection's explicit ones. + assert _sql(base, "INSERT INTO slack.messages (channel_id, body) VALUES (1, 'agent') RETURNING id") == "12" + assert _sql(base, "INSERT INTO slack.channels (name) VALUES ('new') RETURNING id") == "8" + # And that write, the agent's, is the changelog's only entry. + assert _sql(base, "SELECT count(*) FROM public._changelog WHERE operation = 'INSERT'") == "2" + + +@pytest.mark.asyncio +async def test_failing_sql_rolls_back_everything_and_names_the_error(base): + ctx = TaskStepContext() + sql = "INSERT INTO slack.channels (id, name) VALUES (7, 'zombie-bugs');\nINSERT INTO slack.nope VALUES (1);\n" + with pytest.raises(ServiceDbSqlError) as e: + await ApplySqlTaskStep(id="inject", version=1, env_id="test-env", sql=sql).execute(ctx) + assert 'relation "slack.nope" does not exist' in e.value.message + assert "line 2 of the SQL:" in e.value.message # the failing statement's line, counted in the SQL as pasted + assert _sql(base, "SELECT count(*) FROM slack.channels") == "2" # the first insert was rolled back too + assert ctx.metadata["applied_sql"]["inject"]["applied"] is False + + +async def _capture(name, **kw): + ctx = kw.pop("ctx") + await CaptureEnvRowsTaskStep(id=f"capture-{name}", version=1, env_id="test-env", capture=name, **kw).execute(ctx) + + +@pytest.mark.asyncio +async def test_diff_flags_extra_missing_and_changed_rows(base): + ctx = TaskStepContext() + await ApplySqlTaskStep(id="inject", version=1, env_id="test-env", sql=_INJECTION).execute(ctx) + await _capture("expected", ctx=ctx) + assert ctx.metadata["env_row_captures"]["expected"]["tables"] == { + "slack.channels": 3, "slack.messages": 4, "slack.reactions": 1} + + # The universe a run started from: a stale sandbox with an extra message the SQL never wrote, one of the SQL's + # messages missing, and an edited body. + _sql(base, """ + INSERT INTO slack.messages (id, channel_id, body) VALUES (99, 7, 'zombie: stale from an earlier injection'); + DELETE FROM slack.messages WHERE id = 10; + UPDATE slack.messages SET body = 'login is broken' WHERE id = 2; + INSERT INTO slack.reactions VALUES (2, 'eyes'); + """) + await _capture("actual", ctx=ctx) + await DiffEnvRowsTaskStep(id="diff", version=1, env_id="test-env", expected="expected", + actual="actual").execute(ctx) + v = ctx.metadata["verifications"]["diff"] + assert v["match"] is False + assert v["counts"] == {"extra": 2, "missing": 1, "changed": 1} + assert v["by_table"] == {"slack.messages": {"extra": 1, "missing": 1, "changed": 1}, + "slack.reactions": {"extra": 1, "missing": 0, "changed": 0}} + extra = {(e["table"], e["key"]): e["row"] for e in v["examples"]["extra"]} + assert extra[("slack.messages", "[99]")]["body"] == "zombie: stale from an earlier injection" + assert v["examples"]["missing"][0]["row"]["body"] == "Login 500s again: see ENG-1" + changed = v["examples"]["changed"][0] + assert (changed["expected"]["body"], changed["actual"]["body"]) == ("login is broken (prod)", "login is broken") + + with pytest.raises(RuntimeError, match="differ from 'expected'"): + await DiffEnvRowsTaskStep(id="gate", version=1, env_id="test-env", expected="expected", actual="actual", + fail_on_mismatch=True).execute(ctx) + + +@pytest.mark.asyncio +async def test_rebuild_from_base_plus_sql_matches_row_for_row(base): + """Two applies of the same SQL to the same base give identical rows: the clean-match case.""" + ctx = TaskStepContext() + await ApplySqlTaskStep(id="inject", version=1, env_id="test-env", sql=_INJECTION).execute(ctx) + await _capture("expected", ctx=ctx) + + # Rebuild: the base again, the same SQL again. + _sql(base, "DROP SCHEMA slack CASCADE;") + _sql(base, "CREATE SCHEMA slack;") + _sql(base, _BASE) + _sql(base, "DELETE FROM public._changelog;") + await ApplySqlTaskStep(id="inject-again", version=1, env_id="test-env", sql=_INJECTION).execute(ctx) + await _capture("actual", ctx=ctx) + await DiffEnvRowsTaskStep(id="diff", version=1, env_id="test-env", expected="expected", actual="actual", + fail_on_mismatch=True).execute(ctx) + assert ctx.metadata["verifications"]["diff"]["match"] is True + assert ctx.metadata["verifications"]["diff"]["counts"] == {"extra": 0, "missing": 0, "changed": 0} + + +@pytest.mark.asyncio +async def test_ignore_columns_hide_volatile_differences(base): + ctx = TaskStepContext() + await _capture("expected", ctx=ctx, ignore_columns=["posted_at"]) + _sql(base, "UPDATE slack.messages SET posted_at = '2030-01-01T00:00:00Z';") + await _capture("actual", ctx=ctx, ignore_columns=["posted_at"]) + await _capture("actual-strict", ctx=ctx) + await DiffEnvRowsTaskStep(id="loose", version=1, env_id="test-env", expected="expected", + actual="actual").execute(ctx) + assert ctx.metadata["verifications"]["loose"]["match"] is True + await DiffEnvRowsTaskStep(id="strict", version=1, env_id="test-env", expected="expected", + actual="actual-strict").execute(ctx) + # Without the ignore list the capture's rows carry posted_at, so they no longer hash like the expected ones. + assert ctx.metadata["verifications"]["strict"]["counts"]["changed"] == 2 diff --git a/tst/unit/task_step/test_apply_sql.py b/tst/unit/task_step/test_apply_sql.py new file mode 100644 index 000000000..633d923a5 --- /dev/null +++ b/tst/unit/task_step/test_apply_sql.py @@ -0,0 +1,181 @@ +"""apply_sql, capture_env_rows and diff_env_rows: registration, (de)serialization, the scripts they build, and how +they parse what psql prints. The SQL itself runs against a real PostgreSQL in +tst/integration/task_step/test_service_db_sql_postgres.py.""" +import json + +import pytest + +from agent_env.task_step.context import TaskStepContext +from agent_env.task_step.registry import get_task_step_registry +from agent_env.task_step.task_steps import apply_sql as apply_sql_module +from agent_env.task_step.task_steps.apply_sql import ( + SUMMARY_MARKER, + ApplySqlTaskStep, + build_script, + parse_summary, + sql_relative_error, + volatile_functions, +) +from agent_env.task_step.task_steps.env_rows import ( + CaptureEnvRowsTaskStep, + DiffEnvRowsTaskStep, + diff_indexes, + parse_capture_index, + pg_text_array, +) +from agent_env.task_step.task_steps.service_db_sql import ServiceDbSqlError, _psql_error + + +def test_steps_registered_and_roundtrip(): + reg = get_task_step_registry() + assert reg.get("apply_sql") is ApplySqlTaskStep + assert reg.get("capture_env_rows") is CaptureEnvRowsTaskStep + assert reg.get("diff_env_rows") is DiffEnvRowsTaskStep + + s = ApplySqlTaskStep.from_dict(ApplySqlTaskStep(id="a", version=1, env_id="e", sql="SELECT 1;").to_dict()) + assert (s.type, s.env_id, s.sql, s.fail_task_on_error) == ("apply_sql", "e", "SELECT 1;", True) + + c = CaptureEnvRowsTaskStep.from_dict( + CaptureEnvRowsTaskStep(id="c", version=1, env_id="e", capture="expected", ignore_columns=["ts"]).to_dict()) + assert (c.capture, c.ignore_columns) == ("expected", ["ts"]) + + d = DiffEnvRowsTaskStep.from_dict(DiffEnvRowsTaskStep( + id="d", version=1, env_id="e", expected="expected", actual="actual", fail_on_mismatch=True).to_dict()) + assert (d.expected, d.actual, d.fail_on_mismatch, d.max_examples) == ("expected", "actual", True, 50) + + +def test_apply_sql_null_sql_is_blank(): + assert ApplySqlTaskStep.from_dict({"id": "a", "env_id": "e", "sql": None}).sql == "" + assert ApplySqlTaskStep.from_dict({"id": "a", "env_id": "e"}).sql == "" + + +@pytest.mark.parametrize("bad", [ + lambda: ApplySqlTaskStep(id="a", version=1, env_id=""), + lambda: ApplySqlTaskStep.from_dict({"id": "a"}), + lambda: ApplySqlTaskStep(id="a", version=1, env_id="e", sql=["SELECT 1"]), + lambda: CaptureEnvRowsTaskStep(id="c", version=1, env_id="e", capture="../x"), + lambda: CaptureEnvRowsTaskStep(id="c", version=1, env_id="e", capture=""), + lambda: DiffEnvRowsTaskStep(id="d", version=1, env_id="e", expected="a", actual="a"), + lambda: DiffEnvRowsTaskStep(id="d", version=1, env_id="e", expected="a b", actual="c"), +]) +def test_config_errors(bad): + with pytest.raises(ValueError): + bad() + + +def test_volatile_functions(): + sql = ("INSERT INTO s.t VALUES (1, NOW(), random()); INSERT INTO s.t VALUES (2, now(), current_timestamp);" + " SELECT gen_random_uuid ();") + assert volatile_functions(sql) == ["now", "random", "current_timestamp", "gen_random_uuid"] + assert volatile_functions("INSERT INTO s.t (created_at) VALUES ('2026-02-14T09:00:00Z');") == [] + # A column named like a function is not a call. + assert volatile_functions("UPDATE s.t SET random_seed = 3;") == [] + + +def test_build_script_wraps_sql_between_mark_and_cleanup(): + script = build_script("INSERT INTO s.t VALUES (1)") # no trailing semicolon + pre, _, rest = script.partition("INSERT INTO s.t VALUES (1)\n;\n") + assert "CREATE TEMP TABLE _agentenv_apply_sql_mark" in pre + assert rest.index(SUMMARY_MARKER) < rest.index("DELETE FROM public._changelog") < rest.index("setval") + + +def test_sql_relative_error_counts_lines_of_the_pasted_sql(): + assert sql_relative_error('psql::3: ERROR: x') == "line 2 of the SQL: ERROR: x" + # A failure in the preamble keeps psql's own position. + assert sql_relative_error("psql::1: ERROR: y") == "psql::1: ERROR: y" + + +def test_parse_summary(): + changes = [{"schema_name": "slack", "table_name": "messages", "operation": "INSERT", "rows": 2}] + out = f"some user select output\n{SUMMARY_MARKER}\n{json.dumps(changes)}\n" + assert parse_summary(out) == changes + assert parse_summary("no marker") == [] + assert parse_summary(f"{SUMMARY_MARKER}\nnot json\n") == [] + + +def test_psql_error_keeps_error_lines_and_names_a_missing_service(): + err = 'psql::4: ERROR: relation "slack.nope" does not exist\nLINE 1: INSERT INTO slack.nope\n' + assert _psql_error(err, "") == err.strip() + assert "no running 'servicedb'" in _psql_error("service \"servicedb\" is not running", "") + + +def test_pg_text_array_quotes(): + assert pg_text_array([]) == "{}" + assert pg_text_array(["a", 'b"c', "d\\e"]) == '{"a","b\\"c","d\\\\e"}' + + +def test_diff_indexes(): + expected = parse_capture_index( + "s.t\t[1]\taaa\n" + "s.t\t[2]\tbbb\n" + "s.t\t[3]\tccc\n" + "s.nopk\trow:x\tx\n" + ) + actual = parse_capture_index( + "s.t\t[1]\taaa\n" # same + "s.t\t[2]\tBBB\n" # changed + "s.t\t[4]\tddd\n" # extra; [3] missing + "s.nopk\trow:x\tx\n" + "s.nopk\trow:x\tx\n" # a duplicate row in a table without a primary key + "s.new\t[1]\teee\n" # a whole table only in actual + ) + assert diff_indexes(expected, actual) == { + "extra": [("s.new", "[1]"), ("s.nopk", "row:x"), ("s.t", "[4]")], + "missing": [("s.t", "[3]")], + "changed": [("s.t", "[2]")], + } + assert diff_indexes(expected, expected) == {"extra": [], "missing": [], "changed": []} + + +class _FakeHost: + def __init__(self, fail: bool = False): + self.scripts, self.fail = [], fail + + async def run_sql_file(self, what, sql, *flags): + self.scripts.append((sql, flags)) + if self.fail: + raise ServiceDbSqlError(what, 3, 'psql::2: ERROR: syntax error at or near "INSRT"') + return f"{SUMMARY_MARKER}\n" + json.dumps( + [{"schema_name": "s", "table_name": "t", "operation": "INSERT", "rows": 1}]) + + +@pytest.fixture +def host(monkeypatch): + holder = {} + + async def fake(context, env_id, what): + return holder["host"] + + monkeypatch.setattr(apply_sql_module, "service_db_host", fake) + return holder + + +@pytest.mark.asyncio +async def test_apply_sql_execute_records_changes(host): + host["host"] = h = _FakeHost() + ctx = TaskStepContext() + await ApplySqlTaskStep(id="inject", version=1, env_id="e", sql="INSERT INTO s.t VALUES (1, now());").execute(ctx) + rec = ctx.metadata["applied_sql"]["inject"] + assert rec["applied"] is True and rec["env_id"] == "e" and rec["volatile_functions"] == ["now"] + assert rec["changes"] == [{"schema_name": "s", "table_name": "t", "operation": "INSERT", "rows": 1}] + assert len(rec["sql_sha256"]) == 64 + (script, flags), = h.scripts + assert "-1" in flags and "INSERT INTO s.t VALUES (1, now());" in script + + +@pytest.mark.asyncio +async def test_apply_sql_blank_is_a_noop_without_touching_the_env(host): + host["host"] = h = _FakeHost() + ctx = TaskStepContext() + await ApplySqlTaskStep(id="inject", version=1, env_id="e", sql=" \n").execute(ctx) + assert h.scripts == [] + assert ctx.metadata["applied_sql"]["inject"]["applied"] is False + + +@pytest.mark.asyncio +async def test_apply_sql_failure_raises_psql_error(host): + host["host"] = _FakeHost(fail=True) + ctx = TaskStepContext() + with pytest.raises(ServiceDbSqlError, match="syntax error at or near"): + await ApplySqlTaskStep(id="inject", version=1, env_id="e", sql="INSRT INTO s.t VALUES (1);").execute(ctx) + assert ctx.metadata["applied_sql"]["inject"]["applied"] is False From fce6e6ac7903647395086db952c2be2bfdb7e202 Mon Sep 17 00:00:00 2001 From: t Date: Fri, 6 Nov 2026 21:53:51 -0500 Subject: [PATCH 2/4] apply_sql: expose apply_sql_on_host for restores outside a task Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/apply_sql.py | 61 +++++++++++-------- .../task_step/task_steps/service_db_sql.py | 12 +++- 2 files changed, 46 insertions(+), 27 deletions(-) diff --git a/src/agent_env/task_step/task_steps/apply_sql.py b/src/agent_env/task_step/task_steps/apply_sql.py index 38efaf2f0..9ce37ff5a 100644 --- a/src/agent_env/task_step/task_steps/apply_sql.py +++ b/src/agent_env/task_step/task_steps/apply_sql.py @@ -28,7 +28,7 @@ from agent_env.entity.refs import EntityRef from agent_env.task_step.context import TaskStepContext from agent_env.task_step.task_step import TaskStep -from agent_env.task_step.task_steps.service_db_sql import ServiceDbSqlError, service_db_host +from agent_env.task_step.task_steps.service_db_sql import ServiceDbHost, ServiceDbSqlError, service_db_host logger = logging.getLogger(__name__) @@ -158,31 +158,44 @@ def from_dict(cls, data: dict) -> "ApplySqlTaskStep": return cls(**cls._base_from_dict(data), env_id=data["env_id"], sql="" if sql is None else sql) async def execute(self, context: TaskStepContext) -> TaskStepContext: - sql = self.sql or "" - record: dict = { - "env_id": self.env_id, - "sql_sha256": hashlib.sha256(sql.encode("utf-8")).hexdigest(), - "sql_bytes": len(sql.encode("utf-8")), - "volatile_functions": volatile_functions(sql), - "changes": [], - "applied": False, - } + record = sql_record(self.sql, self.env_id) context.metadata.setdefault("applied_sql", {})[self.id] = record - if not sql.strip(): + if not (self.sql or "").strip(): logger.info("apply_sql '%s': no SQL; env '%s' keeps the state it was loaded with", self.id, self.env_id) return context - if record["volatile_functions"]: - logger.warning("apply_sql '%s': the SQL calls %s, which give a different result on every apply", - self.id, ", ".join(record["volatile_functions"])) host = await service_db_host(context, self.env_id, "apply_sql") - what = f"apply_sql '{self.id}' on env '{self.env_id}'" - try: - out = await host.run_sql_file(what, build_script(sql), "-1", "-A", "-t") - except ServiceDbSqlError as e: - raise ServiceDbSqlError(what, e.exit_code, sql_relative_error(e.message)) from None - record["changes"] = parse_summary(out) - record["applied"] = True - logger.info("apply_sql '%s': applied %d bytes to env '%s': %s", self.id, record["sql_bytes"], self.env_id, - ", ".join(f"{c['operation']} {c['rows']} {c['schema_name']}.{c['table_name']}" - for c in record["changes"]) or "no rows written") + await apply_sql_on_host(host, self.sql, record, what=f"apply_sql '{self.id}' on env '{self.env_id}'") return context + + +def sql_record(sql: str, env_id: str) -> dict: + """What ``apply_sql`` records for ``sql`` before applying it; ``apply_sql_on_host`` fills in the rest.""" + sql = sql or "" + return { + "env_id": env_id, + "sql_sha256": hashlib.sha256(sql.encode("utf-8")).hexdigest(), + "sql_bytes": len(sql.encode("utf-8")), + "volatile_functions": volatile_functions(sql), + "changes": [], + "applied": False, + } + + +async def apply_sql_on_host(host: ServiceDbHost, sql: str, record: dict, *, what: str) -> dict: + """Apply ``sql`` to the service database on ``host`` as ``apply_sql`` does, filling in ``record`` (from + ``sql_record``): the rows written per table, and ``applied``. For callers that restore an env outside a task, + such as a trajectory editor rebuilding a run's start state. Raises ``ServiceDbSqlError`` with the failing line + counted in ``sql`` as given.""" + if record["volatile_functions"]: + logger.warning("%s: the SQL calls %s, which give a different result on every apply", what, + ", ".join(record["volatile_functions"])) + try: + out = await host.run_sql_file(what, build_script(sql), "-1", "-A", "-t") + except ServiceDbSqlError as e: + raise ServiceDbSqlError(what, e.exit_code, sql_relative_error(e.message)) from None + record["changes"] = parse_summary(out) + record["applied"] = True + logger.info("%s: applied %d bytes: %s", what, record["sql_bytes"], + ", ".join(f"{c['operation']} {c['rows']} {c['schema_name']}.{c['table_name']}" + for c in record["changes"]) or "no rows written") + return record diff --git a/src/agent_env/task_step/task_steps/service_db_sql.py b/src/agent_env/task_step/task_steps/service_db_sql.py index 104d5db8d..5f278ba7a 100644 --- a/src/agent_env/task_step/task_steps/service_db_sql.py +++ b/src/agent_env/task_step/task_steps/service_db_sql.py @@ -80,12 +80,18 @@ def _psql_error(stderr: str, stdout: str) -> str: async def service_db_host(context: TaskStepContext, env_id: str, what: str) -> ServiceDbHost: """The VM host of ``env_id``'s deploy in this run.""" - from agent_env.providers.sandbox_providers.sandbox import VmSandbox - from agent_env.providers.sandbox_providers.sandbox_provider import build_sandbox_provider, get_env_sandbox_provider - deployed = next((d for d in context.deployed_envs if d.env_id == env_id), None) if deployed is None: raise RuntimeError(f"{what}: env '{env_id}' is not deployed in this run; put a deploy_env step for it first") + return await service_db_host_of(deployed, what) + + +async def service_db_host_of(deployed, what: str) -> ServiceDbHost: + """The VM host of a deployed env's record, wherever it was deployed from (a task run or a direct deploy).""" + from agent_env.providers.sandbox_providers.sandbox import VmSandbox + from agent_env.providers.sandbox_providers.sandbox_provider import build_sandbox_provider, get_env_sandbox_provider + + env_id = deployed.env_id deployed: DeployedSandboxEnv = require_sandbox(deployed, what) if not deployed.sandbox_id: raise RuntimeError(f"{what}: env '{env_id}' has no sandbox_id") From 908c83fdac83dbc2dace1a388d0153991f2e18b8 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 9 Nov 2026 14:53:01 -0500 Subject: [PATCH 3/4] apply_sql: refuse transaction control; drop capture_env_rows and diff_env_rows SQL that would end the step's single transaction early (BEGIN, COMMIT, ROLLBACK, SAVEPOINT, ...) or holds a psql meta-command is now refused before anything runs, so a failing statement can no longer leave earlier writes committed. Literals, comments and dollar-quoted bodies are ignored when looking for them. capture_env_rows and diff_env_rows are removed from this change: nothing needs them to build a run's state, and they can come back separately. apply_sql is now classified in the episode rules. Co-Authored-By: Claude Opus 5.5 --- src/agent_env/task/episode/rules.py | 2 +- src/agent_env/task_step/registry.py | 3 - .../task_step/task_steps/apply_sql.py | 41 ++- .../task_step/task_steps/env_rows.py | 284 ------------------ .../task_step/task_steps/service_db_sql.py | 2 +- .../task_step/test_service_db_sql_postgres.py | 84 +----- tst/unit/task_step/test_apply_sql.py | 82 +++-- 7 files changed, 79 insertions(+), 419 deletions(-) delete mode 100644 src/agent_env/task_step/task_steps/env_rows.py diff --git a/src/agent_env/task/episode/rules.py b/src/agent_env/task/episode/rules.py index a2e9efc50..1abec063a 100644 --- a/src/agent_env/task/episode/rules.py +++ b/src/agent_env/task/episode/rules.py @@ -149,7 +149,7 @@ def _rubrics_verifier(step: RubricsVerifierTaskStep, outside: Outside) -> list[s UNAFFECTED = frozenset({ # They name no agent. - "aggregate_verifiers", "apply_server_config", "browser_activate_tab", "browser_clear_local_storage", + "aggregate_verifiers", "apply_server_config", "apply_sql", "browser_activate_tab", "browser_clear_local_storage", "browser_close_tab", "browser_delete_cookies", "browser_open_tab", "browser_set_cookies", "browser_set_local_storage", "build_mcp_cli", "combine_universe_verdicts", "configure_scheduled_checks", "deploy_env", "deploy_sandbox", "end_computer_recording", "start_computer_recording", "env_outcome_verifier", diff --git a/src/agent_env/task_step/registry.py b/src/agent_env/task_step/registry.py index 5802beecb..57db81710 100644 --- a/src/agent_env/task_step/registry.py +++ b/src/agent_env/task_step/registry.py @@ -50,7 +50,6 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: from agent_env.task_step.task_steps.deploy_sandbox import DeploySandboxTaskStep from agent_env.task_step.task_steps.reset_env import ResetEnvTaskStep from agent_env.task_step.task_steps.apply_sql import ApplySqlTaskStep - from agent_env.task_step.task_steps.env_rows import CaptureEnvRowsTaskStep, DiffEnvRowsTaskStep from agent_env.task_step.task_steps.teardown_sandboxes import TeardownSandboxesTaskStep from agent_env.task_step.task_steps.verifiers.env_outcome_verifier import EnvOutcomeVerifierTaskStep from agent_env.task_step.task_steps.load_artifact import LoadArtifactTaskStep @@ -117,8 +116,6 @@ def _builtin_registry() -> dict[str, type["TaskStep"]]: _get_type(DeploySandboxTaskStep): DeploySandboxTaskStep, _get_type(ResetEnvTaskStep): ResetEnvTaskStep, _get_type(ApplySqlTaskStep): ApplySqlTaskStep, - _get_type(CaptureEnvRowsTaskStep): CaptureEnvRowsTaskStep, - _get_type(DiffEnvRowsTaskStep): DiffEnvRowsTaskStep, _get_type(TeardownSandboxesTaskStep): TeardownSandboxesTaskStep, _get_type(EnvOutcomeVerifierTaskStep): EnvOutcomeVerifierTaskStep, _get_type(LoadArtifactTaskStep): LoadArtifactTaskStep, diff --git a/src/agent_env/task_step/task_steps/apply_sql.py b/src/agent_env/task_step/task_steps/apply_sql.py index 9ce37ff5a..a33be7b6b 100644 --- a/src/agent_env/task_step/task_steps/apply_sql.py +++ b/src/agent_env/task_step/task_steps/apply_sql.py @@ -10,6 +10,9 @@ happens after this step, as a freshly loaded universe does; nothing the SQL did shows up as an agent's change; - every serial / identity sequence in the services' schemas is advanced past its column's largest value, so an insert with explicit ids doesn't make the services' own later inserts collide; +- SQL that would end or leave that transaction (``BEGIN``, ``COMMIT``, ``ROLLBACK``, ``SAVEPOINT``, ... at the start + of a statement) or holds a psql meta-command (a backslash outside a literal) is refused before anything runs, since + a ``COMMIT`` would keep the writes before a later failing statement; - ``context.metadata["applied_sql"][]`` records the SQL's sha256 and size, the rows it wrote per table and operation (read from the changelog before it was cleared), and any volatile functions (``now()``, ``random()``, ...) it calls: those give a different result on every apply, so a rebuild can't reproduce their rows exactly. @@ -79,6 +82,40 @@ """ +# Comments, string literals, quoted identifiers and dollar-quoted bodies, which hold no statement of the script's own. +_LITERAL = re.compile( + r"--[^\n]*|/\*.*?\*/|'(?:[^']|'')*'|\"(?:[^\"]|\"\")*\"|\$([A-Za-z_][A-Za-z0-9_]*|)\$.*?\$\1\$", + re.DOTALL, +) +_TRANSACTION_CONTROL = re.compile( + r"(?:\A|;)\s*(begin|commit|rollback|end|abort|savepoint|release|start\s+transaction|prepare\s+transaction)\b", + re.IGNORECASE, +) + + +class RefusedSqlError(ServiceDbSqlError): + """SQL ``apply_sql`` won't run, refused before it reaches the database.""" + + def __init__(self, what: str, message: str): + RuntimeError.__init__(self, f"{what} refused: {message}") + self.exit_code = None + self.message = message + + +def refusals(sql: str) -> list[str]: + """Why ``sql`` can't run as one ``apply_sql`` transaction, each naming its line; empty when it can.""" + code = _LITERAL.sub(lambda m: re.sub(r"[^\n]", " ", m.group(0)), sql) + found = [] + for m in _TRANSACTION_CONTROL.finditer(code): + line = code.count("\n", 0, m.start(1)) + 1 + found.append(f"line {line} of the SQL: {m.group(1).upper()} controls the transaction apply_sql runs the SQL " + "in; remove it (the step commits on success and rolls everything back on an error)") + for m in re.finditer(r"\\", code): + line = code.count("\n", 0, m.start()) + 1 + found.append(f"line {line} of the SQL: psql meta-commands (a backslash outside a literal) aren't allowed") + return found + + def volatile_functions(sql: str) -> list[str]: """The volatile calls ``sql`` makes, lower-cased and de-duplicated, in order of first use.""" seen: dict[str, None] = {} @@ -185,7 +222,9 @@ async def apply_sql_on_host(host: ServiceDbHost, sql: str, record: dict, *, what """Apply ``sql`` to the service database on ``host`` as ``apply_sql`` does, filling in ``record`` (from ``sql_record``): the rows written per table, and ``applied``. For callers that restore an env outside a task, such as a trajectory editor rebuilding a run's start state. Raises ``ServiceDbSqlError`` with the failing line - counted in ``sql`` as given.""" + counted in ``sql`` as given, or ``RefusedSqlError`` before anything runs.""" + if problems := refusals(sql): + raise RefusedSqlError(what, "; ".join(problems)) if record["volatile_functions"]: logger.warning("%s: the SQL calls %s, which give a different result on every apply", what, ", ".join(record["volatile_functions"])) diff --git a/src/agent_env/task_step/task_steps/env_rows.py b/src/agent_env/task_step/task_steps/env_rows.py deleted file mode 100644 index cfd8e0022..000000000 --- a/src/agent_env/task_step/task_steps/env_rows.py +++ /dev/null @@ -1,284 +0,0 @@ -"""Row-level capture and diff of a deployed env's service database. - -``capture_env_rows`` writes every row of every service table to a file on the env's VM host, one line per row: -``.
TAB TAB TAB ``. The key is the row's primary key as a JSON -array, or ``row:`` for a table without one. ``ignore_columns`` names columns left out of every row (and its md5), -e.g. timestamps a rebuild can't reproduce. - -``diff_env_rows`` compares two captures of the same env (say: a base universe plus declared SQL, and the universe a run -actually started from), row by row, never by SQL text, and records ``context.metadata["verifications"][]``: - - {"type": "env_rows_diff", "match": bool, "expected": , "actual": , - "counts": {"extra": n, "missing": n, "changed": n}, - "by_table": {".
": {"extra": n, "missing": n, "changed": n}}, - "examples": {"extra": [{"table", "key", "row"}], "missing": [...], "changed": [{"table", "key", "expected", - "actual"}]}} - -``extra`` rows are in ``actual`` only, ``missing`` rows in ``expected`` only, ``changed`` rows share a key but differ. -With ``fail_on_mismatch`` a mismatch also fails the step. - -Typical task: ``deploy_env -> load_artifact(base) -> apply_sql -> capture_env_rows(expected) -> -load_artifact(run universe) -> capture_env_rows(actual) -> diff_env_rows``. -""" -from __future__ import annotations - -import json -import logging -import re -import shlex -from collections import defaultdict -from typing import Optional - -from agent_env.entity.refs import EntityRef -from agent_env.task_step.context import TaskStepContext -from agent_env.task_step.task_step import TaskStep -from agent_env.task_step.task_steps.service_db_sql import WORK_DIR, ServiceDbHost, service_db_host - -logger = logging.getLogger(__name__) - -_NAME = re.compile(r"^[A-Za-z0-9_-]{1,64}$") - -# Emits one COPY per service table and runs each (\gexec); every COPY writes that table's rows as capture lines. -_CAPTURE_SQL = """\ -SELECT format( - 'COPY (SELECT %L, %s, md5((to_jsonb(t) - %L::text[])::text), (to_jsonb(t) - %L::text[])::text FROM %I.%I t) TO STDOUT', - n.nspname || '.' || c.relname, - coalesce( - (SELECT 'jsonb_build_array(' || string_agg(format('t.%I', a.attname), ', ' ORDER BY k.ord) || ')::text' - FROM pg_index i - CROSS JOIN LATERAL unnest(i.indkey::int2[]) WITH ORDINALITY AS k(attnum, ord) - JOIN pg_attribute a ON a.attrelid = i.indrelid AND a.attnum = k.attnum - WHERE i.indrelid = c.oid AND i.indisprimary), - format('''row:'' || md5((to_jsonb(t) - %L::text[])::text)', :'ignore')), - :'ignore', :'ignore', n.nspname, c.relname) -FROM pg_class c -JOIN pg_namespace n ON n.oid = c.relnamespace -WHERE c.relkind IN ('r', 'p') - AND NOT c.relispartition - AND n.nspname NOT IN ('pg_catalog', 'information_schema', 'agent_env_internal') - AND n.nspname NOT LIKE 'pg\\_%' - AND NOT (n.nspname = 'public' AND c.relname = '_changelog') -ORDER BY n.nspname, c.relname -\\gexec -""" - - -def _check_name(value: str, field: str, step: str) -> str: - if not isinstance(value, str) or not _NAME.match(value): - raise ValueError(f"{step}: `{field}` must be 1-64 letters, digits, '_' or '-'; got {value!r}") - return value - - -def capture_path(name: str) -> str: - return f"{WORK_DIR}/captures/{name}.tsv" - - -def pg_text_array(values: list[str]) -> str: - """``values`` as a PostgreSQL text-array literal, e.g. ``{"a","b"}``.""" - return "{" + ",".join('"' + v.replace("\\", "\\\\").replace('"', '\\"') + '"' for v in values) + "}" - - -def parse_capture_index(text: str) -> dict[str, dict[str, list[str]]]: - """``
TAB TAB `` lines as {table: {key: [md5, ...]}} (a key repeats in a table without a primary - key).""" - index: dict[str, dict[str, list[str]]] = defaultdict(lambda: defaultdict(list)) - for line in text.splitlines(): - if not line: - continue - parts = line.split("\t") - if len(parts) < 3: - continue - index[parts[0]][parts[1]].append(parts[2]) - return index - - -def diff_indexes(expected: dict, actual: dict) -> dict[str, dict[str, list[str]]]: - """{"extra" | "missing" | "changed": [(table, key), ...]} between two capture indexes, each list sorted.""" - out: dict[str, list] = {"extra": [], "missing": [], "changed": []} - for table in sorted(set(expected) | set(actual)): - exp, act = expected.get(table, {}), actual.get(table, {}) - for key in sorted(set(exp) | set(act)): - e, a = sorted(exp.get(key, [])), sorted(act.get(key, [])) - if e == a: - continue - if not e: - out["extra"].extend([(table, key)] * len(a)) - elif not a: - out["missing"].extend([(table, key)] * len(e)) - elif len(e) == len(a): - out["changed"].append((table, key)) - else: # same key, a different number of copies: a table without a primary key - kind = "extra" if len(a) > len(e) else "missing" - out[kind].extend([(table, key)] * abs(len(a) - len(e))) - return out - - -def _unescape_copy(value: str) -> str: - """Undo COPY text-format escaping (``\\t``, ``\\n``, ``\\\\``, ...) in one field.""" - return re.sub(r"\\(.)", lambda m: {"t": "\t", "n": "\n", "r": "\r", "b": "\b", "f": "\f", "v": "\v"}.get( - m.group(1), m.group(1)), value) - - -def _row(text: Optional[str]): - if text is None: - return None - raw = _unescape_copy(text) - try: - return json.loads(raw) - except ValueError: - return raw - - -class CaptureEnvRowsTaskStep(TaskStep): - """Capture every row of a deployed env's service tables to a named file on its VM host.""" - - type = "capture_env_rows" - entity_refs = (EntityRef.env("env_id"),) - - def __init__(self, id: str, version: Optional[int], env_id: str, capture: str, - ignore_columns: Optional[list[str]] = None, depends_on: Optional[list] = None, - fail_task_on_error: bool = True): - if not env_id: - raise ValueError("capture_env_rows requires an env_id") - super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) - self.env_id = env_id - self.capture = _check_name(capture, "capture", "capture_env_rows") - self.ignore_columns = [str(c) for c in (ignore_columns or [])] - - def to_dict(self) -> dict: - base = super().to_dict() - base.update(env_id=self.env_id, capture=self.capture, ignore_columns=self.ignore_columns) - return base - - @classmethod - def from_dict(cls, data: dict) -> "CaptureEnvRowsTaskStep": - if not data.get("env_id"): - raise ValueError("capture_env_rows step requires 'env_id'") - return cls(**cls._base_from_dict(data), env_id=data["env_id"], capture=data.get("capture", ""), - ignore_columns=data.get("ignore_columns")) - - async def execute(self, context: TaskStepContext) -> TaskStepContext: - host = await service_db_host(context, self.env_id, "capture_env_rows") - path = capture_path(self.capture) - sql_path = f"{WORK_DIR}/capture-{self.capture}.sql" - await host.write(sql_path, _CAPTURE_SQL) - ignore = pg_text_array(self.ignore_columns) - cmd = (f"mkdir -p {shlex.quote(WORK_DIR)}/captures && " - f"{host.psql('-A', '-t', '-v', shlex.quote('ignore=' + ignore), '-f', '-')} " - f"< {shlex.quote(sql_path)} > {shlex.quote(path)}") - code, _, err = await host.run(cmd) - await host.run(f"rm -f {shlex.quote(sql_path)}") - if code != 0: - from agent_env.task_step.task_steps.service_db_sql import ServiceDbSqlError, _psql_error - - raise ServiceDbSqlError(f"capture_env_rows '{self.id}' on env '{self.env_id}'", code, _psql_error(err, "")) - code, counts, err = await host.run(f"cut -f1 {shlex.quote(path)} | sort | uniq -c") - tables = {} - for line in counts.splitlines(): - n, _, table = line.strip().partition(" ") - if table: - tables[table] = int(n) - context.metadata.setdefault("env_row_captures", {})[self.capture] = { - "env_id": self.env_id, "path": path, "rows": sum(tables.values()), "tables": tables, - "ignore_columns": self.ignore_columns, - } - logger.info("capture_env_rows '%s': %d rows in %d tables of env '%s' -> %s", self.id, sum(tables.values()), - len(tables), self.env_id, path) - return context - - -class DiffEnvRowsTaskStep(TaskStep): - """Diff two row captures of an env; record extra, missing and changed rows as a verification.""" - - type = "diff_env_rows" - entity_refs = (EntityRef.env("env_id"),) - - def __init__(self, id: str, version: Optional[int], env_id: str, expected: str, actual: str, - max_examples: int = 50, fail_on_mismatch: bool = False, depends_on: Optional[list] = None, - fail_task_on_error: bool = True): - if not env_id: - raise ValueError("diff_env_rows requires an env_id") - super().__init__(id, version, depends_on=depends_on, fail_task_on_error=fail_task_on_error) - self.env_id = env_id - self.expected = _check_name(expected, "expected", "diff_env_rows") - self.actual = _check_name(actual, "actual", "diff_env_rows") - if self.expected == self.actual: - raise ValueError("diff_env_rows: `expected` and `actual` must name different captures") - self.max_examples = max(0, int(max_examples)) - self.fail_on_mismatch = bool(fail_on_mismatch) - - def to_dict(self) -> dict: - base = super().to_dict() - base.update(env_id=self.env_id, expected=self.expected, actual=self.actual, max_examples=self.max_examples, - fail_on_mismatch=self.fail_on_mismatch) - return base - - @classmethod - def from_dict(cls, data: dict) -> "DiffEnvRowsTaskStep": - if not data.get("env_id"): - raise ValueError("diff_env_rows step requires 'env_id'") - return cls(**cls._base_from_dict(data), env_id=data["env_id"], expected=data.get("expected", ""), - actual=data.get("actual", ""), max_examples=data.get("max_examples", 50), - fail_on_mismatch=data.get("fail_on_mismatch", False)) - - async def _index(self, host: ServiceDbHost, name: str) -> dict: - path = capture_path(name) - code, out, err = await host.run(f"test -f {shlex.quote(path)} && cut -f1-3 {shlex.quote(path)}") - if code != 0: - raise RuntimeError(f"diff_env_rows '{self.id}': no capture '{name}' on env '{self.env_id}'; run a " - f"capture_env_rows step with capture='{name}' first ({err.strip()[-200:]})") - return parse_capture_index(out) - - async def _rows(self, host: ServiceDbHost, name: str, keys: list[tuple[str, str]]) -> dict[tuple[str, str], str]: - """The row JSON of each (table, key) in capture ``name``.""" - if not keys: - return {} - keys_path = f"{WORK_DIR}/diff-{self.id}-{name}.keys" - await host.write(keys_path, "".join(f"{t}\t{k}\n" for t, k in keys)) - script = ("awk -F'\\t' 'NR==FNR{want[$0]=1; next} (($1 FS $2) in want){print $1 FS $2 FS $4}' " - f"{shlex.quote(keys_path)} {shlex.quote(capture_path(name))}; rm -f {shlex.quote(keys_path)}") - _, out, _ = await host.run(script) - rows: dict[tuple[str, str], str] = {} - for line in out.splitlines(): - parts = line.split("\t", 2) - if len(parts) == 3: - rows.setdefault((parts[0], parts[1]), parts[2]) - return rows - - async def execute(self, context: TaskStepContext) -> TaskStepContext: - host = await service_db_host(context, self.env_id, "diff_env_rows") - expected, actual = await self._index(host, self.expected), await self._index(host, self.actual) - diff = diff_indexes(expected, actual) - by_table: dict[str, dict[str, int]] = {} - for kind, items in diff.items(): - for table, _ in items: - by_table.setdefault(table, {"extra": 0, "missing": 0, "changed": 0})[kind] += 1 - n = self.max_examples - extra, missing, changed = diff["extra"][:n], diff["missing"][:n], diff["changed"][:n] - exp_rows = await self._rows(host, self.expected, sorted(set(missing + changed))) - act_rows = await self._rows(host, self.actual, sorted(set(extra + changed))) - result = { - "type": "env_rows_diff", - "match": not any(diff.values()), - "expected": self.expected, - "actual": self.actual, - "counts": {k: len(v) for k, v in diff.items()}, - "by_table": by_table, - "examples": { - "extra": [{"table": t, "key": k, "row": _row(act_rows.get((t, k)))} for t, k in extra], - "missing": [{"table": t, "key": k, "row": _row(exp_rows.get((t, k)))} for t, k in missing], - "changed": [{"table": t, "key": k, "expected": _row(exp_rows.get((t, k))), - "actual": _row(act_rows.get((t, k)))} for t, k in changed], - }, - } - context.metadata.setdefault("verifications", {})[self.id] = result - summary = ", ".join(f"{v} {k}" for k, v in result["counts"].items()) - if result["match"]: - logger.info("diff_env_rows '%s': '%s' and '%s' match row for row", self.id, self.expected, self.actual) - else: - logger.warning("diff_env_rows '%s': '%s' vs '%s' differ: %s (tables: %s)", self.id, self.expected, - self.actual, summary, ", ".join(sorted(by_table))) - if self.fail_on_mismatch: - raise RuntimeError(f"diff_env_rows '{self.id}': the env's rows differ from '{self.expected}': {summary}" - f" in {', '.join(sorted(by_table))}") - return context diff --git a/src/agent_env/task_step/task_steps/service_db_sql.py b/src/agent_env/task_step/task_steps/service_db_sql.py index 5f278ba7a..5a3355d93 100644 --- a/src/agent_env/task_step/task_steps/service_db_sql.py +++ b/src/agent_env/task_step/task_steps/service_db_sql.py @@ -17,7 +17,7 @@ logger = logging.getLogger(__name__) -# The directory on the env's VM host that holds the SQL files and row captures these steps write. +# The directory on the env's VM host that holds the SQL files written for psql to read. WORK_DIR = "/tmp/agentenv-service-db-sql" diff --git a/tst/integration/task_step/test_service_db_sql_postgres.py b/tst/integration/task_step/test_service_db_sql_postgres.py index 667987313..61bc30cf5 100644 --- a/tst/integration/task_step/test_service_db_sql_postgres.py +++ b/tst/integration/task_step/test_service_db_sql_postgres.py @@ -1,10 +1,10 @@ -"""apply_sql, capture_env_rows and diff_env_rows against a real PostgreSQL 16 holding a service schema set up the way a +"""apply_sql against a real PostgreSQL 16 holding a service schema set up the way a gateway env's service database is: the shared changelog objects from the state provider's init script and changelog triggers on the service's tables. -The steps reach the database through ``ServiceDbHost``; here its psql runs through ``docker exec`` into a local +The step reaches the database through ``ServiceDbHost``; here its psql runs through ``docker exec`` into a local postgres container and its shell commands run on this machine, so everything but the env's VM is real: the scripts, -psql's single transaction and ``\\gexec``, the triggers, and the ``cut`` / ``awk`` the diff reads captures with. +psql's single transaction and ``\\gexec``, and the triggers. """ import asyncio import os @@ -17,10 +17,8 @@ from agent_env.providers.env_state.env_state_provider import DatabaseStateProvider from agent_env.task_step.context import TaskStepContext from agent_env.task_step.task_steps import apply_sql as apply_sql_module -from agent_env.task_step.task_steps import env_rows as env_rows_module from agent_env.task_step.task_steps import service_db_sql from agent_env.task_step.task_steps.apply_sql import ApplySqlTaskStep -from agent_env.task_step.task_steps.env_rows import CaptureEnvRowsTaskStep, DiffEnvRowsTaskStep from agent_env.task_step.task_steps.service_db_sql import ServiceDbHost, ServiceDbSqlError from tst.util.capabilities import missing_capability_reason @@ -117,7 +115,6 @@ async def fake(context, env_id, what): return host monkeypatch.setattr(apply_sql_module, "service_db_host", fake) - monkeypatch.setattr(env_rows_module, "service_db_host", fake) return pg @@ -161,78 +158,3 @@ async def test_failing_sql_rolls_back_everything_and_names_the_error(base): assert _sql(base, "SELECT count(*) FROM slack.channels") == "2" # the first insert was rolled back too assert ctx.metadata["applied_sql"]["inject"]["applied"] is False - -async def _capture(name, **kw): - ctx = kw.pop("ctx") - await CaptureEnvRowsTaskStep(id=f"capture-{name}", version=1, env_id="test-env", capture=name, **kw).execute(ctx) - - -@pytest.mark.asyncio -async def test_diff_flags_extra_missing_and_changed_rows(base): - ctx = TaskStepContext() - await ApplySqlTaskStep(id="inject", version=1, env_id="test-env", sql=_INJECTION).execute(ctx) - await _capture("expected", ctx=ctx) - assert ctx.metadata["env_row_captures"]["expected"]["tables"] == { - "slack.channels": 3, "slack.messages": 4, "slack.reactions": 1} - - # The universe a run started from: a stale sandbox with an extra message the SQL never wrote, one of the SQL's - # messages missing, and an edited body. - _sql(base, """ - INSERT INTO slack.messages (id, channel_id, body) VALUES (99, 7, 'zombie: stale from an earlier injection'); - DELETE FROM slack.messages WHERE id = 10; - UPDATE slack.messages SET body = 'login is broken' WHERE id = 2; - INSERT INTO slack.reactions VALUES (2, 'eyes'); - """) - await _capture("actual", ctx=ctx) - await DiffEnvRowsTaskStep(id="diff", version=1, env_id="test-env", expected="expected", - actual="actual").execute(ctx) - v = ctx.metadata["verifications"]["diff"] - assert v["match"] is False - assert v["counts"] == {"extra": 2, "missing": 1, "changed": 1} - assert v["by_table"] == {"slack.messages": {"extra": 1, "missing": 1, "changed": 1}, - "slack.reactions": {"extra": 1, "missing": 0, "changed": 0}} - extra = {(e["table"], e["key"]): e["row"] for e in v["examples"]["extra"]} - assert extra[("slack.messages", "[99]")]["body"] == "zombie: stale from an earlier injection" - assert v["examples"]["missing"][0]["row"]["body"] == "Login 500s again: see ENG-1" - changed = v["examples"]["changed"][0] - assert (changed["expected"]["body"], changed["actual"]["body"]) == ("login is broken (prod)", "login is broken") - - with pytest.raises(RuntimeError, match="differ from 'expected'"): - await DiffEnvRowsTaskStep(id="gate", version=1, env_id="test-env", expected="expected", actual="actual", - fail_on_mismatch=True).execute(ctx) - - -@pytest.mark.asyncio -async def test_rebuild_from_base_plus_sql_matches_row_for_row(base): - """Two applies of the same SQL to the same base give identical rows: the clean-match case.""" - ctx = TaskStepContext() - await ApplySqlTaskStep(id="inject", version=1, env_id="test-env", sql=_INJECTION).execute(ctx) - await _capture("expected", ctx=ctx) - - # Rebuild: the base again, the same SQL again. - _sql(base, "DROP SCHEMA slack CASCADE;") - _sql(base, "CREATE SCHEMA slack;") - _sql(base, _BASE) - _sql(base, "DELETE FROM public._changelog;") - await ApplySqlTaskStep(id="inject-again", version=1, env_id="test-env", sql=_INJECTION).execute(ctx) - await _capture("actual", ctx=ctx) - await DiffEnvRowsTaskStep(id="diff", version=1, env_id="test-env", expected="expected", actual="actual", - fail_on_mismatch=True).execute(ctx) - assert ctx.metadata["verifications"]["diff"]["match"] is True - assert ctx.metadata["verifications"]["diff"]["counts"] == {"extra": 0, "missing": 0, "changed": 0} - - -@pytest.mark.asyncio -async def test_ignore_columns_hide_volatile_differences(base): - ctx = TaskStepContext() - await _capture("expected", ctx=ctx, ignore_columns=["posted_at"]) - _sql(base, "UPDATE slack.messages SET posted_at = '2030-01-01T00:00:00Z';") - await _capture("actual", ctx=ctx, ignore_columns=["posted_at"]) - await _capture("actual-strict", ctx=ctx) - await DiffEnvRowsTaskStep(id="loose", version=1, env_id="test-env", expected="expected", - actual="actual").execute(ctx) - assert ctx.metadata["verifications"]["loose"]["match"] is True - await DiffEnvRowsTaskStep(id="strict", version=1, env_id="test-env", expected="expected", - actual="actual-strict").execute(ctx) - # Without the ignore list the capture's rows carry posted_at, so they no longer hash like the expected ones. - assert ctx.metadata["verifications"]["strict"]["counts"]["changed"] == 2 diff --git a/tst/unit/task_step/test_apply_sql.py b/tst/unit/task_step/test_apply_sql.py index 633d923a5..2d18ededb 100644 --- a/tst/unit/task_step/test_apply_sql.py +++ b/tst/unit/task_step/test_apply_sql.py @@ -1,5 +1,5 @@ -"""apply_sql, capture_env_rows and diff_env_rows: registration, (de)serialization, the scripts they build, and how -they parse what psql prints. The SQL itself runs against a real PostgreSQL in +"""apply_sql: registration, (de)serialization, the script it builds, the SQL it refuses, and how it parses what +psql prints. The SQL itself runs against a real PostgreSQL in tst/integration/task_step/test_service_db_sql_postgres.py.""" import json @@ -11,37 +11,23 @@ from agent_env.task_step.task_steps.apply_sql import ( SUMMARY_MARKER, ApplySqlTaskStep, + RefusedSqlError, build_script, parse_summary, + refusals, sql_relative_error, volatile_functions, ) -from agent_env.task_step.task_steps.env_rows import ( - CaptureEnvRowsTaskStep, - DiffEnvRowsTaskStep, - diff_indexes, - parse_capture_index, - pg_text_array, -) from agent_env.task_step.task_steps.service_db_sql import ServiceDbSqlError, _psql_error def test_steps_registered_and_roundtrip(): reg = get_task_step_registry() assert reg.get("apply_sql") is ApplySqlTaskStep - assert reg.get("capture_env_rows") is CaptureEnvRowsTaskStep - assert reg.get("diff_env_rows") is DiffEnvRowsTaskStep s = ApplySqlTaskStep.from_dict(ApplySqlTaskStep(id="a", version=1, env_id="e", sql="SELECT 1;").to_dict()) assert (s.type, s.env_id, s.sql, s.fail_task_on_error) == ("apply_sql", "e", "SELECT 1;", True) - c = CaptureEnvRowsTaskStep.from_dict( - CaptureEnvRowsTaskStep(id="c", version=1, env_id="e", capture="expected", ignore_columns=["ts"]).to_dict()) - assert (c.capture, c.ignore_columns) == ("expected", ["ts"]) - - d = DiffEnvRowsTaskStep.from_dict(DiffEnvRowsTaskStep( - id="d", version=1, env_id="e", expected="expected", actual="actual", fail_on_mismatch=True).to_dict()) - assert (d.expected, d.actual, d.fail_on_mismatch, d.max_examples) == ("expected", "actual", True, 50) def test_apply_sql_null_sql_is_blank(): @@ -53,10 +39,6 @@ def test_apply_sql_null_sql_is_blank(): lambda: ApplySqlTaskStep(id="a", version=1, env_id=""), lambda: ApplySqlTaskStep.from_dict({"id": "a"}), lambda: ApplySqlTaskStep(id="a", version=1, env_id="e", sql=["SELECT 1"]), - lambda: CaptureEnvRowsTaskStep(id="c", version=1, env_id="e", capture="../x"), - lambda: CaptureEnvRowsTaskStep(id="c", version=1, env_id="e", capture=""), - lambda: DiffEnvRowsTaskStep(id="d", version=1, env_id="e", expected="a", actual="a"), - lambda: DiffEnvRowsTaskStep(id="d", version=1, env_id="e", expected="a b", actual="c"), ]) def test_config_errors(bad): with pytest.raises(ValueError): @@ -99,32 +81,25 @@ def test_psql_error_keeps_error_lines_and_names_a_missing_service(): assert "no running 'servicedb'" in _psql_error("service \"servicedb\" is not running", "") -def test_pg_text_array_quotes(): - assert pg_text_array([]) == "{}" - assert pg_text_array(["a", 'b"c', "d\\e"]) == '{"a","b\\"c","d\\\\e"}' - - -def test_diff_indexes(): - expected = parse_capture_index( - "s.t\t[1]\taaa\n" - "s.t\t[2]\tbbb\n" - "s.t\t[3]\tccc\n" - "s.nopk\trow:x\tx\n" - ) - actual = parse_capture_index( - "s.t\t[1]\taaa\n" # same - "s.t\t[2]\tBBB\n" # changed - "s.t\t[4]\tddd\n" # extra; [3] missing - "s.nopk\trow:x\tx\n" - "s.nopk\trow:x\tx\n" # a duplicate row in a table without a primary key - "s.new\t[1]\teee\n" # a whole table only in actual - ) - assert diff_indexes(expected, actual) == { - "extra": [("s.new", "[1]"), ("s.nopk", "row:x"), ("s.t", "[4]")], - "missing": [("s.t", "[3]")], - "changed": [("s.t", "[2]")], - } - assert diff_indexes(expected, expected) == {"extra": [], "missing": [], "changed": []} +@pytest.mark.parametrize("sql", [ + "INSERT INTO s.t VALUES (1);\nUPDATE s.t SET a = 'COMMIT; BEGIN' WHERE id = 1;", + "DO $$ BEGIN PERFORM 1; END $$;", + "CREATE FUNCTION f() RETURNS int AS $body$ BEGIN RETURN 1; END $body$ LANGUAGE plpgsql;", + "-- COMMIT later\n/* ROLLBACK */ INSERT INTO s.t VALUES (E'it''s back\\tafter');", +]) +def test_refusals_ignore_literals_comments_and_bodies(sql): + assert refusals(sql) == [] + + +@pytest.mark.parametrize(("sql", "found"), [ + ("BEGIN;\nINSERT INTO s.t VALUES (1);\nCOMMIT;\nSELECT 1/0;", ["line 1 of the SQL: BEGIN", "line 3 of the SQL: COMMIT"]), + ("INSERT INTO s.t VALUES (1); savepoint x;", ["line 1 of the SQL: SAVEPOINT"]), + ("INSERT INTO s.t VALUES (1);\n\\! rm -rf /", ["line 2 of the SQL: psql meta-commands"]), +]) +def test_refusals_name_transaction_control_and_meta_commands(sql, found): + problems = refusals(sql) + assert len(problems) == len(found) + assert all(p.startswith(f) for p, f in zip(problems, found)) class _FakeHost: @@ -179,3 +154,14 @@ async def test_apply_sql_failure_raises_psql_error(host): with pytest.raises(ServiceDbSqlError, match="syntax error at or near"): await ApplySqlTaskStep(id="inject", version=1, env_id="e", sql="INSRT INTO s.t VALUES (1);").execute(ctx) assert ctx.metadata["applied_sql"]["inject"]["applied"] is False + + +@pytest.mark.asyncio +async def test_apply_sql_refuses_a_commit_before_running_anything(host): + host["host"] = h = _FakeHost() + ctx = TaskStepContext() + with pytest.raises(RefusedSqlError, match="line 2 of the SQL: COMMIT"): + await ApplySqlTaskStep(id="inject", version=1, env_id="e", + sql="INSERT INTO s.t VALUES (1);\nCOMMIT;\nSELECT 1/0;").execute(ctx) + assert h.scripts == [] + assert ctx.metadata["applied_sql"]["inject"]["applied"] is False From e9e799f06720f4425889589fae946a3609c29c86 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 9 Nov 2026 15:02:37 -0500 Subject: [PATCH 4/4] apply_sql: read E'' escape strings and nested comments when refusing SQL The refusal check found literals with a regex that ended an E'...' string at a backslash-escaped quote, so valid SQL such as E'it\'s\tback' was refused as a psql meta-command. A small scanner now follows PostgreSQL's lexer for the cases that matter: E'' strings take backslash escapes, other strings only '', /* */ comments nest, and $tag$ bodies end at the same tag. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/apply_sql.py | 60 +++++++++++++++++-- tst/unit/task_step/test_apply_sql.py | 6 ++ 2 files changed, 60 insertions(+), 6 deletions(-) diff --git a/src/agent_env/task_step/task_steps/apply_sql.py b/src/agent_env/task_step/task_steps/apply_sql.py index a33be7b6b..d2dde2477 100644 --- a/src/agent_env/task_step/task_steps/apply_sql.py +++ b/src/agent_env/task_step/task_steps/apply_sql.py @@ -82,11 +82,59 @@ """ -# Comments, string literals, quoted identifiers and dollar-quoted bodies, which hold no statement of the script's own. -_LITERAL = re.compile( - r"--[^\n]*|/\*.*?\*/|'(?:[^']|'')*'|\"(?:[^\"]|\"\")*\"|\$([A-Za-z_][A-Za-z0-9_]*|)\$.*?\$\1\$", - re.DOTALL, -) +_DOLLAR_TAG = re.compile(r"\$([A-Za-z_][A-Za-z0-9_]*)?\$") + + +def _blank(text: str) -> str: + return re.sub(r"[^\n]", " ", text) + + +def _code_only(sql: str) -> str: + """``sql`` with its comments, string literals, quoted identifiers and dollar-quoted bodies blanked out (newlines + kept, so positions still give line numbers): what is left is the script's own statements. + + Follows PostgreSQL's lexer where it matters here: ``/* */`` comments nest, ``E'...'`` strings take backslash + escapes (``E'it\\'s'``), other strings only ``''``, and ``$tag$ ... $tag$`` bodies end at the same tag. An + unterminated literal runs to the end, as psql would read it. + """ + out, i, n = [], 0, len(sql) + while i < n: + c, start = sql[i], i + if sql.startswith("--", i): + j = sql.find("\n", i) + i = n if j < 0 else j + elif sql.startswith("/*", i): + depth, i = 1, i + 2 + while i < n and depth: + if sql.startswith("/*", i): + depth, i = depth + 1, i + 2 + elif sql.startswith("*/", i): + depth, i = depth - 1, i + 2 + else: + i += 1 + elif c in "'\"": + escapes = c == "'" and i > 0 and sql[i - 1] in "eE" and (i < 2 or not (sql[i - 2].isalnum() or sql[i - 2] in "_$")) + i += 1 + while i < n: + if escapes and sql[i] == "\\": + i += 2 + elif sql[i] == c: + if sql.startswith(c * 2, i): + i += 2 + else: + i += 1 + break + else: + i += 1 + elif c == "$" and (m := _DOLLAR_TAG.match(sql, i)) and (i == 0 or not (sql[i - 1].isalnum() or sql[i - 1] == "_")): + j = sql.find(m.group(0), m.end()) + i = n if j < 0 else j + len(m.group(0)) + else: + out.append(c) + i += 1 + continue + out.append(_blank(sql[start:i])) + return "".join(out) _TRANSACTION_CONTROL = re.compile( r"(?:\A|;)\s*(begin|commit|rollback|end|abort|savepoint|release|start\s+transaction|prepare\s+transaction)\b", re.IGNORECASE, @@ -104,7 +152,7 @@ def __init__(self, what: str, message: str): def refusals(sql: str) -> list[str]: """Why ``sql`` can't run as one ``apply_sql`` transaction, each naming its line; empty when it can.""" - code = _LITERAL.sub(lambda m: re.sub(r"[^\n]", " ", m.group(0)), sql) + code = _code_only(sql) found = [] for m in _TRANSACTION_CONTROL.finditer(code): line = code.count("\n", 0, m.start(1)) + 1 diff --git a/tst/unit/task_step/test_apply_sql.py b/tst/unit/task_step/test_apply_sql.py index 2d18ededb..9912b8b7f 100644 --- a/tst/unit/task_step/test_apply_sql.py +++ b/tst/unit/task_step/test_apply_sql.py @@ -86,6 +86,9 @@ def test_psql_error_keeps_error_lines_and_names_a_missing_service(): "DO $$ BEGIN PERFORM 1; END $$;", "CREATE FUNCTION f() RETURNS int AS $body$ BEGIN RETURN 1; END $body$ LANGUAGE plpgsql;", "-- COMMIT later\n/* ROLLBACK */ INSERT INTO s.t VALUES (E'it''s back\\tafter');", + "INSERT INTO s.t VALUES (E'it\\'s\\tback; COMMIT');", + "/* outer /* inner */ COMMIT; still a comment */ INSERT INTO s.t VALUES (1);", + "INSERT INTO s.t VALUES ($tag$ COMMIT; \\! $notag$ $tag$);", ]) def test_refusals_ignore_literals_comments_and_bodies(sql): assert refusals(sql) == [] @@ -95,6 +98,9 @@ def test_refusals_ignore_literals_comments_and_bodies(sql): ("BEGIN;\nINSERT INTO s.t VALUES (1);\nCOMMIT;\nSELECT 1/0;", ["line 1 of the SQL: BEGIN", "line 3 of the SQL: COMMIT"]), ("INSERT INTO s.t VALUES (1); savepoint x;", ["line 1 of the SQL: SAVEPOINT"]), ("INSERT INTO s.t VALUES (1);\n\\! rm -rf /", ["line 2 of the SQL: psql meta-commands"]), + # Without the E prefix a backslash is an ordinary character, so the quote after it ends the string. + ("INSERT INTO s.t VALUES ('a\\');\nCOMMIT;", ["line 2 of the SQL: COMMIT"]), + ("INSERT INTO s.t VALUES (E'a\\'');\nCOMMIT;", ["line 2 of the SQL: COMMIT"]), ]) def test_refusals_name_transaction_control_and_meta_commands(sql, found): problems = refusals(sql)