From fa8218d286cb6b9d9e6588575537cc5937d69d6f Mon Sep 17 00:00:00 2001 From: Konstantin Koschel Date: Sun, 8 Nov 2026 20:54:24 +0100 Subject: [PATCH] fix: recover interrupted insight evidence migrations --- .../a8b9c0d1e2f3_add_insight_evidence.py | 19 +++++++ .../tests/test_insight_evidence_migration.py | 50 +++++++++++++++++++ website/guide/operations/updating.md | 9 ++++ 3 files changed, 78 insertions(+) create mode 100644 backend/tests/test_insight_evidence_migration.py diff --git a/backend/migrations/versions/a8b9c0d1e2f3_add_insight_evidence.py b/backend/migrations/versions/a8b9c0d1e2f3_add_insight_evidence.py index 56e4efd3..de42df85 100644 --- a/backend/migrations/versions/a8b9c0d1e2f3_add_insight_evidence.py +++ b/backend/migrations/versions/a8b9c0d1e2f3_add_insight_evidence.py @@ -18,6 +18,12 @@ def upgrade() -> None: + # SQLite DDL can outlive a failed migration unless a real transaction exists. + with op.get_bind().begin_nested(): + _upgrade() + + +def _upgrade() -> None: inspector = sa.inspect(op.get_bind()) columns = {column["name"] for column in inspector.get_columns("insights")} for column in ( @@ -36,6 +42,19 @@ def upgrade() -> None: # Prototype installations may already have the current metadata schema. constraints = {constraint["name"] for constraint in inspector.get_unique_constraints("insights")} if "uq_insights_dedupe_key" not in constraints: + if inspector.has_table("_alembic_tmp_insights"): + metadata = sa.MetaData() + insights = sa.Table("insights", metadata, autoload_with=op.get_bind()) + temporary = sa.Table("_alembic_tmp_insights", metadata, autoload_with=op.get_bind()) + # Only discard a failed batch copy when every row survives in the source. + if set(temporary.c.keys()) != set(insights.c.keys()): + raise RuntimeError("Cannot safely recover interrupted insights migration: column mismatch") + missing = sa.select(*temporary.c).except_( + sa.select(*(insights.c[column.name] for column in temporary.c)) + ).limit(1) + if op.get_bind().scalar(sa.select(missing.exists())): + raise RuntimeError("Cannot safely recover interrupted insights migration: unpreserved rows") + op.drop_table("_alembic_tmp_insights") with op.batch_alter_table("insights") as batch_op: batch_op.create_unique_constraint("uq_insights_dedupe_key", ["dedupe_key"]) diff --git a/backend/tests/test_insight_evidence_migration.py b/backend/tests/test_insight_evidence_migration.py new file mode 100644 index 00000000..7a79c701 --- /dev/null +++ b/backend/tests/test_insight_evidence_migration.py @@ -0,0 +1,50 @@ +from pathlib import Path + +import pytest +from alembic import command +from alembic.config import Config +from sqlalchemy import create_engine, inspect, text +from sqlalchemy.exc import IntegrityError + +from app.core import settings as settings_module + + +@pytest.mark.parametrize("temporary_rows", [0, 1, 2]) +def test_insight_migration_recovers_only_preserved_batch_rows(tmp_path: Path, monkeypatch, temporary_rows: int): + database_url = f"sqlite:///{tmp_path / 'insights.db'}" + monkeypatch.setattr(settings_module.settings, "database_url", database_url) + config = Config(str(Path(__file__).resolve().parents[1] / "alembic.ini")) + command.upgrade(config, "e7f8a9b0c1d2") + engine = create_engine(database_url) + with engine.begin() as connection: + connection.execute(text( + "INSERT INTO insights (id, timestamp, type, title) " + "VALUES (1, '2026-01-01', 'web.scan', 'Existing insight')" + )) + for definition in ( + "occurred_at DATETIME", "correlation_key VARCHAR(160)", + "dedupe_key VARCHAR(255)", "evidence JSON", + ): + connection.execute(text(f"ALTER TABLE insights ADD COLUMN {definition}")) + connection.execute(text("CREATE TABLE _alembic_tmp_insights AS SELECT * FROM insights WHERE 0")) + if temporary_rows: + connection.execute(text("INSERT INTO _alembic_tmp_insights SELECT * FROM insights")) + if temporary_rows == 2: + connection.execute(text("UPDATE _alembic_tmp_insights SET title = 'Unpreserved insight'")) + + if temporary_rows == 2: + with pytest.raises(RuntimeError, match="unpreserved rows"): + command.upgrade(config, "a8b9c0d1e2f3") + assert inspect(engine).has_table("_alembic_tmp_insights") + else: + command.upgrade(config, "head") + assert not inspect(engine).has_table("_alembic_tmp_insights") + with engine.begin() as connection: + assert connection.execute(text("SELECT title FROM insights WHERE id = 1")).scalar_one() == "Existing insight" + connection.execute(text("UPDATE insights SET dedupe_key = 'unique' WHERE id = 1")) + with pytest.raises(IntegrityError): + connection.execute(text( + "INSERT INTO insights (timestamp, type, title, dedupe_key) " + "VALUES ('2026-01-01', 'web.scan', 'Duplicate', 'unique')" + )) + engine.dispose() diff --git a/website/guide/operations/updating.md b/website/guide/operations/updating.md index 14978f95..90c5b778 100644 --- a/website/guide/operations/updating.md +++ b/website/guide/operations/updating.md @@ -93,3 +93,12 @@ After the container or service restarts: 3. If CrowdSec is enabled, confirm `plugin · crowdsec` reports the log reader and `crowdsec · lapi` reports decision synchronization separately. 4. Send a test notification if SMTP is enabled. 5. Reload an open browser tab so versioned JavaScript and CSS assets are refreshed. + +### Interrupted insights migration + +The insight evidence migration automatically retries an interrupted SQLite table +rebuild when its temporary table is empty or all its rows are preserved in +`insights`. The rebuild runs inside a savepoint so a failed retry rolls back its +schema changes. If the temporary table contains differing rows or columns, startup +stops with a recovery error and preserves both tables for investigation. Back up +the stopped instance's data directory before attempting manual database recovery.