Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions backend/migrations/versions/a8b9c0d1e2f3_add_insight_evidence.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,12 @@


def upgrade() -> None:
# SQLite DDL can outlive a failed migration unless a real transaction exists.
with op.get_bind().begin_nested():
_upgrade()


def _upgrade() -> None:
inspector = sa.inspect(op.get_bind())
columns = {column["name"] for column in inspector.get_columns("insights")}
for column in (
Expand All @@ -36,6 +42,19 @@ def upgrade() -> None:
# Prototype installations may already have the current metadata schema.
constraints = {constraint["name"] for constraint in inspector.get_unique_constraints("insights")}
if "uq_insights_dedupe_key" not in constraints:
if inspector.has_table("_alembic_tmp_insights"):
metadata = sa.MetaData()
insights = sa.Table("insights", metadata, autoload_with=op.get_bind())
temporary = sa.Table("_alembic_tmp_insights", metadata, autoload_with=op.get_bind())
# Only discard a failed batch copy when every row survives in the source.
if set(temporary.c.keys()) != set(insights.c.keys()):
raise RuntimeError("Cannot safely recover interrupted insights migration: column mismatch")
missing = sa.select(*temporary.c).except_(
sa.select(*(insights.c[column.name] for column in temporary.c))
).limit(1)
if op.get_bind().scalar(sa.select(missing.exists())):
raise RuntimeError("Cannot safely recover interrupted insights migration: unpreserved rows")
op.drop_table("_alembic_tmp_insights")
with op.batch_alter_table("insights") as batch_op:
batch_op.create_unique_constraint("uq_insights_dedupe_key", ["dedupe_key"])

Expand Down
50 changes: 50 additions & 0 deletions backend/tests/test_insight_evidence_migration.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
from pathlib import Path

import pytest
from alembic import command
from alembic.config import Config
from sqlalchemy import create_engine, inspect, text
from sqlalchemy.exc import IntegrityError

from app.core import settings as settings_module


@pytest.mark.parametrize("temporary_rows", [0, 1, 2])
def test_insight_migration_recovers_only_preserved_batch_rows(tmp_path: Path, monkeypatch, temporary_rows: int):
database_url = f"sqlite:///{tmp_path / 'insights.db'}"
monkeypatch.setattr(settings_module.settings, "database_url", database_url)
config = Config(str(Path(__file__).resolve().parents[1] / "alembic.ini"))
command.upgrade(config, "e7f8a9b0c1d2")
engine = create_engine(database_url)
with engine.begin() as connection:
connection.execute(text(
"INSERT INTO insights (id, timestamp, type, title) "
"VALUES (1, '2026-01-01', 'web.scan', 'Existing insight')"
))
for definition in (
"occurred_at DATETIME", "correlation_key VARCHAR(160)",
"dedupe_key VARCHAR(255)", "evidence JSON",
):
connection.execute(text(f"ALTER TABLE insights ADD COLUMN {definition}"))
connection.execute(text("CREATE TABLE _alembic_tmp_insights AS SELECT * FROM insights WHERE 0"))
if temporary_rows:
connection.execute(text("INSERT INTO _alembic_tmp_insights SELECT * FROM insights"))
if temporary_rows == 2:
connection.execute(text("UPDATE _alembic_tmp_insights SET title = 'Unpreserved insight'"))

if temporary_rows == 2:
with pytest.raises(RuntimeError, match="unpreserved rows"):
command.upgrade(config, "a8b9c0d1e2f3")
assert inspect(engine).has_table("_alembic_tmp_insights")
else:
command.upgrade(config, "head")
assert not inspect(engine).has_table("_alembic_tmp_insights")
with engine.begin() as connection:
assert connection.execute(text("SELECT title FROM insights WHERE id = 1")).scalar_one() == "Existing insight"
connection.execute(text("UPDATE insights SET dedupe_key = 'unique' WHERE id = 1"))
with pytest.raises(IntegrityError):
connection.execute(text(
"INSERT INTO insights (timestamp, type, title, dedupe_key) "
"VALUES ('2026-01-01', 'web.scan', 'Duplicate', 'unique')"
))
engine.dispose()
9 changes: 9 additions & 0 deletions website/guide/operations/updating.md
Original file line number Diff line number Diff line change
Expand Up @@ -93,3 +93,12 @@ After the container or service restarts:
3. If CrowdSec is enabled, confirm `plugin · crowdsec` reports the log reader and `crowdsec · lapi` reports decision synchronization separately.
4. Send a test notification if SMTP is enabled.
5. Reload an open browser tab so versioned JavaScript and CSS assets are refreshed.

### Interrupted insights migration

The insight evidence migration automatically retries an interrupted SQLite table
rebuild when its temporary table is empty or all its rows are preserved in
`insights`. The rebuild runs inside a savepoint so a failed retry rolls back its
schema changes. If the temporary table contains differing rows or columns, startup
stops with a recovery error and preserves both tables for investigation. Back up
the stopped instance's data directory before attempting manual database recovery.
Loading