From f3a4305364394bbebfc1fa3873fc7fe00acd3355 Mon Sep 17 00:00:00 2001 From: Konstantin Koschel Date: Sun, 8 Nov 2026 21:36:29 +0100 Subject: [PATCH] fix: allow sufficient temporary storage for SQLite upgrades --- .github/workflows/docker-publish.yml | 6 ++- .github/workflows/tests.yml | 4 +- backend/tests/performance/README.md | 5 +++ .../tests/performance/upgrade_benchmark.py | 37 +++++++++++++++++-- docker/docker-compose.example.yml | 2 +- website/guide/getting-started/quickstart.md | 2 +- website/guide/installation/docker.md | 10 ++++- website/guide/operations/updating.md | 24 ++++++++++++ 8 files changed, 79 insertions(+), 11 deletions(-) diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 52266461..e8a19dbe 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -191,10 +191,12 @@ jobs: --iterations 20 --default-range 24h --enforce-gates \ --output /reports/profile-large.json docker run --rm --user "${runner_user}" --memory 1g --cpus 2 \ + --read-only --tmpfs /tmp:size=1g,mode=1777 \ + --tmpfs /data:size=256m,mode=1777 \ --entrypoint python -v "${report_dir}:/reports" \ opensecdash:release-candidate-a \ -m tests.performance.upgrade_benchmark --events 10000 \ - --enforce-gates --output /reports/profile-upgrade.json + --data-dir /data --enforce-gates --output /reports/profile-upgrade.json - name: Smoke test hardened image with scratch storage shell: bash @@ -212,7 +214,7 @@ jobs: -c 'touch /data/root-owned && chown root:root /data/root-owned' docker run -d --name opensecdash-release-smoke \ --read-only \ - --tmpfs /tmp:size=16m,mode=1777 \ + --tmpfs /tmp:size=1g,mode=1777 \ --security-opt no-new-privileges:true \ --cap-drop ALL --cap-add CHOWN --cap-add SETGID --cap-add SETUID \ --pids-limit 256 --memory 1g --cpus 2 \ diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5691bbce..e41a5fb3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -111,7 +111,7 @@ jobs: - name: Run upgrade benchmark gate # Catch migration/startup regressions in PR CI, not only at release time. - # The 10k-event benchmark takes ~240ms locally, so it is a cheap gate. + # Includes legacy insight payloads large enough to exercise table rebuilds. working-directory: backend run: .venv/bin/python -m tests.performance.upgrade_benchmark --events 10000 --enforce-gates @@ -186,7 +186,7 @@ jobs: -c 'touch /data/root-owned && chown root:root /data/root-owned' docker run -d --name opensecdash-ci-smoke \ --read-only \ - --tmpfs /tmp:size=16m,mode=1777 \ + --tmpfs /tmp:size=1g,mode=1777 \ --security-opt no-new-privileges:true \ --cap-drop ALL --cap-add CHOWN --cap-add SETGID --cap-add SETUID \ --pids-limit 256 --memory 1g --cpus 2 \ diff --git a/backend/tests/performance/README.md b/backend/tests/performance/README.md index 17a58508..a64cde1c 100644 --- a/backend/tests/performance/README.md +++ b/backend/tests/performance/README.md @@ -34,6 +34,11 @@ readiness calls, connection cleanup, and a machine-readable gate result. The upg profile builds a representative supported legacy schema from the first migration, preserves synthetic legacy events, migrates secrets, runs startup maintenance, confirms auth remains off, and proves the second start skips the event scan. +It also preserves 10,000 legacy insights with about 40 MiB of description payload +through the table rebuild. The Docker Upgrade profile uses a read-only root +filesystem and the documented 1 GiB `/tmp` tmpfs. Its scratch database lives on a +separate writable `/data` mount via `--data-dir /data`, so database storage cannot +consume the temporary SQLite workspace being tested. ## Level 2 anchor history regression diff --git a/backend/tests/performance/upgrade_benchmark.py b/backend/tests/performance/upgrade_benchmark.py index ce4e40a4..c5a4b780 100644 --- a/backend/tests/performance/upgrade_benchmark.py +++ b/backend/tests/performance/upgrade_benchmark.py @@ -23,6 +23,7 @@ from app.database import init_db as init_db_module from app.database import session as session_module from app.database.migrations import alembic_config +from app.models.core import Insight from app.models.events import Event from app.models.settings import Setting from app.models.users import User, UserSession @@ -35,6 +36,9 @@ LEGACY_REVISION = "7f4a4a9c2b1e" DEFAULT_EVENT_COUNT = 10_000 +# A batch rebuild must spill more than the former 16 MiB /tmp allowance. +LEGACY_INSIGHT_COUNT = 10_000 +LEGACY_INSIGHT_DESCRIPTION = "Synthetic legacy probe evidence. " * 128 # A real v0.4.2 install carries its general settings but predates auth, so the @@ -88,17 +92,31 @@ def _insert_legacy_rows(database_path: Path, event_count: int) -> None: connection.commit() -def run(event_count: int) -> dict[str, object]: +def _insert_legacy_insights(database_path: Path) -> None: + with sqlite3.connect(database_path) as connection: + # Existing application databases use WAL, including during batch rebuilds. + connection.execute("PRAGMA journal_mode=WAL") + connection.executemany( + "INSERT INTO insights (id, timestamp, type, title, description) VALUES (?, ?, ?, ?, ?)", + ( + (index + 1, "2026-11-01 12:00:00", "web.scan", "Legacy probe", LEGACY_INSIGHT_DESCRIPTION) + for index in range(LEGACY_INSIGHT_COUNT) + ), + ) + + +def run(event_count: int, data_dir: Path | None = None) -> dict[str, object]: original_database_url = settings.database_url original_engine = init_db_module.engine original_session_factory = session_module.SessionLocal try: - with tempfile.TemporaryDirectory(prefix="opensecdash-upgrade-benchmark-") as directory: + with tempfile.TemporaryDirectory(prefix="opensecdash-upgrade-benchmark-", dir=data_dir) as directory: database_path = Path(directory) / "legacy.db" settings.database_url = f"sqlite:///{database_path}" config = alembic_config() command.upgrade(config, LEGACY_REVISION) _insert_legacy_rows(database_path, event_count) + _insert_legacy_insights(database_path) migration_started = time.perf_counter() command.upgrade(config, "head") @@ -132,6 +150,13 @@ def capture_statement(connection, cursor, statement, parameters, context, execut db = session_factory() try: events_after = db.query(func.count(Event.id)).scalar() or 0 + insights_after = db.query(func.count(Insight.id)).scalar() or 0 + preserved_insights = db.query(func.count(Insight.id)).filter( + Insight.id.between(1, LEGACY_INSIGHT_COUNT), + Insight.type == "web.scan", + Insight.title == "Legacy probe", + Insight.description == LEGACY_INSIGHT_DESCRIPTION, + ).scalar() or 0 auth_enabled = db.query(Setting).filter(Setting.key == AUTH_ENABLED_SETTING).first() onboarding = db.query(Setting).filter(Setting.key == AUTH_ONBOARDING_STATE_SETTING).first() users_after = db.query(func.count(User.id)).scalar() or 0 @@ -151,6 +176,7 @@ def capture_statement(connection, cursor, statement, parameters, context, execut # The open legacy install stays open and fully readable, proving # the review state blocks no data path. "all_legacy_events_preserved": events_after == event_count, + "all_legacy_insights_preserved": insights_after == preserved_insights == LEGACY_INSIGHT_COUNT, "auth_remains_disabled": auth_enabled is not None and auth_enabled.value == "false", "onboarding_review_required": onboarding is not None and onboarding.value == AUTH_ONBOARDING_LEGACY_REVIEW_REQUIRED, @@ -163,6 +189,10 @@ def capture_statement(connection, cursor, statement, parameters, context, execut "profile": "upgrade", "legacy_revision": LEGACY_REVISION, "events": event_count, + "insights": LEGACY_INSIGHT_COUNT, + "insight_payload_mib": round( + LEGACY_INSIGHT_COUNT * len(LEGACY_INSIGHT_DESCRIPTION.encode("utf-8")) / (1024 * 1024), 2 + ), "database_mib": round(database_path.stat().st_size / (1024 * 1024), 2), "migration_ms": round(migration_ms, 2), "first_start_ms": round(first_start_ms, 2), @@ -181,10 +211,11 @@ def main() -> None: parser.add_argument("--events", type=int, default=DEFAULT_EVENT_COUNT) parser.add_argument("--enforce-gates", action="store_true") parser.add_argument("--output", type=Path) + parser.add_argument("--data-dir", type=Path, help="Parent directory for the temporary legacy database") arguments = parser.parse_args() if arguments.events < 1: parser.error("--events must be positive") - report = run(arguments.events) + report = run(arguments.events, arguments.data_dir) serialized = json.dumps(report, indent=2) if arguments.output is None: print(serialized) diff --git a/docker/docker-compose.example.yml b/docker/docker-compose.example.yml index 5db41228..536486ca 100644 --- a/docker/docker-compose.example.yml +++ b/docker/docker-compose.example.yml @@ -17,7 +17,7 @@ services: # - ./assets/assets.json:/assets/assets.json:ro read_only: true tmpfs: - - /tmp:size=16m,mode=1777 + - /tmp:size=1g,mode=1777 security_opt: - no-new-privileges:true cap_drop: diff --git a/website/guide/getting-started/quickstart.md b/website/guide/getting-started/quickstart.md index c5270051..eddef29d 100644 --- a/website/guide/getting-started/quickstart.md +++ b/website/guide/getting-started/quickstart.md @@ -15,7 +15,7 @@ services: - opensecdash-data:/data read_only: true tmpfs: - - /tmp:size=16m,mode=1777 + - /tmp:size=1g,mode=1777 security_opt: - no-new-privileges:true cap_drop: diff --git a/website/guide/installation/docker.md b/website/guide/installation/docker.md index 7c3b8735..3d7f8176 100644 --- a/website/guide/installation/docker.md +++ b/website/guide/installation/docker.md @@ -55,7 +55,7 @@ services: - opensecdash-data:/data read_only: true tmpfs: - - /tmp:size=16m,mode=1777 + - /tmp:size=1g,mode=1777 security_opt: - no-new-privileges:true cap_drop: @@ -99,7 +99,7 @@ services: - ./assets/assets.json:/assets/assets.json:ro read_only: true tmpfs: - - /tmp:size=16m,mode=1777 + - /tmp:size=1g,mode=1777 security_opt: - no-new-privileges:true cap_drop: @@ -137,6 +137,12 @@ With `network_mode: "host"` the container shares the host's network stack, so th +The examples allow up to 1 GiB of temporary files in `/tmp` for SQLite migrations +on existing databases. Memory is consumed as files are written and shares the +container's memory allowance with the application. When upgrading a Compose file +that still uses `size=16m`, update the mount and follow the +[migration recovery instructions](../operations/updating.md#interrupted-insights-migration). + Start the app: ```bash diff --git a/website/guide/operations/updating.md b/website/guide/operations/updating.md index 90c5b778..04c9875c 100644 --- a/website/guide/operations/updating.md +++ b/website/guide/operations/updating.md @@ -96,6 +96,30 @@ After the container or service restarts: ### Interrupted insights migration +SQLite table rebuilds can require temporary storage in `/tmp` as well as free +space in `/data`. Compose examples before this correction limited `/tmp` to +16 MiB, which can cause `database or disk is full` during an upgrade with existing +insights even when the data volume has ample free space. A subsequent +`no such savepoint` error can hide this initial failure. + +Set the temporary storage limit in your Compose service to 1 GiB: + +```yaml +tmpfs: + - /tmp:size=1g,mode=1777 +``` + +Recreate the container to apply the mount change and retry the migration: + +```bash +docker compose up -d --force-recreate opensecdash +docker compose logs -f --tail=100 opensecdash +``` + +The tmpfs limit is a ceiling, with memory consumed as files are written. Its usage +counts toward the container's memory limit alongside the application. Larger +databases may require more temporary storage and a corresponding memory allowance. + The insight evidence migration automatically retries an interrupted SQLite table rebuild when its temporary table is empty or all its rows are preserved in `insights`. The rebuild runs inside a savepoint so a failed retry rolls back its