Own source capture and publish revision-safe search

.env.example

@@ -14,15 +14,15 @@ REDIS_HOST=localhost
 REDIS_PORT=6379
 
 # Authentication
-# Generated by start.sh if empty
-FIRST_SUPERUSER=
-# Generated by start.sh if empty
-FIRST_SUPERUSER_PASSWORD=
+# Local demo bootstrap only; omit these in api_key mode.
+# FIRST_SUPERUSER=
+# FIRST_SUPERUSER_PASSWORD=
 ENCRYPTION_KEY=
-AUTH_ENABLED=false
+AUTH_MODE=api_key
+# Opt into AUTH_MODE=local only for isolated local development.
 STATE_SECRET=
 
-# Auth0 Configuration (required if AUTH_ENABLED=true)
+# Auth0 Configuration (required if AUTH_MODE=auth0)
 AUTH0_DOMAIN=
 AUTH0_AUDIENCE=
 AUTH0_RULE_NAMESPACE=
@@ -123,8 +123,6 @@ ADDITIONAL_CORS_ORIGINS=
 # Other Settings
 PROJECT_NAME=Airweave
 LOG_LEVEL=INFO
-RUN_ALEMBIC_MIGRATIONS=true
-RUN_DB_SYNC=true
 CODE_SUMMARIZER_ENABLED=false
 
 SVIX_URL=http://localhost:8071

.github/workflows/monke.yml

@@ -1,13 +1,9 @@
 name: Monke – parallel runners
 
 on:
+  # The owned fork uses credential-free source-store checks on pull requests.
+  # Upstream's live connector mutators require separate disposable accounts.
   workflow_dispatch:
-  pull_request:
-    paths:
-      # Only trigger on platform-related changes (reduced footprint)
-      - "monke/**"
-      - "backend/airweave/platform/**"
-      - ".github/workflows/monke.yml"
 
 concurrency:
   group: monke-${{ github.ref }}

.github/workflows/owned-source-store.yml

@@ -0,0 +1,126 @@
+name: Owned source store
+
+on:
+  pull_request:
+    branches: [dev, main]
+    paths:
+      - backend/**
+      - evaluation/**
+      - vespa/app/**
+      - Makefile
+      - .github/workflows/owned-source-store.yml
+  push:
+    branches: [dev]
+    paths:
+      - backend/**
+      - evaluation/**
+      - vespa/app/**
+      - Makefile
+      - .github/workflows/owned-source-store.yml
+
+permissions:
+  contents: read
+
+concurrency:
+  group: owned-store-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  evaluation:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
+        with:
+          python-version: '3.13'
+      - name: Verify offline retrieval metrics and file boundaries
+        run: |
+          python -m venv .venv-evaluation
+          .venv-evaluation/bin/python -m pip install ir-measures==0.4.3 pydantic==2.11.9 pytest==8.3.5
+          .venv-evaluation/bin/python -m pytest evaluation/tests -q --confcutdir=evaluation
+
+  correctness:
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    services:
+      vespa:
+        image: vespaengine/vespa@sha256:5c30f5c41e7563498c4f925db6a837a3848f04726a3ed26aed4a7c8ab69f18fd
+        ports:
+          - 8081:8081
+          - 19071:19071
+        options: >-
+          --hostname vespa
+          --health-cmd "curl -f http://localhost:19071/state/v1/health"
+          --health-interval 10s --health-timeout 5s --health-retries 30
+      minilm:
+        image: semitechnologies/transformers-inference@sha256:c9c13f047ee31488e686c4a7b2ea7ccfcdec16ee825447450dc047cf97791590
+        env:
+          ENABLE_CUDA: '0'
+        ports:
+          - 8080:8080
+        options: --memory=2g --cpus=2
+      postgres:
+        image: postgres:16.15
+        env:
+          POSTGRES_USER: sync_test
+          POSTGRES_PASSWORD: disposable-ci-only
+          POSTGRES_DB: sync_tests
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U sync_test -d sync_tests"
+          --health-interval 5s --health-timeout 5s --health-retries 10
+    env:
+      CANONICAL_TEST_DATABASE_URL: postgresql+asyncpg://sync_test:disposable-ci-only@localhost:5432/sync_tests
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405
+        with:
+          python-version: '3.13'
+      - uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a
+        with:
+          version: 2.3.2
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+      - name: Install locked dependencies
+        working-directory: backend
+        run: poetry install --with dev,lint --no-root --no-interaction
+      - name: Verify durable records and publication
+        run: make test-store
+      - name: Verify source capture and transport
+        run: make test-capture
+      - name: Verify connection provisioning and recovery
+        run: make test-provisioning
+      - name: Verify indexed retrieval and visibility
+        run: make test-search
+      - name: Verify service authentication
+        run: make test-auth
+      - name: Verify dependency readiness
+        run: make test-health
+      - name: Verify OCR execution boundaries and fallback
+        run: make test-ocr
+      - name: Verify projection recovery and maintenance workflows
+        run: make test-worker
+      - name: Record tested Vespa image
+        run: docker image inspect vespaengine/vespa@sha256:5c30f5c41e7563498c4f925db6a837a3848f04726a3ed26aed4a7c8ab69f18fd --format '{{json .RepoDigests}}'
+      - name: Verify real Vespa schema and retrieval
+        env:
+          OWNED_VESPA_TEST: '1'
+          OWNED_MINILM_TEST: '1'
+          EMBEDDING_DIMENSIONS: '384'
+        run: make test-index
+
+  image:
+    needs: correctness
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd
+      - name: Build the deployable image without publishing
+        run: make build IMAGE=almanac-source-store:ci
+      - name: Verify packaged Python and entrypoint syntax
+        run: |
+          docker run --rm --entrypoint python almanac-source-store:ci -m compileall -q airweave
+          docker run --rm --entrypoint /bin/sh almanac-source-store:ci -n /app/entrypoint.sh
+          docker run --rm --entrypoint python almanac-source-store:ci -m scripts.smoke_local_ocr --tessdata /usr/share/tessdata

Makefile

@@ -0,0 +1,67 @@
+.DEFAULT_GOAL := help
+BACKEND := backend
+PYTHON := .venv/bin/python
+PYTEST := .venv/bin/pytest
+IMAGE ?= almanac-source-store:local
+
+.PHONY: help setup check test-store test-provisioning test-capture test-search test-auth test-health test-worker test-ocr test-index validate-deploy build
+
+help:
+	@echo 'setup         Install the locked backend and development dependencies'
+	@echo 'build         Build the backend image locally; set IMAGE to choose its tag'
+	@echo 'test-store    Verify record transactions and publication on disposable PostgreSQL'
+	@echo 'test-capture  Verify source capture, transport and blob handling'
+	@echo 'test-search   Verify search services and visibility behavior'
+	@echo 'test-auth     Verify service-key and source authorization boundaries'
+	@echo 'test-health   Verify dependency probes and readiness behavior'
+	@echo 'test-worker   Verify owned projection, recovery and maintenance workflows'
+	@echo 'test-ocr      Verify local OCR boundaries and per-file fallback'
+	@echo 'test-index    Deploy/test schemas on explicitly disposable local Vespa'
+	@echo 'check         Run store, capture, search, authentication and health checks'
+	@echo 'validate-deploy Validate the Porter application manifest locally (no deployment)'
+	@echo ''
+	@echo 'test-store requires CANONICAL_TEST_DATABASE_URL; it creates and removes test schemas.'
+	@echo 'See deploy/README.md for staging prerequisites and explicit deployment steps.'
+
+build:
+	docker build --tag "$(IMAGE)" $(BACKEND)
+
+validate-deploy:
+	porter apply validate -f porter.yaml
+
+setup:
+	cd $(BACKEND) && POETRY_VIRTUALENVS_IN_PROJECT=true uvx --from poetry==2.3.2 poetry install --with dev,lint --no-root --no-interaction
+
+test-store:
+	@test -n "$$CANONICAL_TEST_DATABASE_URL" || (echo 'Set CANONICAL_TEST_DATABASE_URL to a disposable PostgreSQL database.' >&2; exit 1)
+	cd $(BACKEND) && $(PYTEST) -q -o log_cli=false airweave/domains/entities/canonical/tests airweave/domains/native_ingestion/tests airweave/domains/syncs/tests
+
+test-capture:
+	cd $(BACKEND) && $(PYTEST) -q -o log_cli=false airweave/platform/sources/tests tests/unit/platform/sources/records tests/unit/platform/sources/test_*_capture.py tests/unit/domains/entities tests/unit/platform/http_client/test_composio_transport.py tests/unit/platform/http_client/test_logging_privacy.py airweave/domains/storage/tests/test_file_service.py
+
+test-provisioning:
+	@test -n "$$CANONICAL_TEST_DATABASE_URL" || (echo 'Set CANONICAL_TEST_DATABASE_URL to a disposable PostgreSQL database.' >&2; exit 1)
+	cd $(BACKEND) && $(PYTEST) -q -o log_cli=false airweave/domains/owned_provisioning/tests airweave/domains/source_connections/tests airweave/domains/temporal/tests airweave/domains/sources/tests/test_lifecycle.py airweave/domains/sync_pipeline/tests/test_factory.py
+
+test-search:
+	cd $(BACKEND) && $(PYTEST) -q -o log_cli=false airweave/domains/search airweave/domains/embedders
+	PYTHONPATH=$(BACKEND):. $(BACKEND)/$(PYTEST) -q -o log_cli=false $(BACKEND)/tests/unit/search/test_owned_evaluation.py $(BACKEND)/tests/unit/search/test_owned_replay.py
+
+test-auth:
+	cd $(BACKEND) && $(PYTEST) -q -o log_cli=false airweave/api/tests/test_service_auth.py airweave/api/tests/test_sync_authorization.py airweave/api/tests/test_context_resolver_auth.py
+
+test-health:
+	cd $(BACKEND) && $(PYTEST) -q -o log_cli=false airweave/core/health/tests airweave/adapters/health/tests airweave/core/container/tests/test_health_wiring.py
+
+test-worker:
+	@test -n "$$CANONICAL_TEST_DATABASE_URL" || (echo 'Set CANONICAL_TEST_DATABASE_URL to a disposable PostgreSQL database.' >&2; exit 1)
+	cd $(BACKEND) && $(PYTEST) -q -o log_cli=false airweave/domains/temporal/activities/tests/test_project_canonical_records.py airweave/domains/temporal/activities/tests/test_discover_native_projection.py airweave/domains/temporal/activities/tests/test_cleanup_stuck_sync_jobs.py airweave/domains/temporal/workflows/tests/test_native_projection_recovery.py airweave/domains/temporal/workflows/tests/test_source_projection_replay.py airweave/domains/temporal/workflows/tests/test_reproject_command.py airweave/domains/temporal/workflows/tests/test_cleanup_workflows.py airweave/domains/temporal/workflows/tests/test_canonical_projection.py airweave/domains/temporal/worker/tests/test_wiring.py airweave/domains/temporal/worker/tests/test_config.py airweave/domains/temporal/worker/tests/test_worker.py airweave/domains/temporal/worker/tests/test_worker_ocr_guard.py
+
+test-ocr:
+	cd $(BACKEND) && $(PYTEST) -q -o log_cli=false airweave/domains/ocr/tests airweave/core/container/tests/test_ocr_wiring.py
+
+test-index:
+	@test "$$OWNED_VESPA_TEST" = 1 || (echo 'Set OWNED_VESPA_TEST=1 only for a disposable Vespa at localhost:8081/19071; this deploys schemas.' >&2; exit 1)
+	cd $(BACKEND) && $(PYTEST) -q -o log_cli=false tests/integration/test_owned_vespa.py tests/integration/test_canonical_search_prefilters.py airweave/domains/entities/canonical/tests/test_real_vespa_projection.py
+
+check: test-store test-provisioning test-capture test-search test-auth test-health test-worker test-ocr

backend/Dockerfile

@@ -28,6 +28,13 @@ COPY pyproject.toml poetry.lock ./
 RUN --mount=type=cache,target=/opt/poetry-cache \
     poetry install --only=main --no-interaction --no-ansi --no-root
 
+# Optional local OCR models: pinned upstream artifacts, verified by BuildKit.
+# Inference never downloads weights. Enable explicitly with LOCAL_OCR_TESSDATA_PATH.
+ADD --chmod=644 --checksum=sha256:7d4322bd2a7749724879683fc3912cb542f19906c83bcc1a52132556427170b2 https://raw.githubusercontent.com/tesseract-ocr/tessdata_fast/87416418657359cb625c412a48b6e1d6d41c29bd/eng.traineddata /usr/share/tessdata/eng.traineddata
+ADD --chmod=644 --checksum=sha256:4c73ffc59d497c186b19d1e90f5d721d678ea6b2e277b719bee4e2af12271825 https://raw.githubusercontent.com/tesseract-ocr/tessdata_fast/87416418657359cb625c412a48b6e1d6d41c29bd/hin.traineddata /usr/share/tessdata/hin.traineddata
+
+ADD --chmod=644 --checksum=sha256:cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30 https://raw.githubusercontent.com/tesseract-ocr/tessdata_fast/87416418657359cb625c412a48b6e1d6d41c29bd/LICENSE /usr/share/tessdata/LICENSE
+
 # Copy entrypoint script with proper permissions
 COPY --chmod=755 entrypoint.sh /app/entrypoint.sh
 

backend/SERVICE_AUTH.md

@@ -0,0 +1,55 @@
+# Standalone service authentication and database setup
+
+Use `AUTH_MODE=api_key` for the owned service. Every ordinary API request needs
+an existing organization-scoped `X-API-Key`. The default is `api_key`: omitting
+credentials does not select a demo user. This mode makes no Auth0 network calls
+and does not create users from service credentials. User-only routes still
+require user authentication; existing Connect sessions retain their own token
+and scope checks. Auth0 mode preserves the existing user-first authentication
+precedence when both user and API-key credentials are supplied.
+
+`AUTH_MODE=auth0` retains hosted Auth0 user authentication and API-key access.
+The existing Auth0 configuration remains required in that mode.
+`AUTH_MODE=local` explicitly enables insecure demo authentication and is allowed
+only for `ENVIRONMENT=local` or `test`. Never deploy local mode. The deprecated
+`AUTH_ENABLED=true/false` is accepted as a migration input mapping to auth0/local;
+contradicting it with AUTH_MODE is an error. Remove AUTH_ENABLED when adopting
+AUTH_MODE. The environment label is operator-controlled, not an isolation boundary.
+
+Keys are checked against persisted credentials on every request; expiry and
+revocation take effect without waiting for a key-to-organization cache. All
+record and job authorization still uses the key's organization. Authenticated
+sync SSE subscriptions authorize job ownership before subscribing.
+
+## Explicit database setup
+
+Neither API startup nor the container entrypoint migrates the database or seeds
+users, organizations, or keys. `RUN_ALEMBIC_MIGRATIONS` is no longer a runtime
+switch. Run setup as a separately authorized deployment job with reviewed,
+isolated database credentials:
+
+```sh
+# From backend/, against the intended isolated target only:
+poetry run alembic upgrade head
+poetry run python -m airweave.db.bootstrap
+```
+
+The second command installs native destination definitions and the initial embedding
+metadata. Ordinary startup validates that metadata without inserting it. Service
+mode does not require FIRST_SUPERUSER or FIRST_SUPERUSER_PASSWORD; these are
+local bootstrap inputs only. Provision the
+intended service organization and key through the existing model/CRUD operator
+workflow, store the key in secret management, then start the API and workers.
+Do not print keys in logs. Runtime replicas should not need migration privileges.
+Temporal database setup is separate and must also target isolated resources.
+
+For a disposable local demo only:
+
+```sh
+AUTH_MODE=local poetry run python -m airweave.db.bootstrap --local-superuser
+```
+
+This opts into the legacy demo user/org/key provisioning. It is refused in
+api_key or auth0 mode. The production entrypoint starts Uvicorn without reload.
+Almanac/WorkOS remains end-user identity authority; this service does not
+introduce an independent customer login system.

backend/airweave/adapters/health/temporal.py

@@ -15,17 +15,23 @@ class TemporalHealthProbe(HealthProbe):
     """Probes Temporal via the gRPC health check on its service client."""
 
     def __init__(self, get_client: Callable[[], TemporalClientType | None]) -> None:
+        """Use the application's current connection without creating a probe-owned client."""
         self._get_client = get_client
 
     @property
     def name(self) -> str:
+        """Identify this dependency in readiness configuration and responses."""
         return "temporal"
 
     async def check(self) -> DependencyCheck:
+        """Report the actual service health, or skipped until a client is available."""
         client = self._get_client()
         if client is None:
             return DependencyCheck(status=CheckStatus.skipped)
         start = time.perf_counter()
-        await client.service_client.check_health()
+        serving = await client.service_client.check_health()
         latency = (time.perf_counter() - start) * 1000
-        return DependencyCheck(status=CheckStatus.up, latency_ms=round(latency, 2))
+        return DependencyCheck(
+            status=CheckStatus.up if serving else CheckStatus.down,
+            latency_ms=round(latency, 2),
+        )

backend/airweave/adapters/health/tests/test_health_probes.py

@@ -6,7 +6,6 @@
 
 from airweave.schemas.health import CheckStatus
 
-
 # ---------------------------------------------------------------------------
 # PostgresHealthProbe
 # ---------------------------------------------------------------------------
@@ -104,7 +103,7 @@ async def test_client_available_returns_up(self):
         from airweave.adapters.health.temporal import TemporalHealthProbe
 
         client = MagicMock()
-        client.service_client.check_health = AsyncMock()
+        client.service_client.check_health = AsyncMock(return_value=True)
         probe = TemporalHealthProbe(lambda: client)
 
         result = await probe.check()
@@ -114,6 +113,16 @@ async def test_client_available_returns_up(self):
         assert result.latency_ms >= 0
         client.service_client.check_health.assert_awaited_once()
 
+    @pytest.mark.asyncio
+    async def test_server_not_serving_returns_down(self):
+        from airweave.adapters.health.temporal import TemporalHealthProbe
+
+        client = MagicMock()
+        client.service_client.check_health = AsyncMock(return_value=False)
+        result = await TemporalHealthProbe(lambda: client).check()
+
+        assert result.status == CheckStatus.down
+
     @pytest.mark.asyncio
     async def test_client_available_but_unhealthy(self):
         from airweave.adapters.health.temporal import TemporalHealthProbe

backend/airweave/adapters/identity/auth0.py

@@ -17,7 +17,7 @@
     wait_exponential,
 )
 
-from airweave.core.config import settings
+from airweave.core.config import AuthMode, settings
 from airweave.core.logging import logger
 from airweave.core.protocols.identity import (
     IdentityProvider,
@@ -416,7 +416,7 @@ async def add_enabled_connection_to_organization(
 # ---------------------------------------------------------------------------
 
 auth0_management_client: Optional[Auth0ManagementClient] = None
-if settings.AUTH_ENABLED:
+if settings.AUTH_MODE == AuthMode.AUTH0:
     auth0_management_client = Auth0ManagementClient()