diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..3a2033a --- /dev/null +++ b/.editorconfig @@ -0,0 +1,22 @@ +# EditorConfig — consistent diffs matter for auditable pipeline changes. +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space +indent_size = 2 + +[Makefile] +indent_style = tab + +[*.md] +trim_trailing_whitespace = false + +[*.py] +indent_size = 4 + +[*.sql] +indent_size = 2 diff --git a/.envrc.example b/.envrc.example new file mode 100644 index 0000000..763859c --- /dev/null +++ b/.envrc.example @@ -0,0 +1,21 @@ +# .envrc.example — operator environment template (direnv-compatible). +# Copy to .envrc and fill in from Vaultwarden. .envrc is gitignored. +# NEVER commit real values; NEVER paste tokens into issue trackers or chat. +# ADR-0002: analytics never holds production credentials — these point at +# the staging/dev warehouse and scoped read-replica extract roles only. + +# ── Analytics warehouse (dbt profile: warehouse/profiles.yml.example) ───── +export WH_HOST="" +export WH_PORT="5432" +export WH_DB="analytics_dev" +export WH_SCHEMA="dbt" +export WH_USER="" +export WH_PASSWORD="" + +# ── Extraction (read-replica only, least privilege — ADR-0002) ──────────── +export EXTRACT_PG_DSN="" # read-only role on the read replica +export EXTRACT_STRIPE_KEY="" # restricted key: charges/payouts read only # gitleaks:allow + +# ── Quality & freshness gates ───────────────────────────────────────────── +export GE_CONTEXT_ROOT="quality" +# export SLACK_WEBHOOK_URL="" # freshness-monitor alerts (optional) diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6762f0d --- /dev/null +++ b/.gitattributes @@ -0,0 +1,19 @@ +# .gitattributes — diff semantics for the data platform. +# Data artifacts get whole-line semantics; binaries never fake text diffs. + +* text=auto eol=lf + +# Registers, seeds and translation memories are data, not prose +*.csv text eol=lf +*.yml text eol=lf +*.yaml text eol=lf + +# Binary artifacts — no textual diff, no merge guessing +*.png binary +*.jpg binary +*.jpeg binary +*.zip binary + +# Language identification for review tooling +*.md linguist-documentation +*.sql linguist-language SQL diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..ee0a861 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,31 @@ +# CODEOWNERS — routes reviews; the approving-review COUNT is enforced by +# branch protection / rulesets (see CONTRIBUTING.md, risk classes). +# CODEOWNERS alone cannot require N approvers — do not rely on it for that. + +# Default: data platform operators +* @journeyoflife-org/data-operators + +# Catalog & classification — the governance record (§3 DPO gate) +/governance/ @journeyoflife-org/data-operators @journeyoflife-org/compliance + +# Anything touching personal-data machinery — DPO required (§3) +/ingestion/pipelines/pseudonymizer/ @journeyoflife-org/data-operators @journeyoflife-org/compliance +/warehouse/models/marts/compliance/ @journeyoflife-org/data-operators @journeyoflife-org/compliance +/lifecycle/ @journeyoflife-org/data-operators @journeyoflife-org/compliance +/governance/classification.md @journeyoflife-org/compliance +/governance/retention-map.md @journeyoflife-org/compliance + +# Taxonomy is the marketplace domain model — product owner in the loop +/seed/taxonomy/ @journeyoflife-org/data-operators @journeyoflife-org/marketplace-product +/seed/tax/ @journeyoflife-org/data-operators @journeyoflife-org/finance + +# Finance marts +/warehouse/models/marts/finance/ @journeyoflife-org/data-operators @journeyoflife-org/finance + +# Automation touching pipelines needs security in the loop +/.github/workflows/ @journeyoflife-org/data-operators @journeyoflife-org/security +/scripts/ @journeyoflife-org/data-operators @journeyoflife-org/security + +# PII scanning & erasure verification — DPO + security +/scripts/scan-warehouse-pii.py @journeyoflife-org/data-operators @journeyoflife-org/compliance +/scripts/verify-anonymization.py @journeyoflife-org/data-operators @journeyoflife-org/compliance diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..a4521a1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,48 @@ +name: Bug report +description: Defect in a pipeline, model, seed file, or this repo's automation (NOT an incident — use SECURITY.md for those) +labels: ["bug"] +body: + - type: textarea + id: behavior + attributes: + label: Observed vs expected behavior + description: What is wrong, what should be true. Do NOT paste personal data or credentials into this issue. + validations: + required: true + - type: dropdown + id: area + attributes: + label: Affected area + options: + - warehouse-models + - ingestion + - quality + - seed-taxonomy + - governance + - lifecycle + - ml + - ci-cd + - scripts + - docs + - other + validations: + required: true + - type: dropdown + id: severity + attributes: + label: Severity + description: If personal data may have leaked into the warehouse, STOP and follow SECURITY.md instead. + options: + - minor (cosmetic/dashboards) + - moderate (wrong metric) + - major (wrong money/VAT figure) + - critical (pseudonymization or retention defect) + validations: + required: true + - type: textarea + id: repro + attributes: + label: Reproduction / evidence + description: Pipeline/model name, failing test or expectation, date window affected. + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/dataset_request.yml b/.github/ISSUE_TEMPLATE/dataset_request.yml new file mode 100644 index 0000000..85947f3 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/dataset_request.yml @@ -0,0 +1,51 @@ +name: Dataset request +description: Register a new dataset — forces purpose, classification, and retention declaration before any code +labels: ["dataset", "governance"] +body: + - type: input + id: dataset + attributes: + label: Dataset name + description: Stable identifier, e.g. fct_payouts or seed_geo_lockers. + validations: + required: true + - type: textarea + id: purpose + attributes: + label: Purpose (RoPA linkage) + description: > + Why does this dataset exist and which RoPA purpose in + jol-m-compliance covers it? If no personal data is involved, + state that explicitly. + validations: + required: true + - type: dropdown + id: classification + attributes: + label: Classification tier + description: Per governance/classification.md. + options: [PUBLIC, INTERNAL, CONFIDENTIAL, RESTRICTED] + validations: + required: true + - type: dropdown + id: retention + attributes: + label: Retention class + description: Per governance/retention-map.md; "none" only for synthetic/reference data. + options: [short-term (≤ 90 days), operational (≤ 2 years), statutory (≤ 10 years), indefinite-reference, none (synthetic)] + validations: + required: true + - type: textarea + id: ownership + attributes: + label: Owner & steward + description: Business owner and technical steward names/teams — orphan datasets are blocked by catalog-lint. + validations: + required: true + - type: textarea + id: sources + attributes: + label: Sources & lineage + description: Upstream systems and fields; which identifiers are present and how they are pseudonymized. + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..d06c071 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,34 @@ +name: Feature request +description: New capability for the data platform or its automation +labels: ["enhancement"] +body: + - type: textarea + id: capability + attributes: + label: Capability + description: What should the data platform/repository be able to do, and why now? + validations: + required: true + - type: textarea + id: driver + attributes: + label: Business / regulatory driver + description: Which need drives this (VAT-OSS reporting, seller health, DSA transparency, retention duty, cost)? + validations: + required: true + - type: dropdown + id: area + attributes: + label: Target area + options: + - warehouse-models + - ingestion + - quality + - seed-taxonomy + - governance + - lifecycle + - ml + - automation + - other + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/pii_incident.yml b/.github/ISSUE_TEMPLATE/pii_incident.yml new file mode 100644 index 0000000..57b1165 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/pii_incident.yml @@ -0,0 +1,50 @@ +name: PII incident +description: Personal data detected in analytics or committed data — DPO notification. NEVER paste the data itself. +labels: ["pii", "incident", "dpo"] +body: + - type: markdown + attributes: + value: | + **Stop:** do not include the personal data itself, record + contents, or more rows than strictly needed to locate the issue. + This issue carries triage metadata only. Follow + SECURITY.md / docs/runbooks/pii-detected-in-warehouse.md. + - type: dropdown + id: severity + attributes: + label: Severity + options: + - low (synthetic-looking false positive to verify) + - medium (PII-shaped values in a non-restricted dataset) + - high (cleartext identifiers downstream of ingestion) + - critical (direct identifiers in marts or committed to git) + validations: + required: true + - type: input + id: location + attributes: + label: Where detected + description: Table/model/file path and column — no row contents. + validations: + required: true + - type: input + id: detected_at + attributes: + label: Detected at (UTC) + description: ISO 8601 timestamp. + validations: + required: true + - type: dropdown + id: dpo_notified + attributes: + label: DPO notified + options: ["yes", "no — please page"] + validations: + required: true + - type: textarea + id: containment + attributes: + label: Containment actions taken + description: Quarantine, pipeline pause, purge request — what has already been done. + validations: + required: false diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..3cb06e9 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,30 @@ +## Change summary + + + +## Dataset checklist (mandatory — CONTRIBUTING.md) + +- Data classification: [ ] PUBLIC [ ] INTERNAL [ ] CONFIDENTIAL [ ] RESTRICTED +- RoPA purpose reference: +- Retention class: +- Catalog entry + ownership row added for new datasets: [ ] yes / n/a +- No personal data committed (fixtures are synthetic): [ ] confirmed +- No warehouse credentials or connection strings introduced: [ ] confirmed + +## Warehouse impact + +- Models added/changed: +- Staging still pseudonymous (hashed IDs, no names/emails): [ ] confirmed / n/a +- dbt tests cover the change (unique/not_null/relationships + custom): [ ] yes / n/a +- Backfill required: [ ] yes / no — + +## Seed impact + +- Taxonomy change class: [ ] MAJOR (breaking for consumers) [ ] MINOR [ ] PATCH [ ] n/a +- Translations complete for lt/lv/et/en: [ ] yes / n/a +- Fixture regeneration documented (new identities on reseed): [ ] yes / n/a + +## Compliance notes + + diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..d700c3e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,20 @@ +# Dependabot — supply-chain hygiene for dependency ecosystems present in +# this repo. SOC 2 CC7.1: vulnerabilities in tooling are monitored. +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + groups: + actions: + patterns: ["*"] + commit-message: + prefix: "chore(deps)" + + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "monthly" + commit-message: + prefix: "chore(deps)" diff --git a/.github/workflows/ci-raw-schema.sql b/.github/workflows/ci-raw-schema.sql new file mode 100644 index 0000000..615f34a --- /dev/null +++ b/.github/workflows/ci-raw-schema.sql @@ -0,0 +1,58 @@ +-- ci-raw-schema.sql — synthetic source stand-in for dbt-ci ONLY. +-- Mirrors ingestion/contracts/* field allow-lists so staging models can +-- build in CI without ever touching the real read replica. Values are +-- synthetic literals; keep them that way (PII scan runs on this file). + +create schema if not exists raw; + +create table raw.orders ( + id bigint primary key, + buyer_id bigint not null, + seller_id bigint not null, + status text not null, + amount_cents integer not null, + currency char(3) not null, + category_code text not null, + country_code char(2) not null, + vat_rate_pct numeric(5, 2) not null, + created_at timestamptz not null +); + +create table raw.products ( + id bigint primary key, + seller_id bigint not null, + category_code text not null, + title text not null, + price_cents integer not null, + currency char(3) not null, + status text not null, + created_at timestamptz not null +); + +create table raw.users ( + id bigint primary key, + role text not null, + country_code char(2) not null, + consent_marketing boolean not null, + created_at timestamptz not null +); + +insert into raw.orders values + (1, 101, 201, 'paid', 12100, 'EUR', 'vestments', 'LT', 21.00, '2026-08-01 10:00:00+00'), + (2, 102, 201, 'paid', 5900, 'EUR', 'icons', 'LV', 21.00, '2026-08-02 11:00:00+00'), + (3, 103, 202, 'shipped', 3400, 'EUR', 'books', 'EE', 22.00, '2026-08-03 12:00:00+00'), + (4, 104, 202, 'refunded', 9900, 'EUR', 'funeral', 'LT', 21.00, '2026-08-04 13:00:00+00'), + (5, 105, 203, 'paid', 25000, 'EUR', 'services', 'LT', 21.00, '2026-08-05 14:00:00+00'); + +insert into raw.products values + (1001, 201, 'vestments', 'Synthetic chasuble', 12100, 'EUR', 'active', '2026-07-01 09:00:00+00'), + (1002, 201, 'icons', 'Synthetic icon panel', 5900, 'EUR', 'active', '2026-07-02 09:00:00+00'), + (1003, 202, 'books', 'Synthetic hymnal', 3400, 'EUR', 'active', '2026-07-03 09:00:00+00'), + (1004, 203, 'services', 'Synthetic ceremony svc', 25000, 'EUR', 'inactive', '2026-07-04 09:00:00+00'); + +insert into raw.users values + (101, 'buyer', 'LT', true, '2026-01-10 08:00:00+00'), + (102, 'buyer', 'LV', false, '2026-02-11 08:00:00+00'), + (201, 'seller', 'LT', true, '2026-03-12 08:00:00+00'), + (202, 'seller', 'EE', false, '2026-04-13 08:00:00+00'), + (203, 'seller', 'LT', true, '2026-05-14 08:00:00+00'); diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..71620bd --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,44 @@ +# ci.yml — inherited baseline gate: lint + validate everything committed. +# Job name MUST stay "ci": branch protection requires the "ci" status check. +name: ci +on: + pull_request: + push: + branches: [main] +permissions: + contents: read +jobs: + ci: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install gate dependencies + run: pip install --quiet "yamllint>=1.35" "pyyaml>=6.0" "jsonschema>=4.23" + + - name: YAML lint + run: | + YAMLLINT_CFG="{extends: default, rules: {line-length: {max: 140}," + YAMLLINT_CFG="$YAMLLINT_CFG comments: {min-spaces-from-content: 1}," + YAMLLINT_CFG="$YAMLLINT_CFG truthy: disable, document-start: disable," + YAMLLINT_CFG="$YAMLLINT_CFG comments-indentation: disable}}" + yamllint -d "$YAMLLINT_CFG" .github/ .pre-commit-config.yaml qodana.yaml + + - name: Seed & taxonomy schema validation + run: python3 scripts/validate-seed.py + + - name: Catalog lint (no orphan datasets) + run: python3 scripts/catalog-lint.py + + - name: PII pattern scan (committed files) + run: bash scripts/check-personal-data.sh + + - name: Shellcheck operator scripts + run: | + sudo apt-get update -qq && sudo apt-get install -y -qq shellcheck + shellcheck scripts/*.sh diff --git a/.github/workflows/codeql.yml.disabled b/.github/workflows/codeql.yml.disabled new file mode 100644 index 0000000..d3f3a36 --- /dev/null +++ b/.github/workflows/codeql.yml.disabled @@ -0,0 +1,30 @@ +# codeql.yml — inherited baseline, ENABLED for this repository. +# jol-m-data carries Python gate scripts and pipeline code — CodeQL +# analyzes them on every PR. Pin actions to full SHAs before first merge +# to main (fleet supply-chain discipline). +name: codeql +on: + pull_request: + push: + branches: [main] + schedule: + - cron: "17 4 * * 1" +permissions: + contents: read +jobs: + analyze: + runs-on: ubuntu-latest + permissions: + contents: read # checkout on private repos fails without it + actions: read # codeql/analyze fetches workflow-run metadata + security-events: write + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: python + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 diff --git a/.github/workflows/compliance-check.yml b/.github/workflows/compliance-check.yml new file mode 100644 index 0000000..4785cde --- /dev/null +++ b/.github/workflows/compliance-check.yml @@ -0,0 +1,46 @@ +# compliance-check.yml — inherited baseline: governance/policy presence. +# Private repo: the LICENSE is an internal-use notice; every governance +# file required by CONTRIBUTING.md must exist and stay non-empty. +name: compliance-check +on: + pull_request: + push: + branches: [main] +permissions: + contents: read +jobs: + compliance: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Governance files present + run: | + for f in README.md LICENSE SECURITY.md CONTRIBUTING.md CHANGELOG.md \ + .github/CODEOWNERS .github/PULL_REQUEST_TEMPLATE.md; do + [ -s "$f" ] || { echo "MISSING/EMPTY: $f"; exit 1; } + done + + - name: Internal-use notice intact + run: grep -q "INTERNAL USE ONLY" LICENSE + + - name: Data governance record complete + run: | + for f in governance/data-catalog.md governance/classification.md \ + governance/ownership-register.csv governance/retention-map.md; do + [ -s "$f" ] || { echo "MISSING/EMPTY: $f"; exit 1; } + done + + - name: Pseudonymization doctrine documented at the boundary + run: | + grep -qi "pseudonym" ingestion/README.md \ + || { echo "pseudonymization doctrine missing: ingestion/README.md"; exit 1; } + grep -qi "pseudonym" warehouse/README.md \ + || { echo "pseudonymization doctrine missing: warehouse/README.md"; exit 1; } + + - name: Retention machinery declares its policy source + run: grep -q "jol-m-compliance" lifecycle/README.md + + - name: ADR numbering continuous + run: | + ls docs/adr/[0-9]*.md 2>/dev/null | sed 's/.*\///;s/-.*//' | awk 'NR>1 && $1!=p+1 {print "gap in ADR numbering"; exit 1} {p=$1}' diff --git a/.github/workflows/data-quality.yml b/.github/workflows/data-quality.yml new file mode 100644 index 0000000..18650f5 --- /dev/null +++ b/.github/workflows/data-quality.yml @@ -0,0 +1,42 @@ +# data-quality.yml — Great Expectations / dbt tests on the STAGING +# warehouse. Scheduled daily + manual. Requires WH_* secrets; skips +# cleanly until the warehouse environment is provisioned. +name: data-quality +on: + schedule: + - cron: "40 5 * * *" + workflow_dispatch: +permissions: + contents: read +jobs: + quality: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Guard — skip until warehouse secrets exist + if: ${{ vars.WH_HOST == '' }} + run: echo "Warehouse environment not provisioned yet — skipping." + + - name: Set up Python + if: ${{ vars.WH_HOST != '' }} + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install dbt + if: ${{ vars.WH_HOST != '' }} + run: pip install --quiet "dbt-core>=1.8" "dbt-postgres>=1.8" + + - name: Run quality gates (dbt tests on staging + quality suites) + if: ${{ vars.WH_HOST != '' }} + env: + WH_HOST: ${{ vars.WH_HOST }} + WH_PORT: ${{ vars.WH_PORT }} + WH_DB: ${{ vars.WH_DB }} + WH_SCHEMA: ${{ vars.WH_SCHEMA }} + WH_USER: ${{ secrets.WH_USER }} + WH_PASSWORD: ${{ secrets.WH_PASSWORD }} + run: | + cp warehouse/profiles.yml.example warehouse/profiles.yml + make quality diff --git a/.github/workflows/dbt-ci.yml b/.github/workflows/dbt-ci.yml new file mode 100644 index 0000000..3e07e0e --- /dev/null +++ b/.github/workflows/dbt-ci.yml @@ -0,0 +1,82 @@ +# dbt-ci.yml — warehouse gate: sqlfluff -> dbt parse -> dbt build (slim +# CI against an ephemeral warehouse seeded with synthetic fixtures) -> +# dbt test. Never connects to staging/production. +name: dbt-ci +on: + pull_request: + push: + branches: [main] +permissions: + contents: read +jobs: + dbt: + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16-alpine + env: + POSTGRES_USER: dbt_ci + POSTGRES_PASSWORD: dbt_ci + POSTGRES_DB: analytics_ci + ports: ["5432:5432"] + options: >- + --health-cmd "pg_isready -U dbt_ci" + --health-interval 10s --health-timeout 5s --health-retries 5 + env: + WH_HOST: 127.0.0.1 + WH_PORT: "5432" + WH_DB: analytics_ci + WH_SCHEMA: dbt_ci + WH_USER: dbt_ci + WH_PASSWORD: dbt_ci + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install dbt + sqlfluff + run: pip install --quiet "dbt-core>=1.8" "dbt-postgres>=1.8" "sqlfluff>=3.2" "sqlfluff-templater-dbt>=3.2" + + - name: Write CI dbt profile (ephemeral warehouse only) + working-directory: warehouse + run: | + cat > profiles.yml <<'EOF' + jol_m_data: + target: dev + outputs: + dev: + type: postgres + host: "{{ env_var('WH_HOST') }}" + port: "{{ env_var('WH_PORT') | int }}" + user: "{{ env_var('WH_USER') }}" + password: "{{ env_var('WH_PASSWORD') }}" + dbname: "{{ env_var('WH_DB') }}" + schema: "{{ env_var('WH_SCHEMA') }}" + EOF + + - name: Load synthetic raw schema (fixtures, never production data) + run: | + sudo apt-get update -qq && sudo apt-get install -y -qq postgresql-client + psql "postgresql://dbt_ci:dbt_ci@127.0.0.1:5432/analytics_ci" -f .github/workflows/ci-raw-schema.sql + + - name: dbt deps + working-directory: warehouse + run: dbt deps --no-version-check + + - name: sqlfluff lint (dbt templater) + run: sqlfluff lint warehouse/models --config .sqlfluff + + - name: dbt parse + working-directory: warehouse + run: dbt parse --no-version-check + + - name: dbt build (slim CI) + working-directory: warehouse + run: dbt build --no-version-check + + - name: dbt test + working-directory: warehouse + run: dbt test --no-version-check diff --git a/.github/workflows/freshness-monitor.yml b/.github/workflows/freshness-monitor.yml new file mode 100644 index 0000000..bd9ccb7 --- /dev/null +++ b/.github/workflows/freshness-monitor.yml @@ -0,0 +1,55 @@ +# freshness-monitor.yml — daily source freshness vs SLA. A stale +# pipeline means dashboards and VAT-OSS support figures silently lie; +# staleness past the SLA alerts. Requires warehouse secrets; skips +# cleanly until provisioned. +name: freshness-monitor +on: + schedule: + - cron: "20 6 * * *" + workflow_dispatch: +permissions: + contents: read + issues: write +jobs: + freshness: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Guard — skip until warehouse secrets exist + if: ${{ vars.WH_HOST == '' }} + run: echo "Warehouse environment not provisioned yet — skipping." + + - name: Set up Python + if: ${{ vars.WH_HOST != '' }} + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install dependencies + if: ${{ vars.WH_HOST != '' }} + run: pip install --quiet "pyyaml>=6.0" "psycopg2-binary>=2.9" + + - name: Freshness report vs SLA + if: ${{ vars.WH_HOST != '' }} + env: + WH_HOST: ${{ vars.WH_HOST }} + WH_PORT: ${{ vars.WH_PORT }} + WH_DB: ${{ vars.WH_DB }} + WH_SCHEMA: ${{ vars.WH_SCHEMA }} + WH_USER: ${{ secrets.WH_USER }} + WH_PASSWORD: ${{ secrets.WH_PASSWORD }} + run: python3 scripts/freshness-report.py --fail-on-stale + + - name: Open issue on stale sources + if: failure() && vars.WH_HOST != '' + env: + GH_TOKEN: ${{ github.token }} + run: | + gh issue create \ + --title "freshness-monitor: source SLA breach $(date -u +%Y-%m-%d)" \ + --label "data-quality" \ + --body "Daily freshness check failed: at least one source is + older than its SLA. See the run report and follow + docs/runbooks/pipeline-failure.md. + Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" diff --git a/.github/workflows/pii-scan.yml b/.github/workflows/pii-scan.yml new file mode 100644 index 0000000..3b7984f --- /dev/null +++ b/.github/workflows/pii-scan.yml @@ -0,0 +1,52 @@ +# pii-scan.yml — scheduled scan of seed data (always) and the staging +# warehouse (when provisioned) for PII-shaped values. Findings page the +# DPO: an issue labeled pii/incident/dpo is opened and never contains the +# offending values themselves. +name: pii-scan +on: + schedule: + - cron: "10 4 * * *" + workflow_dispatch: +permissions: + contents: read + issues: write +jobs: + scan: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Set up Python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 + with: + python-version: "3.12" + + - name: Install scan dependencies + run: pip install --quiet "pyyaml>=6.0" + + - name: Scan committed seed & fixture data + run: python3 scripts/scan-warehouse-pii.py --local + + - name: Scan staging warehouse (if provisioned) + if: ${{ vars.WH_HOST != '' }} + env: + WH_HOST: ${{ vars.WH_HOST }} + WH_PORT: ${{ vars.WH_PORT }} + WH_DB: ${{ vars.WH_DB }} + WH_SCHEMA: ${{ vars.WH_SCHEMA }} + WH_USER: ${{ secrets.WH_USER }} + WH_PASSWORD: ${{ secrets.WH_PASSWORD }} + run: python3 scripts/scan-warehouse-pii.py --warehouse + + - name: Open DPO issue on failure + if: failure() + env: + GH_TOKEN: ${{ github.token }} + run: | + gh issue create \ + --title "pii-scan: PII-shaped values detected $(date -u +%Y-%m-%d)" \ + --label "pii" \ + --body "Scheduled pii-scan failed. See the run for affected + table/file locations (values are NOT included here by design). + Follow docs/runbooks/pii-detected-in-warehouse.md and notify + the DPO. Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b917043 --- /dev/null +++ b/.gitignore @@ -0,0 +1,61 @@ +# .gitignore — paranoid baseline for the data platform. +# RULE: no production data, no extracts, no credentials ever reach git. +# The repository holds definitions and synthetic data only. + +# ── Data extracts & dumps (NEVER in git) ────────────────────────────────── +*.dump +*.sql.gz +*.parquet +*.sqlite +*.db +# CSV extracts — committed CSV is allow-listed explicitly below +*.csv +!warehouse/seeds/*.csv +!governance/*.csv +!ml/translation-memory/*.csv + +# ── dbt build artifacts & packages ─────────────────────────────────────── +warehouse/target/ +warehouse/dbt_packages/ +warehouse/logs/ +dbt_packages/ +target/ + +# ── Great Expectations runtime ─────────────────────────────────────────── +quality/uncommitted/ +great_expectations/uncommitted/ + +# ── Secrets & key material ─────────────────────────────────────────────── +*.pem +*.key +*.p12 +*.pfx +*.gpg +*.agekey +id_rsa* +id_ed25519* +profiles.yml +!profiles.yml.example + +# ── Environment & credential files ─────────────────────────────────────── +.env +.env.* +!.envrc.example +.envrc + +# ── Working artifacts ──────────────────────────────────────────────────── +*.tmp +*~ +__pycache__/ +*.pyc + +# ── Editors / OS ───────────────────────────────────────────────────────── +.idea/ +.vscode/ +*.swp +.DS_Store + +# ── Python virtual environments (tooling harness only) ─────────────────── +.venv/ +venv/ +warehouse/.user.yml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..14cb151 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,72 @@ +# Pre-commit hooks — local gate before CI. Install once: pre-commit install +# Data-specific: PII pattern scan + sqlfluff + dbt parse run alongside +# secret scanning. Hook revisions are PINNED; bump via review, never ad hoc. +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: trailing-whitespace + exclude: '\.md$' + - id: end-of-file-fixer + - id: check-yaml + args: [--allow-multiple-documents] + exclude: '^warehouse/.*\.(yml|yaml)$' # dbt YAML validated by dbt parse + - id: check-json + - id: check-merge-conflict + - id: detect-private-key + - id: check-added-large-files + args: ['--maxkb=512'] + + - repo: https://github.com/gitleaks/gitleaks + rev: v8.21.2 + hooks: + - id: gitleaks # warehouse credentials must never reach git history + + # sqlfluff with the dbt templater needs warehouse/profiles.yml (never + # committed) + sqlfluff-templater-dbt; the hard gate is CI dbt-ci.yml, + # which builds an ephemeral profile. Locally the hook degrades to a + # loud skip, mirroring the dbt-parse hook below. + + - repo: local + hooks: + - id: check-csv + name: CSV structural gate (registers + seeds) + entry: python3 scripts/check-csv.py + language: system + files: '\.csv$' + - id: sqlfluff-lint + name: sqlfluff warehouse lint (dbt templater; dbt-ci is the hard gate) + entry: >- + bash -c 'if [ ! -f warehouse/profiles.yml ]; then + echo "sqlfluff skipped — warehouse/profiles.yml absent (credential-free checkout); dbt-ci is the hard gate"; + exit 0; fi; + command -v sqlfluff >/dev/null || { + echo "sqlfluff skipped — sqlfluff not installed locally"; exit 0; }; + sqlfluff lint warehouse/models --config .sqlfluff' + language: system + files: '^warehouse/.*\.sql$' + pass_filenames: false + - id: pii-pattern-scan + name: PII pattern scan (seed + tracked text tripwire) + entry: bash scripts/check-personal-data.sh + language: system + # Flags Baltic national IDs, IBANs, bulk email lists, and + # PII-shaped values in tracked files. Tripwire only — fixtures + # must be synthetic by construction (CONTRIBUTING.md). + files: '\.(md|csv|txt|yml|yaml|sql)$' + pass_filenames: false + - id: seed-schema-validate + name: seed/taxonomy JSON Schema validation + entry: python3 scripts/validate-seed.py + language: system + files: '^(seed|governance)/' + pass_filenames: false + - id: dbt-parse + name: dbt parse (warehouse config sanity) + entry: >- + bash -c 'command -v dbt >/dev/null && + dbt parse --project-dir warehouse --profiles-dir warehouse + --no-version-check || echo "dbt not installed — skipped"' + language: system + files: '^warehouse/' + pass_filenames: false diff --git a/.sqlfluff b/.sqlfluff new file mode 100644 index 0000000..6982e01 --- /dev/null +++ b/.sqlfluff @@ -0,0 +1,27 @@ +# sqlfluff — warehouse SQL style. Lint runs in pre-commit and dbt-ci. +# Keep rules conservative: dbt-templated SQL must lint clean without +# disabling safety rules. + +[sqlfluff] +dialect = postgres +templater = dbt +max_line_length = 100 +exclude_rules = LT05 + +[sqlfluff:templater:dbt] +project_dir = warehouse +profiles_dir = warehouse + +[sqlfluff:indentation] +# Models are written with 4-space indents (dbt convention in this repo); +# keep the linter aligned with the code, not the other way round. +tab_space_size = 4 + +[sqlfluff:rules:capitalisation.keywords] +capitalisation_policy = lower + +[sqlfluff:rules:capitalisation.identifiers] +capitalisation_policy = lower + +[sqlfluff:rules:capitalisation.functions] +capitalisation_policy = lower diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..8018659 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,33 @@ +# Changelog — jol-m-data + +All notable changes to this data-platform repository are documented +here. Format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/); +commits follow Conventional Commits. Never rewrite released entries. + +## [Unreleased] + +### Added + +- Repository scaffold: governance (catalog, classification, ownership + register, retention map, lineage), seed (taxonomy + lt/lv/et/en/ru + translations, geo, tax, synthetic fixtures, JSON Schema validators), + warehouse (dbt project with staging/intermediate/marts, custom tests, + macros, static seeds), ingestion (postgres/stripe extracts, + pseudonymizer, schema contracts), quality (expectations, anomaly + rules, scorecards), lifecycle (retention jobs, erasure verification, + legal holds), synthetic (generators, PII canaries, regression goldens), + ml (embeddings, evaluation, translation memory), docs, scripts. +- Root compliance baseline: README (prime directive, access tiers), + LICENSE (internal use + synthetic-data exception), SECURITY.md + (PII incidents = highest severity, DPO first), CONTRIBUTING.md (seed + doctrine, warehouse doctrine, governance mandate). +- CI/CD: ci + compliance-check gates, dbt-ci (sqlfluff → parse → slim + build → tests), data-quality, pii-scan (scheduled), freshness-monitor. +- Pre-commit baseline + gitleaks + sqlfluff + PII pattern scan + dbt + parse check. +- Scripts: validate-seed, scan-warehouse-pii, catalog-lint, + freshness-report, verify-anonymization, check-personal-data. +- ADR-0001: pseudonymize at ingestion; ADR-0002: no production + credentials in analytics. +- Metrics dictionary (GMV, take-rate, active seller — one definition + each), runbooks (pipeline-failure, pii-detected, restore-analytics). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..826416e --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,76 @@ +# Contributing — jol-m-data + +Data engineers, analysts, and the DPO. Every merge here changes either +the dataset governance record or the pipeline that transforms personal +data — both are auditable surfaces. Commits follow Conventional Commits; +notable changes land in `CHANGELOG.md`. + +## Seed doctrine (seed/) + +1. **Synthetic only.** Fixtures are generated by faker-seeded generators + (`synthetic/generators/`); no real person, company, or address may be + committed. Regeneration with a new seed must produce new identities. +2. **Deterministic & versioned.** Seed files are committed artifacts + with schema validation (`seed/validators/`); CI blocks malformed + taxonomy. Changes to taxonomy are MINOR/MAJOR changes for consumers + (`jol-m-marketplace`) — describe impact in the PR. +3. **Translations are complete.** Category names exist in lt/lv/et/en + (ru reserved); a missing translation is a CI failure, not a TODO. + +## Warehouse doctrine (warehouse/) + +1. **PII is stripped in staging.** Staging models hash identifiers and + drop names/emails — a model that re-introduces an identifier is a + defect of the highest class. +2. **Every model is documented.** `models/**/_models.yml` entries with + owner + tests (unique, not_null, relationships) are mandatory; + undocumented models fail `dbt-ci`. +3. **Tests encode the invariants.** Custom tests (`warehouse/tests/`) + guard no-null-pii-columns, id-hash-format, eur-only, vat-rate-bounds. +4. **No production credentials.** Warehouse access is via scoped + service accounts from env vars (ADR-0002); nothing in-repo can reach + production, and extraction is read-replica only. + +## Governance is mandatory + +- Every new dataset lands with: a `governance/data-catalog.md` entry, + an owner + steward row in `ownership-register.csv`, a classification + tier, a RoPA purpose reference, and a retention class. `catalog-lint` + blocks orphans. +- Retention class changes require DPO review (CODEOWNERS routes it). +- Policy text lives in `jol-m-compliance`; this repo implements it. + +## Personal data — absolute rules + +- Never commit personal data, real or "temporarily". Synthetic only. +- Never commit warehouse credentials, connection strings with + passwords, or API keys — gitleaks enforces. +- `scripts/check-personal-data.sh` and the PII pattern scan are + tripwires, not a license: passing them does not make a commit lawful. + +## Workflow + +1. **Issue first.** `dataset_request` for new datasets, `pii_incident` + for PII findings (never put the data itself in the issue). +2. **Branch per change.** One dataset/pipeline concern per PR. +3. **CI is a merge gate.** `ci`, `compliance-check`, `dbt-ci` (parse + + slim build + tests) must be green. +4. **CODEOWNERS routing is binding.** DPO review on anything touching + RESTRICTED paths, lifecycle jobs, and compliance marts. + +**Solo-era operation (current):** the org operates with a single data +operator, so human review gates ride on automated checks + CODEOWNERS +routing until the second operator onboards. Tracked deviation, not an +exemption. + +## Change-risk classes + +| Class | Examples | Gate | +|-------|----------|------| +| Low | Docs, reserved scaffolding, dashboards definitions | 1 review | +| Med | New staging model, seed translation additions | 1 review + CI | +| High | New dataset (catalog entry), mart logic, retention job change | CI + owner review; DPO if RESTRICTED | +| Crit | Pseudonymizer change, erasure/legal-hold logic, classification tier change | DPO + data platform owner; adversarial verification | + +If you cannot say which retention class the data falls under and how it +is erased, the change is not ready. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..b450d17 --- /dev/null +++ b/LICENSE @@ -0,0 +1,26 @@ +Copyright (c) 2026 Journey of Life. All rights reserved. + +This repository is PRIVATE. It contains the analytics warehouse +definitions, ingestion pipelines, dataset governance records, and +reference data of the Journey of Life marketplace. It is designed so +that no personal data in cleartext is ever committed — but dataset +structures, business logic, and commercial references are confidential. + +INTERNAL USE ONLY. No part of this repository may be copied, distributed, +published, mirrored, or incorporated into any public repository, artifact, +or derivative work without prior written authorization from Journey of +Life. + +EXCEPTION — synthetic seed data: files under seed/ and synthetic/ are +100% synthetic by construction (faker-seeded, no real persons). They may +be used in demos and staging environments of authorized fleet +repositories. They may not be republished externally as if they were +real marketplace data. + +Open-source posture for public marketplace code lives in the respective +public repositories (see jol-m-marketplace). This notice governs +internal custody of data-platform artifacts only. + +Unauthorized disclosure of warehouse credentials, seller/buyer +aggregates, or any personal data found in this repository is treated as +a security incident of the highest severity class (see SECURITY.md). diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..82491ef --- /dev/null +++ b/Makefile @@ -0,0 +1,39 @@ +# jol-m-data — data operator hygiene targets. +# Credential-free targets (check, seed-validate) must always pass locally. +# Warehouse targets (dbt-build, quality) require .envrc vars — ADR-0002. + +SHELL := /bin/bash +PY := python3 +DBT := dbt --project-dir warehouse --profiles-dir warehouse + +.PHONY: help check seed-validate catalog-lint dbt-parse dbt-build quality anonymize-verify lint-docs + +help: ## Show targets + @grep -E '^[a-zA-Z_-]+:.*?## ' $(MAKEFILE_LIST) | awk 'BEGIN{FS=":.*?## "}{printf " %-16s %s\n", $$1, $$2}' + +check: seed-validate catalog-lint ## Gate = seed schema + catalog integrity + PII tripwire (mirrors CI) + bash scripts/check-personal-data.sh + +seed-validate: ## JSON Schema validation of every seed/taxonomy file + $(PY) scripts/validate-seed.py + +catalog-lint: ## Every dataset registered, owned, classified, retention-mapped + $(PY) scripts/catalog-lint.py + +dbt-parse: ## dbt parse — config/compile sanity, no warehouse connection + $(DBT) parse --no-version-check || echo "dbt not installed — skipped" + +dbt-build: ## dbt build (slim) against the dev warehouse profile (needs env) + $(DBT) build --no-version-check + +quality: ## Great Expectations / dbt tests on the staging warehouse (needs env) + $(DBT) test --select staging,quality --no-version-check + +anonymize-verify: ## Adversarial re-identification sampler (lifecycle) + $(PY) scripts/verify-anonymization.py + +lint-docs: ## Markdown/YAML hygiene + @command -v yamllint >/dev/null && yamllint .github/ .pre-commit-config.yaml qodana.yaml \ + || echo "yamllint not installed — skipped" + @command -v markdownlint >/dev/null && markdownlint '**/*.md' \ + || echo "markdownlint not installed — skipped" diff --git a/QODER.md b/QODER.md new file mode 100644 index 0000000..c339220 --- /dev/null +++ b/QODER.md @@ -0,0 +1,110 @@ +# QODER.md + +Behavioral guidelines to reduce common LLM coding mistakes when using Qoder in PyCharm. Merge with project-specific instructions as needed. + +**Tradeoff:** These guidelines bias toward caution over speed. For trivial tasks, use judgment. + +## 1. Think Before Coding + +**Don't assume. Don't hide confusion. Surface tradeoffs.** + +Before implementing: +- State your assumptions explicitly. If uncertain, ask. +- If multiple interpretations exist, present them - don't pick silently. +- If a simpler approach exists, say so. Push back when warranted. +- If something is unclear, stop. Name what's confusing. Ask. + +## 2. Simplicity First + +**Minimum code that solves the problem. Nothing speculative.** + +- No features beyond what was asked. +- No abstractions for single-use code. +- No "flexibility" or "configurability" that wasn't requested. +- No error handling for impossible scenarios. +- If you write 200 lines and it could be 50, rewrite it. + +Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify. + +## 3. Surgical Changes + +**Touch only what you must. Clean up only your own mess.** + +When editing existing code: +- Don't "improve" adjacent code, comments, or formatting. +- Don't refactor things that aren't broken. +- Match existing style, even if you'd do it differently. +- If you notice unrelated dead code, mention it - don't delete it. +- Prefer PyCharm's built-in refactoring tools (Rename, Extract, Move, etc.) over manual text manipulation when the IDE can do it safely. + +When your changes create orphans: +- Remove imports/variables/functions that YOUR changes made unused. +- Don't remove pre-existing dead code unless asked. + +The test: Every changed line should trace directly to the user's request. + +## 4. Goal-Driven Execution + +**Define success criteria. Loop until verified.** + +Transform tasks into verifiable goals: +- "Add validation" → "Write tests for invalid inputs, then make them pass" +- "Fix the bug" → "Write a test that reproduces it, then make it pass" +- "Refactor X" → "Ensure tests pass before and after" + +For multi-step tasks, state a brief plan: +``` +1. [Step] → verify: [check] +2. [Step] → verify: [check] +3. [Step] → verify: [check] +``` + +Strong success criteria let you loop independently. Weak criteria ("make it work") require constant clarification. + +--- + +**These guidelines are working if:** fewer unnecessary changes in diffs, fewer rewrites due to overcomplication, and clarifying questions happen before implementation rather than after mistakes. + +--- + +## Project-Specific Guidelines — jol-m-data + +This repository governs pseudonymous analytics over marketplace data; +every merge either changes the governance record or transforms data that +was personal. The full rules live in `CONTRIBUTING.md`; these constrain +AI-assisted changes: + +### Personal data boundaries + +- Never generate, fetch, or commit personal data — real or plausibly + real. Fixtures come from `synthetic/generators/` and are faker-seeded. +- Never re-introduce identifiers into warehouse models: staging hashes + IDs and drops names/emails; any model that joins them back is a + critical defect. +- Compliance marts stay aggregates-only; never emit per-subject rows. + +### Pseudonymization & retention + +- Changes to `ingestion/pipelines/pseudonymizer/` or + `lifecycle/` are critical-risk: require DPO review and + `make anonymize-verify` evidence before merge. +- Retention jobs execute the schedule defined in + `governance/retention-map.md`; policy text lives in `jol-m-compliance` + — do not duplicate or reword policy here. +- Legal holds suspend retention; they never delete. + +### Governance gates + +- New datasets must land with catalog entry + ownership row + + classification + retention class; `scripts/catalog-lint.py` enforces. +- `make check` (seed schema + catalog lint + PII tripwire) must pass + before proposing a change as complete. +- Never suggest bypassing pre-commit (`--no-verify`) or loosening the + PII scan patterns to make a commit pass. + +### Secrets & access + +- No warehouse credentials, connection strings, or API keys in any + file; `profiles.yml.example`/`.envrc.example` document env vars only. +- Extraction connects to the read replica with a read-only role — never + propose production credentials for analytics (ADR-0002). diff --git a/README.md b/README.md index 12afe3b..6d07ff7 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,92 @@ -# jol-m-data -Marketplace data schemas and migrations (no production data in git). +# jol-m-data — Data Platform Repository + +**Private** repository for the Journey of Life marketplace data function +(`jol-m-*` fleet). Seed & reference data, the analytics warehouse (dbt), +ingestion contracts, data quality, retention machinery, synthetic data, +and ML dataset governance live here. + +> **Prime directive:** personal data never lands in the warehouse in +> cleartext. Ingestion pseudonymizes at the boundary (ADR-0001); the +> warehouse models aggregates and pseudonymous keys only. If you find +> cleartext PII anywhere downstream of ingestion, that is an incident — +> see [docs/runbooks/pii-detected-in-warehouse.md](docs/runbooks/pii-detected-in-warehouse.md). + +## What this repository is + +- **Governance catalog is the heart** (`governance/`): every dataset is + registered, owned, classified, and retention-mapped. Orphan datasets + are blocked by CI (`catalog-lint.py`). +- **Seed & reference data** (`seed/`): the marketplace taxonomy + (categories, attributes, translations LT/LV/EE/EN/RU), geo/tax + references, and 100% synthetic fixtures — the knowledge spine that + `jol-m-marketplace` and demo environments load. +- **Analytics warehouse** (`warehouse/`): dbt project; staging strips + identifiers, marts serve finance (VAT OSS), marketplace, and + compliance analytics. EU region only. +- **Retention & anonymization as code** (`lifecycle/`): executes the + retention schedule whose policy text lives in `jol-m-compliance`. + +## Access tiers & data classification + +Four tiers, handling rules in [governance/classification.md](governance/classification.md): + +| Tier | Meaning | Examples here | +|------|---------|---------------| +| PUBLIC | publishable | taxonomy structure (via marketplace), docs in public artifacts | +| INTERNAL | any org member | dashboards definitions, dbt models, runbooks | +| CONFIDENTIAL | data/finance/compliance roles | commission models, seller aggregates, VAT filings support | +| RESTRICTED | personal data / DPO-controlled | erasure logs (aggregates), consent metrics, legal-hold flags | + +## Ownership + +| Role | Contact | Owns | +|------|---------|------| +| Data platform owner | TBD — fill on onboarding | `warehouse/`, `ingestion/`, `scripts/` | +| DPO (`jol-m-compliance`) | TBD | anything touching RESTRICTED data; CODEOWNERS gate (§3) | +| Marketplace product owner | TBD | `seed/taxonomy/`, `seed/fixtures/` | +| Finance | TBD | `warehouse/models/marts/finance/`, `seed/tax/` | + +Cross-repo boundaries: retention **policy** and GDPR evidence (RoPA, +DSAR logs) live in `jol-m-compliance`; this repo executes the policy as +code. Legal texts and glossaries sync from `jol-m-legal` +(`ml/translation-memory/`). Production backups are **not** here — +analytics is rebuildable from sources ([docs/runbooks/restore-analytics.md](docs/runbooks/restore-analytics.md)). + +## Repository map + +| Path | Purpose | +|------|---------| +| `governance/` | Catalog, classification, ownership register, retention map, lineage | +| `seed/` | Taxonomy + translations, geo/tax references, synthetic fixtures, JSON Schema validators | +| `warehouse/` | dbt project: staging (PII stripped), intermediate, marts, tests, macros, seeds | +| `ingestion/` | Extract pipelines, the pseudonymizer, schema contracts (field allow-lists) | +| `quality/` | Great Expectations suites, anomaly rules, weekly scorecards | +| `lifecycle/` | Retention jobs, erasure verification, legal holds, adversarial re-ID tests | +| `synthetic/` | Generators (lt/lv/et-aware), PII canaries, golden regression datasets | +| `ml/` | Embedding builds, relevance eval sets, translation memory | +| `docs/` | Architecture, ADRs, metrics dictionary, runbooks, DPIA template | +| `scripts/` | Seed validation, PII scan, catalog lint, freshness report, anonymization verify | +| `audits/` | Internal audit records for this repository | + +## Quickstart + +```bash +python3 -m venv .venv && . .venv/bin/activate && pip install -e . +make check # seed schema validation + catalog lint + PII tripwire (no credentials) +make seed-validate # JSON Schema validation of every seed/taxonomy file +make quality # data-quality expectations (staging warehouse, needs env) +make dbt-build # dbt build against the dev warehouse profile (needs env) +make anonymize-verify # adversarial re-identification sampler (lifecycle) +``` + +Warehouse/ingestion targets require the environment variables from +[.envrc.example](.envrc.example) (never real credentials in-repo — +ADR-0002). `make check` runs without any credentials. + +## Change discipline + +Every dataset change carries classification, RoPA purpose, and retention +class (PR template enforces). Commits follow Conventional Commits; +notable changes land in `CHANGELOG.md`. PII detected in committed data +or in the warehouse is a severity-1 class incident with DPO notification +([SECURITY.md](SECURITY.md)). diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 0000000..268fc7e --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,52 @@ +# Security Policy — jol-m-data + +## Severity doctrine + +**Any incident involving personal data in this repository or in the +warehouse it manages is the highest severity class.** The entire design +of this repository (pseudonymize-at-ingestion, field allow-lists, +aggregate-only compliance marts) exists so that a warehouse compromise +cannot become a personal-data breach. Finding cleartext PII downstream +of ingestion means that boundary failed — treat it as such. + +Order of notification: + +1. DPO (`jol-m-compliance`) — immediately; GDPR 72h assessment clock is + the DPO's, and the PII runbook is DPO-controlled. +2. Data platform owner (repo owner) — immediately, in parallel. +3. Security function / incident commander (`jol-m-infrastructure` + runbooks). +4. General counsel (`jol-m-legal`) if regulatory notification becomes + likely. + +## Reporting + +Report vulnerabilities and data incidents privately. **Never** open a +public issue, paste credentials into chat, or describe affected records +in a tracker. + +- Security function contact: per `jol-m-infrastructure/SECURITY.md`. +- DPO contact: per `jol-m-compliance` (RoPA point of contact). +- Use the `pii_incident` issue template only for **non-sensitive** + metadata (where, when, which pipeline) — never the data itself. + +## Scope-specific risks + +| Risk | Control | +|------|---------| +| Warehouse credentials committed | gitleaks pre-commit + CI; `.env*` gitignored; profiles.yml.example documents env vars only | +| Cleartext PII lands in warehouse | pseudonymizer at ingestion boundary (unit-tested); field allow-list contracts; scheduled `pii-scan.yml` | +| Cleartext PII committed to seed/fixtures | fixtures are faker-seeded synthetic; pre-commit PII pattern scan; `validate-seed.py` | +| Re-identification via join of marts | compliance marts are aggregates only; `verify-anonymization.py` adversarial sampling | +| Retention bypassed | retention jobs driven by `governance/retention-map.md`; legal holds suspend, not delete | +| SAQ-A boundary erosion | Stripe extract takes charge/payout metadata only — never PAN or full card data | + +## Do not + +- Do not connect extraction jobs to production with write privileges — + read-replica, read-only role, least privilege (ADR-0002). +- Do not copy production dumps, `*.dump`, or CSV extracts into this + repository — they are gitignored for a reason. +- Do not "fix" a PII finding by quietly deleting history; follow + [docs/runbooks/pii-detected-in-warehouse.md](docs/runbooks/pii-detected-in-warehouse.md) + so the DPO can assess notification duty. diff --git a/audits/README.md b/audits/README.md new file mode 100644 index 0000000..3ac5816 --- /dev/null +++ b/audits/README.md @@ -0,0 +1,19 @@ +# Audits — jol-m-data + +Internal audit records for this repository. + +- `internal/` — audit workpapers, findings, and remediation tracking + for audits scoped to this repository (catalog reconciliation, + retention execution evidence reviews, scanner effectiveness). +- Cross-repo audit evidence (GDPR records, SOC 2 evidence) is owned by + `jol-m-compliance`; this directory holds only what was produced by or + about this repository's controls. + +Rules: + +1. Audit records are append-only; findings are closed with evidence, + not deleted. +2. No personal data in audit workpapers — counts, key prefixes, and + references only. +3. Warehouse rebuild proofs and adversarial verification results land + here with a pointer to the durable copy in `jol-m-compliance`. diff --git a/docs/DPIA-template.md b/docs/DPIA-template.md new file mode 100644 index 0000000..d8f1e70 --- /dev/null +++ b/docs/DPIA-template.md @@ -0,0 +1,49 @@ +# DPIA template — GDPR Art. 35 (template-inherited, data-platform copy) + +**When required:** any change introducing or altering processing of personal +data in the warehouse or pipelines (new source, new field on an allow-list, +new mart over pseudonymous keys, new retention class, new ML dataset). Attach +the completed DPIA to the change request BEFORE implementation. + +--- + +## 1. Processing description + +- Purpose of processing (RoPA reference in `jol-m-compliance`): +- Categories of data subjects (buyers/sellers/visitors): +- Categories of personal data (flag special categories Art. 9): +- Data flows (source → pseudonymizer → landing → models → consumers): +- Legal basis (Art. 6): +- Retention period & deletion mechanism (retention class + job): + +## 2. Necessity & proportionality + +- Why is each field on the ingestion allow-list necessary? +- Minimization measures (hash at boundary, generalization, aggregation): +- Alternatives considered and rejected: + +## 3. Risk assessment (to rights & freedoms) + +| Risk scenario | Likelihood | Impact | Mitigation | +|---------------|------------|--------|------------| +| re-identification via mart join | | | aggregates-only rule, adversarial verification | +| cleartext PII landing | | | fail-closed pseudonymizer, pii-scan | +| retention bypass | | | legal-hold aware jobs, DPO release | + +## 4. Technical & organizational measures + +- Encryption at rest / in transit (keys per `jol-m-infrastructure` custody): +- Access control (warehouse roles, review cadence): +- Residency: EU-only (no cross-region replication): +- Breach detection & notification path (72h, Art. 33 — DPO owns the clock): + +## 5. Processor/sub-processor check + +- New third parties introduced (embedding providers, label platforms)? + List, DPA status, location: + +## 6. Sign-off + +- DPO: +- Data platform owner: +- Date: diff --git a/docs/adr/0001-pseudonymize-at-ingestion.md b/docs/adr/0001-pseudonymize-at-ingestion.md new file mode 100644 index 0000000..e2a1206 --- /dev/null +++ b/docs/adr/0001-pseudonymize-at-ingestion.md @@ -0,0 +1,37 @@ +# ADR-0001: Pseudonymize at ingestion + +- Status: Accepted +- Date: 2026-08-15 +- Deciders: data platform owner, DPO + +## Context + +The marketplace needs analytics (seller health, VAT-OSS support, +search relevance) over data that is personal at source. GDPR Art. 25 +(data protection by design) and Art. 32 (security of processing) +require minimization and pseudonymization; a warehouse holding +cleartext identities would make every analytics incident a personal +data breach and expand DSAR/erasure surface to every mart. + +## Decision + +Personal identifiers are pseudonymized **at the ingestion boundary** — +before landing in the analytics warehouse: + +1. Numeric subject ids are salted hashes (`hash_id`); free-text + identifiers are dropped or hashed (`pseudonymizer/rules.yml`, + fail-closed default: drop). +2. Names, emails, phones, registry codes never leave the source + (contract allow-lists in `ingestion/contracts/`). +3. The hash salt is held outside the warehouse; re-identification + requires collusion across system boundaries. + +## Consequences + +- The warehouse cannot answer "who" questions — by design; identity + questions route back to the product boundary (DSAR process). +- Joins across marts stay possible (stable pseudonymous keys). +- Every new source field needs an explicit rule/allow-list entry — + friction by intent; the alternative is silent PII landing. +- Erasure becomes verifiable: `lifecycle/anonymization/` checks that + nothing attributable to an erased subject remains. diff --git a/docs/adr/0002-no-production-credentials-in-analytics.md b/docs/adr/0002-no-production-credentials-in-analytics.md new file mode 100644 index 0000000..da6be7c --- /dev/null +++ b/docs/adr/0002-no-production-credentials-in-analytics.md @@ -0,0 +1,37 @@ +# ADR-0002: No production credentials in analytics + +- Status: Accepted +- Date: 2026-08-15 +- Deciders: data platform owner, security function + +## Context + +Analytics pipelines historically become the weakest credential +boundary: dashboards, notebooks, and ETL jobs accumulate broad +database access. A compromised analytics credential must not yield a +production path. SOC 2 CC6 (logical access) and least-privilege +practice require separation. + +## Decision + +1. Analytics never holds production credentials. The warehouse and + notebooks connect to a dedicated analytics database in the EU + region; credentials come from environment secrets only + (`profiles.yml.example`/`.envrc.example` document names, never + values). +2. Extraction connects to the **read replica** with a read-only, + table-scoped role (`ingestion/pipelines/postgres_extract/extract-role.sql`); + no write path to production exists from this repository's tooling. +3. Third-party extracts (Stripe) use restricted-scope keys + (`ingestion/contracts/stripe.yml`). +4. CI never connects to staging/production — it builds an ephemeral + warehouse from the synthetic raw schema. + +## Consequences + +- Analytics cannot "just fix" production data incidents — correct + behavior; production changes route through `jol-m-marketplace` + change management. +- Two credential custody chains exist (infra-owned); rotation follows + `jol-m-infrastructure` runbooks. +- A leaked analytics credential degrades to analytics scope only. diff --git a/docs/adr/README.md b/docs/adr/README.md new file mode 100644 index 0000000..5e4f7ee --- /dev/null +++ b/docs/adr/README.md @@ -0,0 +1,10 @@ +# Architecture Decision Records + +Numbering is continuous (enforced by `compliance-check.yml`). ADRs are +append-only: superseded ADRs keep their text and gain a "Superseded by" +line. Evidence copy goes to `jol-m-compliance`. + +| ADR | Title | Status | +|------|--------------------------------------|----------| +| 0001 | Pseudonymize at ingestion | Accepted | +| 0002 | No production credentials in analytics | Accepted | diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..32ac3b0 --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,57 @@ +# Architecture — jol-m-data + +Template-inherited baseline, extended with the platform data flow. + +## Position in the fleet + +| Repo | Relationship | +|------|--------------| +| `jol-m-marketplace` | Source of truth (production); this repo reads a pseudonymized copy, never writes | +| `jol-m-compliance` | Retention policy, RoPA, GDPR evidence; this repo executes policy as code | +| `jol-m-legal` | Legal glossary (translation-memory sync), DSA transparency data requests | +| `jol-m-infrastructure` | Warehouse hosting, secrets, network planes; runs extract-role grants | + +## Data flow (prod → pseudonymized → marts) + +```mermaid +flowchart LR + subgraph PROD [jol-m-marketplace production] + P[(orders/products/users)] --> R[(read replica)] + end + subgraph BOUNDARY [pseudonymization boundary — ADR-0001] + R --> EX[postgres_extract\nread-only role] + S[Stripe API] --> SE[stripe_extract\nmetadata only, never PAN] + EX --> PZ[pseudonymizer\nfail-closed rules] + SE --> PZ + end + PZ --> RAW[(warehouse raw\nEU region)] + RAW --> STG[staging — PII stripped] + STG --> INT[intermediate] + INT --> MARTS[marts: core / finance / marketplace / compliance] + MARTS --> C1[dashboards] + MARTS --> C2[VAT-OSS filing support] + MARTS --> C3[ai_service_app embeddings] +``` + +The boundary is the design: nothing downstream of the pseudonymizer can +re-identify a subject without external collusion (salt held outside the +warehouse, keys held in the product boundary). + +## Key components + +- **Governance catalog** (`governance/`) — registration, ownership, + classification, retention for every dataset; CI blocks orphans. +- **Seed & taxonomy** (`seed/`) — the marketplace domain model and + synthetic fixtures; schema-validated. +- **Warehouse** (`warehouse/`) — dbt models from staging to marts; + custom tests encode pseudonymization/EUR/VAT invariants. +- **Lifecycle machinery** (`lifecycle/`) — retention jobs, erasure + verification, legal holds, adversarial re-identification sampling. + +## Constraints + +1. EU region only for warehouse storage and processing. +2. No production credentials in analytics (ADR-0002). +3. Compliance marts are aggregates only. +4. Analytics is rebuildable from sources; production backups live in + `jol-m-infrastructure`, not here. diff --git a/docs/metrics-dictionary.md b/docs/metrics-dictionary.md new file mode 100644 index 0000000..5b132eb --- /dev/null +++ b/docs/metrics-dictionary.md @@ -0,0 +1,26 @@ +# Metrics Dictionary + +Business definitions — **ONE definition each, used everywhere**. A +dashboard or model that deviates from these definitions is a bug. +Metric changes are PRs against this file first; the SQL follows. + +| Metric | Definition | Implementation | Owner | +|--------|------------|----------------|-------| +| GMV | Gross merchandise value: sum of `amount_eur` over orders with status in (paid, shipped, delivered). Refunds reduce GMV at refund event time. | `fct_orders` filtered by `is_revenue` | data-platform | +| Net GMV | GMV minus refunded amounts within the same reporting period. | `fct_orders` | data-platform | +| Take rate | Commission earned ÷ GMV over the same period, per seller cohort or platform-wide. | `fct_commission` ÷ `fct_orders` | finance | +| Active seller | Seller with ≥ 1 order in (paid, shipped, delivered) in the trailing 90 days. | `int_seller_lifecycle.lifecycle_stage = 'active'` | marketplace-product | +| Conversion | Orders ÷ distinct listing views (search/telemetry source pending — scaffold). | `listing_funnel` proxy until search telemetry lands | marketplace-product | +| Refund rate | Refunded orders ÷ total orders per seller/period. | `seller_health.refund_rate` | marketplace-product | +| Consent rate | Marketing opt-ins ÷ accounts in cohort month, per country. Aggregate only. | `consent_rates` | dpo | +| DSR SLA adherence | DSARs answered within statutory deadline ÷ DSARs received, per month. Aggregate only. | `dsr_sla_metrics` | dpo | +| OSS-support net | Net taxable amount per destination country/quarter/rate (reporting support, not the filing). | `fct_vat_oss` | finance | + +## Rules + +1. New metrics require: definition here, implementation reference, + owner — and a catalog entry if backed by a new dataset. +2. Periods are UTC calendar periods; VAT reporting uses calendar + quarters. +3. Money is EUR, decimal(12,2), converted from integer cents by + `cents_to_eur` — never floats. diff --git a/docs/runbooks/pii-detected-in-warehouse.md b/docs/runbooks/pii-detected-in-warehouse.md new file mode 100644 index 0000000..05de448 --- /dev/null +++ b/docs/runbooks/pii-detected-in-warehouse.md @@ -0,0 +1,33 @@ +# Runbook — PII detected in the warehouse + +**Severity: highest class. DPO owns the process from step 2.** Ties to +the incident process in `jol-m-infrastructure` and SECURITY.md. + +## Steps + +1. **Quarantine.** Pause downstream refresh of the affected marts; + restrict query access to the affected tables (data platform owner, + immediately). +2. **Notify the DPO** (`jol-m-compliance`) — before further technical + action. The GDPR 72h assessment clock is the DPO's call. +3. **Scope.** Determine: which tables/columns, since when, how many + rows (counts only — do not extract the values), which consumers + read it. Use `scripts/scan-warehouse-pii.py --warehouse` for the + sweep. +4. **Purge/repair.** Remove the offending data via a reviewed change: + fix the pseudonymizer/contract defect first, then purge or + re-pseudonymize the landed rows, then rebuild downstream marts. +5. **Root cause.** Which boundary failed — extract allow-list, + pseudonymizer rule, or a model that re-introduced an identifier? + The fix must close the class of failure, not the instance. +6. **Evidence.** Record timeline, scope counts, actions, and the DPO's + notification decision. Evidence custody: `jol-m-compliance`. +7. **Verify.** Run `make anonymize-verify` and the pii-scan self-test; + both must pass before the marts are un-paused. + +## Never + +- Do not quietly delete and move on — notification duty assessment must + happen even when the fix looks trivial. +- Do not paste offending values into issues, chat, or this runbook's + records. diff --git a/docs/runbooks/pipeline-failure.md b/docs/runbooks/pipeline-failure.md new file mode 100644 index 0000000..949dc33 --- /dev/null +++ b/docs/runbooks/pipeline-failure.md @@ -0,0 +1,28 @@ +# Runbook — pipeline failure + +**Goal:** restore the affected pipeline, quantify the stale window, and +decide whether downstream consumers must be marked stale. + +## Triage per pipeline + +| Pipeline | First check | Likely cause | Fix path | +|----------|-------------|--------------|----------| +| postgres_extract | extract role connectivity to replica | replica lag, role grant drift, network plane | verify replica health (`jol-m-infrastructure`), re-run nightly batch | +| stripe_extract | restricted key scope/quota | key rotation, Stripe outage | rotate via `.envrc`, backfill the window | +| pseudonymizer | rule parse errors / fail-closed drop | rules.yml change, unknown field | never "fix" by loosening rules — open DPO-reviewed contract change | +| dbt build | failing test vs failing model | upstream drift, schema change | run `dbt build` locally with dev profile; check `_staging.yml` contract | +| quality gates | which expectation failed | real data issue vs threshold | real issue → bug report; threshold → anomaly-rules review with owner | + +## Who to page + +- Data platform owner: pipeline/dbt failures. +- DPO: anything where the failure may involve personal data handling + (pseudonymizer, erasure propagation). +- Finance: VAT-OSS support figures stale during filing window. + +## After recovery + +1. Backfill the stale window; record it in the freshness report. +2. Mark affected dashboards stale until refresh completes. +3. If the failure exceeded the freshness SLA, the freshness-monitor + issue is the record; close it with the backfill evidence. diff --git a/docs/runbooks/restore-analytics.md b/docs/runbooks/restore-analytics.md new file mode 100644 index 0000000..246df58 --- /dev/null +++ b/docs/runbooks/restore-analytics.md @@ -0,0 +1,32 @@ +# Runbook — restore analytics (warehouse rebuild) + +**Key property: analytics is rebuildable from sources.** Production +backups are NOT in this repository (they live in +`jol-m-infrastructure`); a full warehouse loss is a rebuild, not a +restore-from-backup. + +## Rebuild procedure + +1. Provision an empty EU-region Postgres (`jol-m-infrastructure` + change management); apply encryption and access baseline. +2. Provide env vars per `.envrc.example`; write `warehouse/profiles.yml` + from `profiles.yml.example`. +3. Recreate the raw landing schema and extract-role grants + (`ingestion/pipelines/postgres_extract/extract-role.sql`). +4. Re-run extraction from the read replica + Stripe for the retention + window required by `governance/retention-map.md` — no further back: + the rebuild must respect retention, not undo it. +5. `dbt deps && dbt build` — seeds, staging, intermediate, marts. +6. Run quality gates: `make quality` + freshness report; all blocking + expectations green before consumers reconnect. +7. Re-run erasure verification (`make anonymize-verify`) — a rebuild + must not resurrect erased subjects; any hit is a severity-1 class + incident (previous runbook). + +## Boundaries + +- Do not rebuild beyond the retention horizon "for completeness". +- Legal holds apply to the rebuild too: held entities' data follows + the hold rules from day one. +- Record the rebuild in `audits/` (scope, window, verification + results). diff --git a/governance/README.md b/governance/README.md new file mode 100644 index 0000000..73f037a --- /dev/null +++ b/governance/README.md @@ -0,0 +1,39 @@ +# Governance — data governance operating model + +The catalog is the heart of this repository. If a dataset is not in the +catalog, it does not exist; if it exists but is not in the catalog, CI +blocks it (`scripts/catalog-lint.py` — no orphan datasets). + +## Operating model + +| Role | Responsibility | +|------|----------------| +| Data platform owner | Catalog accuracy, pipeline ownership, this repo | +| Business owner (per dataset) | Purpose, classification, retention decisions | +| Technical steward (per dataset) | Schema, quality, lineage maintenance | +| DPO (`jol-m-compliance`) | RESTRICTED tier approval, retention classes, RoPA linkage | + +## Review cadence + +- **Weekly:** quality scorecards (`quality/scorecards/`) reviewed by the + data platform owner. +- **Monthly:** ownership register reconciliation — every dataset has a + living owner and steward; orphans escalate. +- **Quarterly:** retention-map review with the DPO; classification + re-attestation for CONFIDENTIAL/RESTRICTED datasets. +- **On change:** every dataset PR carries classification + RoPA purpose + + retention class (PR template enforces). + +## Files + +| File | Role | +|------|------| +| [data-catalog.md](data-catalog.md) | Master index — every dataset with owner, classification, purpose, retention | +| [classification.md](classification.md) | The 4 tiers and per-tier handling rules | +| [ownership-register.csv](ownership-register.csv) | Machine-readable ownership (parsed by catalog-lint) | +| [retention-map.md](retention-map.md) | Dataset → retention class → enforcement mechanism | +| [lineage.md](lineage.md) | Source → ingestion → warehouse → consumer per critical dataset | + +Rules for registering a dataset: open a `dataset_request` issue first — +purpose, classification, retention, and ownership are declared before +any pipeline or model code lands. diff --git a/governance/classification.md b/governance/classification.md new file mode 100644 index 0000000..0c9e6eb --- /dev/null +++ b/governance/classification.md @@ -0,0 +1,52 @@ +# Data Classification — four tiers + +Every dataset in [data-catalog.md](data-catalog.md) carries exactly one +tier. Tier changes are critical-risk changes: DPO approval required, +CODEOWNERS routes it. + +## Tiers + +### PUBLIC + +Publishable outside the org (typically only via product surfaces). + +- Examples: taxonomy structure, synthetic fixtures, calendar dimension. +- Handling: still no secrets, no credentials; review before any + external publication path. + +### INTERNAL + +Any org member; not for external distribution. + +- Examples: dbt models, dashboards definitions, runbooks, funnel + aggregates. +- Handling: standard access; no personal data. + +### CONFIDENTIAL + +Need-to-know: data, finance, and compliance roles. + +- Examples: seller/buyer aggregates, commission and VAT figures, + pseudonymized dimensions whose join key is held elsewhere. +- Handling: no export to personal devices; dashboard access gated; + quarterly re-attestation. + +### RESTRICTED + +Personal data or data directly derived from it; DPO-controlled. + +- Examples: DSAR SLA metrics, consent rates, erasure execution + aggregates. +- Handling: aggregates only — never per-subject rows in this + repository's artifacts; DPO review on every change; access logged; + retention enforced by `lifecycle/retention-jobs/`. + +## Handling rules common to all tiers + +1. No credentials, ever (gitleaks enforces). +2. No cleartext personal data, ever — the pseudonymization boundary is + at ingestion (ADR-0001), and committed fixtures are synthetic. +3. Tier is declared in the catalog at dataset creation; "decide later" + is not a valid state — undeclared datasets are blocked. +4. When in doubt between two tiers, choose the higher one and raise it + in the quarterly review. diff --git a/governance/data-catalog.md b/governance/data-catalog.md new file mode 100644 index 0000000..8174f13 --- /dev/null +++ b/governance/data-catalog.md @@ -0,0 +1,41 @@ +# Data Catalog — master index + +Every dataset this repository creates, transforms, or serves. One row = +one dataset. The machine-readable mirror is +[ownership-register.csv](ownership-register.csv); `catalog-lint` keeps +them consistent and blocks orphans. + +Tiers per [classification.md](classification.md); retention classes per +[retention-map.md](retention-map.md); RoPA purposes live in +`jol-m-compliance` (referenced by id, never restated here). + +| dataset_id | Description | Business owner | Technical steward | Classification | RoPA purpose | Retention class | +|------------|-------------|----------------|-------------------|----------------|--------------|-----------------| +| fct_orders | Order facts, pseudonymous keys, EUR amounts | Marketplace product | Data platform | CONFIDENTIAL | ropa-analytics-marketplace | operational | +| dim_products | Product dimension (taxonomy-linked) | Marketplace product | Data platform | INTERNAL | ropa-analytics-marketplace | operational | +| dim_sellers | Seller dimension, pseudonymized | Marketplace product | Data platform | CONFIDENTIAL | ropa-analytics-marketplace | operational | +| dim_date | Calendar dimension (synthetic reference) | Data platform | Data platform | PUBLIC | n/a | none-synthetic | +| fct_vat_oss | VAT OSS reporting support facts | Finance | Data platform | CONFIDENTIAL | ropa-tax-compliance | statutory | +| fct_commission | Commission facts per order | Finance | Data platform | CONFIDENTIAL | ropa-tax-compliance | statutory | +| fct_payouts | Payout facts (Stripe metadata only) | Finance | Data platform | CONFIDENTIAL | ropa-tax-compliance | statutory | +| seller_health | Seller health aggregates | Marketplace product | Data platform | CONFIDENTIAL | ropa-analytics-marketplace | operational | +| listing_funnel | Listing funnel aggregates | Marketplace product | Data platform | INTERNAL | ropa-analytics-marketplace | operational | +| search_analytics | Search relevance aggregates | Marketplace product | Data platform | INTERNAL | ropa-analytics-marketplace | short-term | +| dsr_sla_metrics | DSAR/DSR SLA aggregates | DPO | Data platform | RESTRICTED | ropa-gdpr-operations | operational | +| consent_rates | Consent opt-in aggregates | DPO | Data platform | RESTRICTED | ropa-gdpr-operations | operational | +| erasure_execution_log | Erasure execution aggregates (no subject rows) | DPO | Data platform | RESTRICTED | ropa-gdpr-operations | statutory | +| seed_taxonomy_categories | Marketplace category tree (synthetic reference) | Marketplace product | Data platform | PUBLIC | n/a | none-synthetic | +| seed_geo_references | LT/LV/EE municipalities, parishes, locker refs | Marketplace product | Data platform | PUBLIC | n/a | none-synthetic | +| seed_tax_vat_rates | VAT rate reference per category/country | Finance | Data platform | INTERNAL | n/a | none-synthetic | +| seed_fixtures | Synthetic sellers/products/orders fixtures | Marketplace product | Data platform | PUBLIC | n/a | none-synthetic | +| ml_embeddings_products | Product-description embedding builds | Data platform | Data platform | INTERNAL | ropa-search-relevance | operational | +| ml_translation_memory | Domain glossary pairs lt/lv/et/en | Marketplace product | Data platform | PUBLIC | n/a | none-synthetic | + +## Adding a dataset + +1. Open a `dataset_request` issue (purpose, classification, retention, + ownership are mandatory fields). +2. Add the row here **and** in `ownership-register.csv`. +3. Register the retention class in `retention-map.md`. +4. For critical datasets, add a lineage diagram in `lineage.md`. +5. `make catalog-lint` must pass before merge. diff --git a/governance/lineage.md b/governance/lineage.md new file mode 100644 index 0000000..ff735eb --- /dev/null +++ b/governance/lineage.md @@ -0,0 +1,59 @@ +# Lineage — source → ingestion → warehouse → consumer + +Per critical dataset. The pseudonymization boundary is the single most +important edge in every diagram: **nothing downstream of it may carry +cleartext identifiers** (ADR-0001). + +## Orders (fct_orders) + +```mermaid +flowchart LR + subgraph Production [jol-m-marketplace production] + A[(orders table)] --> B[(read replica)] + end + B -->|"read-only role\nleast privilege"| C[pseudonymizer\nhash buyer/seller IDs] + C -->|"pseudonymous landing\n(ingestion contract)"| D[(warehouse raw)] + D --> E[stg_orders — PII stripped] + E --> F[int_order_items_enriched] + F --> G[fct_orders] + G --> H1[dashboards] + G --> H2[fct_vat_oss] +``` + +## Sellers (dim_sellers) + +```mermaid +flowchart LR + A[(users table — sellers)] --> B[(read replica)] + B --> C[pseudonymizer — hash IDs, drop names/emails] + C --> D[(warehouse raw.users)] + D --> E[stg_users — role/country/consent aggregates only] + E --> F[dim_sellers — pseudonymized] + F --> G[seller_health] +``` + +## Payments (fct_payouts) + +```mermaid +flowchart LR + S[Stripe API] -->|"restricted key:\ncharges/payouts metadata only\n(never PAN — SAQ-A)"| C[pseudonymizer — account refs hashed] + C --> D[(warehouse raw)] + D --> E[fct_payouts] + E --> F[fct_commission] +``` + +## Erasure propagation + +```mermaid +flowchart LR + M[jol-m-marketplace: DSAR erasure executed] --> V[lifecycle/anonymization verify job] + V --> W{warehouse rows for hashed subject gone/anonymized?} + W -->|yes| P[lifecycle/verification: proof recorded] + W -->|no| I[pii_incident issue → DPO] +``` + +## Maintenance rule + +Any new critical dataset (CONFIDENTIAL/RESTRICTED or feeding finance +reporting) must land with a lineage diagram here in the same PR; +`dataset_request` intake asks for sources explicitly. diff --git a/governance/ownership-register.csv b/governance/ownership-register.csv new file mode 100644 index 0000000..2fb646d --- /dev/null +++ b/governance/ownership-register.csv @@ -0,0 +1,20 @@ +dataset_id,business_owner,technical_steward,classification,ropa_purpose,retention_class +fct_orders,marketplace-product,data-platform,CONFIDENTIAL,ropa-analytics-marketplace,operational +dim_products,marketplace-product,data-platform,INTERNAL,ropa-analytics-marketplace,operational +dim_sellers,marketplace-product,data-platform,CONFIDENTIAL,ropa-analytics-marketplace,operational +dim_date,data-platform,data-platform,PUBLIC,n/a,none-synthetic +fct_vat_oss,finance,data-platform,CONFIDENTIAL,ropa-tax-compliance,statutory +fct_commission,finance,data-platform,CONFIDENTIAL,ropa-tax-compliance,statutory +fct_payouts,finance,data-platform,CONFIDENTIAL,ropa-tax-compliance,statutory +seller_health,marketplace-product,data-platform,CONFIDENTIAL,ropa-analytics-marketplace,operational +listing_funnel,marketplace-product,data-platform,INTERNAL,ropa-analytics-marketplace,operational +search_analytics,marketplace-product,data-platform,INTERNAL,ropa-analytics-marketplace,short-term +dsr_sla_metrics,dpo,data-platform,RESTRICTED,ropa-gdpr-operations,operational +consent_rates,dpo,data-platform,RESTRICTED,ropa-gdpr-operations,operational +erasure_execution_log,dpo,data-platform,RESTRICTED,ropa-gdpr-operations,statutory +seed_taxonomy_categories,marketplace-product,data-platform,PUBLIC,n/a,none-synthetic +seed_geo_references,marketplace-product,data-platform,PUBLIC,n/a,none-synthetic +seed_tax_vat_rates,finance,data-platform,INTERNAL,n/a,none-synthetic +seed_fixtures,marketplace-product,data-platform,PUBLIC,n/a,none-synthetic +ml_embeddings_products,data-platform,data-platform,INTERNAL,ropa-search-relevance,operational +ml_translation_memory,marketplace-product,data-platform,PUBLIC,n/a,none-synthetic diff --git a/governance/retention-map.md b/governance/retention-map.md new file mode 100644 index 0000000..96cc41c --- /dev/null +++ b/governance/retention-map.md @@ -0,0 +1,48 @@ +# Retention Map — dataset → retention class → enforcement + +This is the enforcement-side view of the retention schedule. **Policy +text lives in `jol-m-compliance` (retention schedule)**; this map binds +each dataset in [data-catalog.md](data-catalog.md) to a class and the +mechanism that executes it. Changes require DPO review (CODEOWNERS). + +## Retention classes + +| Class | Horizon | Meaning | +|-------|---------|---------| +| short-term | ≤ 90 days | Debugging/relevance telemetry; purge window runs monthly | +| operational | ≤ 2 years | Live analytics need; rolling purge yearly | +| statutory | ≤ 10 years | Accounting/tax support (LT VAT OSS: 10y per national rules) | +| indefinite-reference | review every 2y | Synthetic/reference data with no personal data | +| none-synthetic | n/a | 100% synthetic or derived constants; exempt from purge | + +## Map + +| dataset_id | Class | Enforcement mechanism | +|------------|-------|-----------------------| +| fct_orders | operational | `lifecycle/retention-jobs/` rolling partition drop + re-aggregation | +| dim_products | operational | retention job; tombstoned listings anonymized in place | +| dim_sellers | operational | retention job; seller erasure propagates from product DB (see `lifecycle/anonymization/`) | +| dim_date | none-synthetic | exempt | +| fct_vat_oss | statutory | legal-hold aware; purge only after DPO release | +| fct_commission | statutory | legal-hold aware; purge only after DPO release | +| fct_payouts | statutory | legal-hold aware; purge only after DPO release | +| seller_health | operational | retention job (rolling window) | +| listing_funnel | operational | retention job (rolling window) | +| search_analytics | short-term | monthly purge job | +| dsr_sla_metrics | operational | retention job; aggregates only by construction | +| consent_rates | operational | retention job; aggregates only by construction | +| erasure_execution_log | statutory | legal-hold aware; execution proofs are audit evidence | +| seed_* | none-synthetic | exempt (synthetic by construction) | +| ml_embeddings_products | operational | rebuild on purge; embeddings of erased listings dropped | +| ml_translation_memory | none-synthetic | exempt | + +## Enforcement rules + +1. **Legal holds suspend, never delete.** A hold flag + (`lifecycle/legal-hold/`) pauses the job for the affected entities; + counsel-controlled release. +2. **Erasure propagates.** When the product DB anonymizes a subject + (`jol-m-marketplace`), `lifecycle/anonymization/` verifies the + warehouse followed — verification is evidence, kept statutory. +3. **Proofs are sampled.** Post-run proofs + adversarial + re-identification sampling live in `lifecycle/verification/`. diff --git a/ingestion/README.md b/ingestion/README.md new file mode 100644 index 0000000..31432ec --- /dev/null +++ b/ingestion/README.md @@ -0,0 +1,34 @@ +# Ingestion — read-replica → strip identifiers → land pseudonymized + +Doctrine: + +1. **Read-replica only, least privilege.** Extraction never touches the + primary; the extract role is read-only and table-scoped (ADR-0002). +2. **The pseudonymizer is the boundary.** Identifiers are stripped or + hashed BEFORE landing (ADR-0001). The warehouse must be unable to + re-identify without external collusion — that is a design property, + not an aspiration. +3. **Contracts are allow-lists.** `contracts/` declares what fields may + leave each source. Anything not on the allow-list stays at the + source; widening an allow-list is a DPO-reviewed change. +4. **Never PAN.** The Stripe extract takes charge/payout metadata only; + the SAQ-A boundary holds in analytics too. + +## Layout + +| Path | Content | +|------|---------| +| `pipelines/postgres_extract/` | Batch/CDC extract from the read replica (read-only role) | +| `pipelines/stripe_extract/` | Charges/payouts metadata extract | +| `pipelines/pseudonymizer/` | THE critical component: strips/hashes identifiers pre-landing, unit-tested | +| `contracts/` | Schema contracts per source: field allow-lists | + +## Adding a source + +1. `dataset_request` issue first (governance). +2. Write the contract (allow-list) in `contracts/` — DPO review if any + field is personal data. +3. Implement the extract + pseudonymizer rules; unit tests mandatory + for any pseudonymization logic. +4. Register the landed tables in `warehouse/models/staging/_staging.yml` + with freshness SLAs. diff --git a/ingestion/contracts/postgres.yml b/ingestion/contracts/postgres.yml new file mode 100644 index 0000000..19292ab --- /dev/null +++ b/ingestion/contracts/postgres.yml @@ -0,0 +1,40 @@ +# Schema contract — jol-m-marketplace read replica (postgres_extract). +# ALLOW-LIST: only these fields may leave the source. Widening is a +# DPO-reviewed change. Pseudonymizer actions: see +# ../pipelines/pseudonymizer/rules.yml (fail-closed default: drop). +schema_version: 1 +source: marketplace_read_replica +role: jol_extract # read-only, table-scoped (extract-role.sql) +tables: + orders: + allow: + - id # hashed at landing + - buyer_id # hashed at landing + - seller_id # hashed at landing + - status + - amount_cents + - currency + - category_code + - country_code + - vat_rate_pct + - created_at + prohibited_examples: [buyer_name, billing_address, notes] + products: + allow: + - id # hashed at landing + - seller_id # hashed at landing + - category_code + - title + - price_cents + - currency + - status + - created_at + prohibited_examples: [cost_price, supplier_ref] + users: + allow: + - id # hashed at landing + - role + - country_code + - consent_marketing + - created_at # generalized to day precision + prohibited_examples: [name, email, phone, national_id, bank_ref] diff --git a/ingestion/contracts/stripe.yml b/ingestion/contracts/stripe.yml new file mode 100644 index 0000000..57d7aaa --- /dev/null +++ b/ingestion/contracts/stripe.yml @@ -0,0 +1,29 @@ +# Schema contract — Stripe (stripe_extract). +# ALLOW-LIST of charge/payout METADATA. PAN and cardholder data never +# leave Stripe under any circumstance (SAQ-A boundary holds in +# analytics too). Restricted key, read scope only. +schema_version: 1 +source: stripe_api +key_scope: restricted # charges:read, payouts:read — nothing else +objects: + charges: + allow: + - charge_id # hashed at landing + - payout_id # hashed at landing + - amount_cents + - currency + - status + - created_at + prohibited_examples: + - card_number # PAN — never + - cardholder_name + - bank_account_number + - dispute_evidence + payouts: + allow: + - payout_id # hashed at landing + - amount_cents + - currency + - status + - arrival_period + prohibited_examples: [bank_account_number, recipient_name] diff --git a/ingestion/pipelines/postgres_extract/README.md b/ingestion/pipelines/postgres_extract/README.md new file mode 100644 index 0000000..189b253 --- /dev/null +++ b/ingestion/pipelines/postgres_extract/README.md @@ -0,0 +1,15 @@ +# postgres_extract — read replica → pseudonymized landing + +Batch/CDC extraction from the `jol-m-marketplace` **read replica**. + +- Connection: read-only role, table-scoped grants (see + `extract-role.sql` — applied by `jol-m-infrastructure`, not here). +- Cadence: nightly batch at scaffold; CDC upgrade path is documented + when order volume justifies it. +- Output goes through `../pseudonymizer/` before any landing write. + +## Invariant + +This job has NO write path to production and NO access to fields +outside `../contracts/postgres.yml`. If the contract and the code +disagree, the contract wins and the job fails closed. diff --git a/ingestion/pipelines/postgres_extract/extract-role.sql b/ingestion/pipelines/postgres_extract/extract-role.sql new file mode 100644 index 0000000..33a4413 --- /dev/null +++ b/ingestion/pipelines/postgres_extract/extract-role.sql @@ -0,0 +1,17 @@ +-- extract-role.sql — least-privilege extract role for the READ REPLICA. +-- Executed by jol-m-infrastructure (change management applies); kept +-- here as the source of truth for what analytics is granted. + +create role jol_extract login; + +-- Replica only. No superuser, no create, no production access. +grant connect on database marketplace to jol_extract; +grant usage on schema public to jol_extract; + +-- Table-scoped reads matching ingestion/contracts/postgres.yml +grant select on table public.orders to jol_extract; +grant select on table public.products to jol_extract; +grant select on table public.users to jol_extract; + +-- Explicitly nothing else: no grants on payments, payouts, sessions, +-- consent logs, or identity documents. diff --git a/ingestion/pipelines/pseudonymizer/README.md b/ingestion/pipelines/pseudonymizer/README.md new file mode 100644 index 0000000..c0edea1 --- /dev/null +++ b/ingestion/pipelines/pseudonymizer/README.md @@ -0,0 +1,24 @@ +# pseudonymizer — THE critical component + +Strips or hashes identifiers **before** landing. Everything downstream +inherits its guarantees; a defect here is a breach of the whole +warehouse design (ADR-0001). + +## Rules (`rules.yml`) + +| Action | Meaning | +|--------|---------| +| drop | Field never leaves the source (names, emails, phones, registry codes) | +| hash | Deterministic salted hash (same scheme as warehouse `hash_id`) | +| generalize | Reduce precision (dates → month, geo → municipality) | +| pass | Non-personal field, on the contract allow-list | + +## Hard requirements + +1. **Unit tests mandatory** (`test_pseudonymizer.py`): drop rules must + produce absent fields, hash rules must be deterministic and + irreversible-without-salt, and any new field defaults to DROP + (fail-closed) until explicitly classified in `rules.yml`. +2. Changes are critical-risk (CONTRIBUTING.md): DPO + data platform + review, and `make anonymize-verify` evidence after deploy. +3. The salt is an environment secret (HASH_SALT), never in-repo. diff --git a/ingestion/pipelines/pseudonymizer/pseudonymizer.py b/ingestion/pipelines/pseudonymizer/pseudonymizer.py new file mode 100755 index 0000000..fa3e648 --- /dev/null +++ b/ingestion/pipelines/pseudonymizer/pseudonymizer.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +"""Pseudonymizer — strips/hashes identifiers BEFORE warehouse landing. + +The boundary component of ADR-0001. Fields not explicitly listed in +rules.yml are dropped (fail-closed). Pure stdlib: no dependencies may +be added without review — this code is audited. +""" + +from __future__ import annotations + +import argparse +import datetime +import hashlib +import json +import sys +from pathlib import Path + +try: + import yaml +except ImportError: # pragma: no cover - environment guard + sys.exit("pyyaml is required: pip install -e .") + +RULES_PATH = Path(__file__).with_name("rules.yml") + + +def load_rules(path: Path = RULES_PATH) -> dict: + with open(path, encoding="utf-8") as fh: + rules = yaml.safe_load(fh) + if rules.get("default_action") != "drop": + raise ValueError("pseudonymizer must stay fail-closed (default_action: drop)") + return rules + + +def hash_value(value: object, salt: str) -> str: + """Deterministic salted md5 — same scheme as warehouse hash_id().""" + return hashlib.md5(f"{value}{salt}".encode("utf-8")).hexdigest() + + +def apply_rules(source: str, record: dict, rules: dict, salt: str) -> dict: + """Return the landing-safe view of a source record.""" + table_rules = rules.get("sources", {}).get(source) + if table_rules is None: + raise KeyError(f"no pseudonymizer rules for source table: {source}") + out: dict = {} + for field, value in record.items(): + action = table_rules.get(field, rules["default_action"]) + if action == "drop": + continue + if action == "pass": + out[field] = value + elif action == "hash": + out[field] = hash_value(value, salt) + elif action == "generalize": + out[field] = _generalize(value) + else: + raise ValueError(f"unknown pseudonymizer action: {action}") + return out + + +def _generalize(value: object) -> object: + """Reduce precision: timestamps -> date, datetimes -> month-first-day.""" + if isinstance(value, datetime.datetime): + return value.date() + if isinstance(value, str): + return value[:10] # ISO datetime string -> date prefix + return value + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--source", required=True, help="source table name") + parser.add_argument("--record", required=True, help="JSON record to pseudonymize") + parser.add_argument("--salt", default="dev-only-salt") + args = parser.parse_args() + record = json.loads(args.record) + result = apply_rules(args.source, record, load_rules(), args.salt) + print(json.dumps(result, indent=2, default=str)) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ingestion/pipelines/pseudonymizer/rules.yml b/ingestion/pipelines/pseudonymizer/rules.yml new file mode 100644 index 0000000..1a82a50 --- /dev/null +++ b/ingestion/pipelines/pseudonymizer/rules.yml @@ -0,0 +1,38 @@ +# Pseudonymizer rules per source table. Fields NOT listed here are +# DROPPED by default (fail-closed). Any change is DPO-reviewed. +schema_version: 1 +default_action: drop +sources: + orders: + id: hash + buyer_id: hash + seller_id: hash + status: pass + amount_cents: pass + currency: pass + category_code: pass + country_code: pass # buyer country, 2-letter only + vat_rate_pct: pass + created_at: pass + products: + id: hash + seller_id: hash + category_code: pass + title: pass # product content, reviewed per contract + price_cents: pass + currency: pass + status: pass + created_at: pass + users: + id: hash + role: pass + country_code: pass + consent_marketing: pass # the flag travels; the identity never does + created_at: generalize # day precision retained; time dropped + stripe_charges: + charge_id: hash + payout_id: hash + amount_cents: pass + currency: pass + status: pass + created_at: pass diff --git a/ingestion/pipelines/pseudonymizer/test_pseudonymizer.py b/ingestion/pipelines/pseudonymizer/test_pseudonymizer.py new file mode 100755 index 0000000..64babc7 --- /dev/null +++ b/ingestion/pipelines/pseudonymizer/test_pseudonymizer.py @@ -0,0 +1,55 @@ +"""Unit tests for the pseudonymizer — mandatory coverage (README §Hard +requirements). Run: python3 -m unittest test_pseudonymizer +""" + +import unittest +from pathlib import Path + +from pseudonymizer import apply_rules, hash_value, load_rules + +SALT = "unit-test-salt" + + +class PseudonymizerTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.rules = load_rules(Path(__file__).with_name("rules.yml")) + + def test_fail_closed_default_drops_unlisted_fields(self): + record = {"id": 1, "email": "should-never-land@example.test"} + out = apply_rules("orders", record, self.rules, SALT) + self.assertNotIn("email", out) + self.assertIn("id", out) # listed -> hashed, not raw + + def test_hash_rules_are_deterministic_and_not_cleartext(self): + record = {"id": 42} + first = apply_rules("orders", record, self.rules, SALT) + second = apply_rules("orders", record, self.rules, SALT) + self.assertEqual(first, second) + self.assertNotEqual(first["id"], 42) + self.assertNotEqual(first["id"], "42") + self.assertEqual(first["id"], hash_value(42, SALT)) + + def test_hash_changes_with_salt(self): + self.assertNotEqual( + hash_value(42, "salt-a"), hash_value(42, "salt-b") + ) + + def test_drop_rules_produce_absent_fields(self): + record = {"id": 1, "buyer_name": "X", "phone": "5550000"} + out = apply_rules("users", record, self.rules, SALT) + self.assertNotIn("buyer_name", out) + self.assertNotIn("phone", out) + + def test_generalize_reduces_timestamp_precision(self): + record = {"id": 1, "created_at": "2026-08-15T10:30:00+00:00"} + out = apply_rules("users", record, self.rules, SALT) + self.assertEqual(out["created_at"], "2026-08-15") + + def test_unknown_source_table_fails_closed(self): + with self.assertRaises(KeyError): + apply_rules("payments", {"id": 1}, self.rules, SALT) + + +if __name__ == "__main__": + unittest.main() diff --git a/ingestion/pipelines/stripe_extract/README.md b/ingestion/pipelines/stripe_extract/README.md new file mode 100644 index 0000000..a2f8b5c --- /dev/null +++ b/ingestion/pipelines/stripe_extract/README.md @@ -0,0 +1,16 @@ +# stripe_extract — charges/payouts metadata only + +Extracts settlement metadata from Stripe with a **restricted key**: + +- Allowed: charge ids, amounts, currencies, payout ids/periods, fee + metadata, status timestamps. +- Prohibited: PAN/full card data, cardholder names, full bank account + numbers, dispute evidence content. The SAQ-A boundary holds in + analytics too. + +Key custody follows `jol-m-infrastructure` token rotation discipline; +the key lives in Vaultwarden and is referenced via `EXTRACT_STRIPE_KEY` +(see `.envrc.example`) — never committed. + +Output goes through `../pseudonymizer/` (account references are hashed) +before landing; see `../contracts/stripe.yml` for the field allow-list. diff --git a/lifecycle/README.md b/lifecycle/README.md new file mode 100644 index 0000000..4315e56 --- /dev/null +++ b/lifecycle/README.md @@ -0,0 +1,26 @@ +# Lifecycle — retention & anonymization machinery + +**This code executes the retention schedule; the policy text lives in +`jol-m-compliance`.** Nothing here re-states policy — every job points +at the retention class it implements (`governance/retention-map.md`). + +| Path | Content | +|------|---------| +| `anonymization/` | Erasure-support jobs: verify product-DB anonymization propagated to the warehouse | +| `retention-jobs/` | Scheduled purge/anonymize per retention-map (warehouse only; production tables are owned by jol-m-marketplace) | +| `legal-hold/` | Hold flags that suspend retention jobs for specific entities (counsel-controlled) | +| `verification/` | Post-run proofs: sampled re-identification attempts must fail (adversarial test) | + +## Hard rules + +1. **Scope: warehouse only.** Production erasure happens in + `jol-m-marketplace`; this machinery verifies propagation and purges + analytics copies. +2. **Legal holds suspend, never delete.** A hold pauses the applicable + retention job for the held entities; release is counsel/DPO + controlled and logged. +3. **Every run leaves a proof.** Execution + verification results are + evidence (statutory retention per `governance/retention-map.md`). +4. **Adversarial verification is mandatory** after anonymization runs: + `make anonymize-verify` (scripts/verify-anonymization.py) — sampled + re-identification attempts must fail. diff --git a/lifecycle/anonymization/README.md b/lifecycle/anonymization/README.md new file mode 100644 index 0000000..4eb0070 --- /dev/null +++ b/lifecycle/anonymization/README.md @@ -0,0 +1,25 @@ +# Anonymization — erasure propagation verification + +When `jol-m-marketplace` executes a DSAR erasure (subject anonymized in +the product DB), the warehouse must follow: every model keyed by that +subject's pseudonymous key must be purged or re-aggregated so that no +row attributable to the subject remains. + +## Verification job + +Input: erasure events from the compliance DSAR log (subject → hashed +key mapping happens inside the product boundary; this job receives the +hashed key only). + +Steps: + +1. For each hashed key, query every model in + `warehouse/models/` that carries subject keys (fct_orders, + dim_sellers, seller_health, …). +2. Assert: zero rows for per-subject models; aggregates unchanged in + shape (no per-subject residue). +3. Record the proof in `../verification/` (run id, models checked, + result, timestamp). + +A failed check opens a `pii_incident` issue and pauses downstream +refresh — see `docs/runbooks/pii-detected-in-warehouse.md`. diff --git a/lifecycle/legal-hold/README.md b/lifecycle/legal-hold/README.md new file mode 100644 index 0000000..1472185 --- /dev/null +++ b/lifecycle/legal-hold/README.md @@ -0,0 +1,26 @@ +# Legal holds — counsel-controlled suspension of retention + +A hold suspends retention jobs for specific entities (hashed keys — +never cleartext identities in this file). **Holds suspend; they never +delete and never reveal.** Setting or releasing a hold is a +counsel/DPO-controlled action and is logged. + +## Operating rules + +1. Entities are referenced by pseudonymous key only; the mapping to a + subject exists inside `jol-m-marketplace`/`jol-m-compliance`. +2. Every hold carries: reason class (litigation, regulatory inquiry, + audit), requested by (role, not name in this file), start date. +3. Retention jobs MUST consult `holds.yml` and skip held entities, + reporting them as skipped in the run proof. +4. Quarterly review with counsel; stale holds escalate. + +## holds.yml format + +```yaml +holds: [] +# - entity_key: <32-char hash> +# reason_class: litigation +# requested_by: general-counsel +# started_at: "YYYY-MM-DD" +``` diff --git a/lifecycle/legal-hold/holds.yml b/lifecycle/legal-hold/holds.yml new file mode 100644 index 0000000..4c2c4fe --- /dev/null +++ b/lifecycle/legal-hold/holds.yml @@ -0,0 +1,3 @@ +# Legal holds register — counsel-controlled. See README.md for rules. +# Entities are pseudonymous keys ONLY; no cleartext identities. +holds: [] diff --git a/lifecycle/retention-jobs/README.md b/lifecycle/retention-jobs/README.md new file mode 100644 index 0000000..a1f3492 --- /dev/null +++ b/lifecycle/retention-jobs/README.md @@ -0,0 +1,26 @@ +# Retention jobs — scheduled purge/anonymize per retention-map + +Each job implements exactly one retention class from +`governance/retention-map.md`; the class → mechanism mapping is owned +there, execution is owned here. **Warehouse only** — production +retention is owned by `jol-m-marketplace`. + +## Job contract (every job must) + +1. Declare the retention class and dataset scope in its header. +2. Respect legal holds (`../legal-hold/holds.yml`) — held entities are + skipped and counted in the run report. +3. Run destructively only inside an explicit transaction with a prior + row-count capture (proof input). +4. Write a run proof consumable by `../verification/`. + +## Scheduled jobs (planned at scaffold) + +| Job | Class | Cadence | Mechanism | +|-----|-------|---------|-----------| +| purge-search-analytics | short-term | monthly | drop partitions > 90d | +| purge-operational | operational | yearly | rolling window + re-aggregation to monthly rollups | +| purge-statutory | statutory | yearly | drop only after horizon + DPO release + no hold | + +Jobs land with the warehouse environment; until then this file is the +contract they are built against. diff --git a/lifecycle/verification/README.md b/lifecycle/verification/README.md new file mode 100644 index 0000000..91b31cc --- /dev/null +++ b/lifecycle/verification/README.md @@ -0,0 +1,27 @@ +# Verification — post-run proofs & adversarial testing + +Retention/anonymization runs must leave evidence that stands up in +audit. Proofs are statutory-retention records +(`governance/retention-map.md`). + +## Proof contents (per run) + +- Run id, job, retention class, dataset scope, timestamp. +- Row counts before/after; held entities skipped. +- Verification result: PASS/FAIL + sampled evidence (no subject data — + counts and key prefixes only). + +## Adversarial re-identification sampling + +`make anonymize-verify` (scripts/verify-anonymization.py) attempts +re-identification on samples: + +1. Pick sampled pseudonymous keys from marts. +2. Attempt joins across marts and against committed seed/geo/tax + references to reconstruct an identity. +3. **Every attempt must fail.** Any successful path is a severity-1 + class incident (SECURITY.md) — it means the pseudonymization + boundary has a hole. + +Proofs are written to run logs here (gitignored working files; durable +copies go to the compliance evidence custody in `jol-m-compliance`). diff --git a/ml/README.md b/ml/README.md new file mode 100644 index 0000000..a27aa5d --- /dev/null +++ b/ml/README.md @@ -0,0 +1,27 @@ +# ML / Embedding Datasets — feeds ai_service_app semantic search + +Dataset governance for AI: every dataset here carries provenance, +license basis, and a PII-free certification before it may be consumed. + +| Path | Content | +|------|---------| +| `embeddings/` | Product-description embedding build jobs (pgvector export format) | +| `evaluation/` | Search relevance eval sets (synthetic + human-labeled, anonymized) | +| `translation-memory/` | Domain glossary pairs lt/lv/et/en (legal terms sync with jol-m-legal) | + +## Governance rules (apply to every artifact under ml/) + +1. **Provenance is recorded.** Source dataset, build date, model + + version for every embedding build; catalog entry per dataset + (`governance/data-catalog.md`: ml_embeddings_products, + ml_translation_memory). +2. **License basis declared.** Product descriptions are licensed by + sellers under the seller agreement; eval labels by contributors + under the CLA (`jol-m-legal/intellectual-property/copyright/`). + No scraped external corpora. +3. **PII-free certification.** Builds run on pseudonymous warehouse + marts only; the pii-scan gate applies to committed eval data. + Embeddings of erased listings are dropped on erasure propagation + (`lifecycle/anonymization/`). +4. **No personal data in prompts or eval sets.** Human-labeled sets are + anonymized before commit. diff --git a/ml/embeddings/README.md b/ml/embeddings/README.md new file mode 100644 index 0000000..1c7cefa --- /dev/null +++ b/ml/embeddings/README.md @@ -0,0 +1,22 @@ +# Embeddings — product-description embedding builds + +Build jobs producing product-description vectors for the semantic +search service (`ai_service_app`), exported in **pgvector format**. + +## Build contract + +- Input: `dim_products` (active listings) — pseudonymous warehouse + mart; never production text with seller identity fields. +- Output: `(product_key, embedding vector, model_ref, built_at)` — + product_key is the pseudonymous hash, so erasure propagation can + drop vectors without ever knowing the listing's owner. +- Provenance: every build records model + version + seed data date + (catalog: ml_embeddings_products). + +## Lifecycle + +1. Rebuild on taxonomy MAJOR changes (description semantics shift). +2. Incremental nightly for new/changed listings (when the warehouse + environment is live). +3. On erasure events: drop vectors for the erased subject's listings + — verified by `lifecycle/anonymization/`. diff --git a/ml/evaluation/README.md b/ml/evaluation/README.md new file mode 100644 index 0000000..ad9ab50 --- /dev/null +++ b/ml/evaluation/README.md @@ -0,0 +1,20 @@ +# Evaluation — search relevance eval sets + +Relevance judgments for the semantic search stack. Two families: + +1. **Synthetic sets** — generated query/listing pairs from + `seed/fixtures` + taxonomy; deterministic, committed, used in CI. +2. **Human-labeled sets** — relevance grades by reviewers. Labels are + anonymized before commit: reviewer ids are roles, query logs are + hashed, and no session identifiers travel with labels. + +## Rules + +- Eval sets are versioned; a model comparison is only valid within one + eval-set version. +- Language coverage: lt/lv/et/en queries each — Baltic-locale relevance + is the product differentiator and must be measured per locale. +- PII gate applies: committed eval files run through the same scanners + as seed data (`make check`). +- Label governance: contributors covered by the CLA + (`jol-m-legal/intellectual-property/copyright/`). diff --git a/ml/translation-memory/README.md b/ml/translation-memory/README.md new file mode 100644 index 0000000..ded1a63 --- /dev/null +++ b/ml/translation-memory/README.md @@ -0,0 +1,19 @@ +# Translation memory — domain glossary pairs lt/lv/et/en + +Preferred term pairs for marketplace domain vocabulary, consumed by +listing translation quality checks and `ai_service_app` multilingual +search. + +- `glossary.csv` — the committed term pairs (see below). +- **Legal terms are not owned here.** Canonical legal glossary lives in + `jol-m-legal/docs/glossary.md`; legal terms in this file sync FROM + there (never the reverse) and are marked `source=jol-m-legal`. + +## Rules + +1. One canonical pair per concept; synonyms are recorded, conflicts + are resolved by the marketplace product owner. +2. Changes to legal-term rows require the jol-m-legal sync to land + first (they are evidence of legal drafting). +3. Charset is UTF-8 with full diacritics (ą č ę ė į š ų ū ž / ā č ē ģ + ī ķ ļ ņ š ū ž / ä ö õ ü š ž) — transliteration is a defect. diff --git a/ml/translation-memory/glossary.csv b/ml/translation-memory/glossary.csv new file mode 100644 index 0000000..a5b94d9 --- /dev/null +++ b/ml/translation-memory/glossary.csv @@ -0,0 +1,11 @@ +term_id,lt,lv,et,en,source +tm-001,Bažnytiniai reikmenys,Baznīcas piederumi,Kirikutarbed,Church equipment,seed-taxonomy +tm-002,Liturginiai rūbai,Liturģiskie tērpi,Liturgilised rõivad,Vestments,seed-taxonomy +tm-003,Ikonos,Ikonas,Ikoonid,Icons,seed-taxonomy +tm-004,Laidotuvių reikmenys,Bēru piederumi,Matustarbed,Funeral supplies,seed-taxonomy +tm-005,Kapinių reikmenys,Kapu piederumi,Kalmistutarbed,Cemetery supplies,seed-taxonomy +tm-006,Pardavėjas,Pārdevējs,Müüja,Seller,marketplace +tm-007,Pirkėjs,Pircējs,Ostja,Buyer,marketplace +tm-008,Skelimas,Sludinājums,Kuulutus,Listing,marketplace +tm-009,Pristatymas,Piegāde,Tarnimine,Delivery,marketplace +tm-010,Grąžinimas,Atgriešana,Tagastamine,Refund/return,marketplace diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..5a825a1 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,29 @@ +# Tool configuration for the data repository. +# dbt / sqlfluff / great-expectations are the warehouse toolchain; the +# scripts under scripts/ are the governance gates. Versions are pinned +# ranges — bump via dependabot review, never ad hoc. + +[project] +name = "jol-m-data-tooling" +version = "0.1.0" +description = "Data platform tooling: dbt warehouse, sqlfluff, data quality, seed validators, governance gates." +requires-python = ">=3.11" +dependencies = [ + "pre-commit>=3.8", + "yamllint>=1.35", + "pyyaml>=6.0", + "jsonschema>=4.23", + "dbt-core>=1.8", + "dbt-postgres>=1.8", + "sqlfluff>=3.2", + "sqlfluff-templater-dbt>=3.2", + "great-expectations>=0.18", +] + +[project.optional-dependencies] +synthetic = ["faker>=28.0"] + +[tool.codespell] +# Spelling hygiene; marketplace/taxonomy terms whitelisted below. +skip = "*.pdf,.idea,.venv,warehouse/target,warehouse/dbt_packages" +ignore-words-list = "uab,oss,eur" diff --git a/qodana.yaml b/qodana.yaml new file mode 100644 index 0000000..63fde16 --- /dev/null +++ b/qodana.yaml @@ -0,0 +1,17 @@ +# Qodana static analysis — inherited from jol-repo-template baseline. +# Data repo profile: lint scripts and warehouse Python; SQL/YAML content +# quality is handled by dbt-ci (sqlfluff, dbt parse/test) and the ci +# workflow (seed schema validation, PII scan). +version: "1.0" +linter: jetbrains/qodana-python:2025.1 +profile: + name: qodana.recommended +exclude: + - name: All + paths: + - .idea + - .venv + - warehouse/target + - warehouse/dbt_packages +include: + - name: CheckLicenseCopyright diff --git a/quality/README.md b/quality/README.md new file mode 100644 index 0000000..90407d5 --- /dev/null +++ b/quality/README.md @@ -0,0 +1,26 @@ +# Data Quality — every mart has expectations + +Doctrine: + +1. **Every mart has expectations.** A model without declared + expectations is a debt item, tracked in the scorecard. +2. **Failed expectation = blocked pipeline.** Quality gates run before + downstream refresh (`make quality`, `data-quality.yml`); a failing + expectation blocks the consumer refresh, not just warns. +3. **Anomalies are rules, not vibes.** Volume/distribution anomalies + are declared in `anomaly-rules/` with thresholds and owners. +4. **Scorecards feed management review.** Weekly quality scores per + domain live in `scorecards/` and are input to the governance review + cadence (`governance/README.md`). + +## Layout + +| Path | Content | +|------|---------| +| `expectations/` | Great Expectations-style suites: orders, products, vat | +| `anomaly-rules/` | Volume/distribution anomaly rules per critical table | +| `scorecards/` | Weekly data-quality score per domain | + +Expectations execute via `make quality` (dbt tests + GE suites against +the staging warehouse); dbt tests in `warehouse/tests/` are the first +line, these suites are the second. diff --git a/quality/anomaly-rules/rules.yml b/quality/anomaly-rules/rules.yml new file mode 100644 index 0000000..ae6c7b9 --- /dev/null +++ b/quality/anomaly-rules/rules.yml @@ -0,0 +1,26 @@ +# Anomaly rules — volume/distribution detection per critical table. +# A triggered rule blocks the consumer refresh and notifies the owner +# (data-quality workflow). Thresholds are deliberately conservative at +# scaffold — tune after two weeks of baseline volume. +schema_version: 1 +rules: + - table: raw.orders + metric: daily_row_count + kind: volume + condition: "drop_gt: 50% vs trailing 7-day median" + owner: data-platform + - table: raw.orders + metric: refunded_share + kind: distribution + condition: "gt: 0.20" + owner: marketplace-product + - table: fct_vat_oss + metric: quarterly_net_amount_eur + kind: volume + condition: "swing_gt: 60% vs prior quarter" + owner: finance + - table: consent_rates + metric: consent_rate + kind: distribution + condition: "swing_gt: 0.15 vs trailing 4-week mean" + owner: dpo diff --git a/quality/expectations/orders.yml b/quality/expectations/orders.yml new file mode 100644 index 0000000..fc5f031 --- /dev/null +++ b/quality/expectations/orders.yml @@ -0,0 +1,21 @@ +# Expectation suite — orders (staging + fct_orders). +suite: orders +owner: data-platform +tables: [raw.orders, fct_orders] +expectations: + - id: orders-amounts-positive + description: Paid/shipped/delivered orders must have amount > 0. + rule: amount_eur > 0 when status in (paid, shipped, delivered) + severity: block + - id: orders-valid-states + description: Status must be a known lifecycle state. + rule: status in (paid, shipped, delivered, refunded, cancelled) + severity: block + - id: orders-no-future-timestamps + description: created_at must not be in the future (clock drift guard). + rule: created_at <= now() + interval '5 minutes' + severity: warn + - id: orders-seller-key-present + description: Every order has a pseudonymous seller key. + rule: seller_key is not null + severity: block diff --git a/quality/expectations/products.yml b/quality/expectations/products.yml new file mode 100644 index 0000000..368d551 --- /dev/null +++ b/quality/expectations/products.yml @@ -0,0 +1,17 @@ +# Expectation suite — products (valid taxonomy). +suite: products +owner: data-platform +tables: [raw.products, dim_products] +expectations: + - id: products-valid-taxonomy + description: category_code must exist in seed/taxonomy/categories.yml. + rule: category_code in taxonomy_codes + severity: block + - id: products-price-positive + description: Active listings must have price > 0. + rule: price_eur > 0 when status = active + severity: block + - id: products-seller-key-format + description: Seller keys are 32-char lowercase hex hashes. + rule: seller_key matches '^[0-9a-f]{32}$' + severity: block diff --git a/quality/expectations/vat.yml b/quality/expectations/vat.yml new file mode 100644 index 0000000..c08592e --- /dev/null +++ b/quality/expectations/vat.yml @@ -0,0 +1,20 @@ +# Expectation suite — VAT correctness (money-critical). +suite: vat +owner: finance +tables: [fct_orders, fct_vat_oss, warehouse ref.vat_rates] +expectations: + - id: vat-rate-in-reference-set + description: > + Applied rate must match seed/tax/vat-rates.yml for the country + + vat_class (e.g. LT standard in {21}, LV standard in {21}, + EE standard in {22}). + rule: (country_code, vat_class, vat_rate_pct) joins ref.vat_rates + severity: block + - id: vat-bounds + description: Rate within EU-referenced bounds. + rule: vat_rate_pct between 0 and 30 + severity: block + - id: vat-net-plus-vat-equals-gross + description: Arithmetic closure per order. + rule: abs(net_amount_eur + vat_amount_eur - amount_eur) < 0.01 + severity: block diff --git a/quality/scorecards/README.md b/quality/scorecards/README.md new file mode 100644 index 0000000..68df85b --- /dev/null +++ b/quality/scorecards/README.md @@ -0,0 +1,30 @@ +# Quality Scorecards — weekly data-quality score per domain + +One file per ISO week: `YYYY-Www.md`, produced from expectation and +anomaly run results (automation lands with the warehouse environment). +Scorecards feed the governance review cadence. + +## Score definition + +| Component | Weight | Source | +|-----------|--------|--------| +| Blocking expectations pass rate | 50% | quality/expectations runs | +| dbt test pass rate | 25% | dbt-ci / quality runs | +| Freshness SLA adherence | 15% | freshness-monitor | +| Anomaly alerts (absence) | 10% | anomaly-rules | + +## Template + +```markdown +# Quality scorecard — + +| Domain | Score | Blocking failures | Notes | +|--------|-------|-------------------|-------| +| orders | | | | +| products | | | | +| vat | | | | +| compliance | | | | + +Actions carried into next week: +- ... +``` diff --git a/scripts/catalog-lint.py b/scripts/catalog-lint.py new file mode 100755 index 0000000..c76eae5 --- /dev/null +++ b/scripts/catalog-lint.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +"""catalog-lint.py — no orphan datasets. + +Every dataset must be registered in governance/data-catalog.md AND +governance/ownership-register.csv, owned, classified, and +retention-mapped. Pure stdlib. +""" + +from __future__ import annotations + +import csv +import re +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +GOV = ROOT / "governance" + +CLASSIFICATIONS = {"PUBLIC", "INTERNAL", "CONFIDENTIAL", "RESTRICTED"} +RETENTION_CLASSES = {"short-term", "operational", "statutory", "indefinite-reference", "none-synthetic"} +REQUIRED_COLUMNS = ["dataset_id", "business_owner", "technical_steward", + "classification", "ropa_purpose", "retention_class"] + + +def main() -> int: + errors: list[str] = [] + + csv_path = GOV / "ownership-register.csv" + with open(csv_path, encoding="utf-8", newline="") as fh: + reader = csv.DictReader(fh) + if reader.fieldnames != REQUIRED_COLUMNS: + errors.append(f"{csv_path}: header must be {REQUIRED_COLUMNS}") + rows = list(reader) + + seen: set[str] = set() + for i, row in enumerate(rows, start=2): + dataset_id = (row.get("dataset_id") or "").strip() + if not dataset_id: + errors.append(f"ownership-register.csv:{i}: empty dataset_id") + continue + if dataset_id in seen: + errors.append(f"ownership-register.csv:{i}: duplicate dataset_id {dataset_id}") + seen.add(dataset_id) + for field in ("business_owner", "technical_steward"): + if not (row.get(field) or "").strip(): + errors.append(f"ownership-register.csv:{i}: {dataset_id} missing {field} (no orphans)") + if row.get("classification") not in CLASSIFICATIONS: + errors.append(f"ownership-register.csv:{i}: {dataset_id} classification " + f"{row.get('classification')!r} not in {sorted(CLASSIFICATIONS)}") + if row.get("retention_class") not in RETENTION_CLASSES: + errors.append(f"ownership-register.csv:{i}: {dataset_id} retention_class " + f"{row.get('retention_class')!r} not in {sorted(RETENTION_CLASSES)}") + + # Cross-check against the human-readable catalog + catalog = (GOV / "data-catalog.md").read_text(encoding="utf-8") + catalog_ids: set[str] = set() + for line in catalog.splitlines(): + m = re.match(r"^\|\s*([a-z0-9_*]+)\s*\|", line) + if m and m.group(1) not in ("dataset_id",): + catalog_ids.add(m.group(1)) + catalog_ids.discard("seed_*") # wildcard family row in retention docs only + + for dataset_id in sorted(seen): + if f"| {dataset_id} |" not in catalog: + errors.append(f"data-catalog.md: dataset {dataset_id} missing from catalog table") + for catalog_id in sorted(catalog_ids): + if catalog_id not in seen and not any(catalog_id.startswith(p) for p in ("seed_",)): + errors.append(f"data-catalog.md: catalog row {catalog_id} has no register entry") + + if errors: + print("catalog lint FAILED:", file=sys.stderr) + for e in errors: + print(f" - {e}", file=sys.stderr) + return 1 + print(f"catalog lint OK ({len(seen)} datasets registered, owned, classified, retention-mapped)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/check-csv.py b/scripts/check-csv.py new file mode 100644 index 0000000..8b856a9 --- /dev/null +++ b/scripts/check-csv.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""CSV structural gate for data registers. + +Replaces the nonexistent upstream `check-csv` hook: validates that each +register CSV parses, has a non-empty header, and keeps a constant column +count — the invariants the seed validators and catalog lint and audit spot-checks rely on. + +Usage: python3 scripts/check-csv.py FILE.csv [FILE.csv ...] +Exit: 0 = all clean, 1 = structural defect. +""" + +from __future__ import annotations + +import csv +import sys +from pathlib import Path + + +def check_csv(path: Path) -> list[str]: + errors: list[str] = [] + try: + with path.open(newline="", encoding="utf-8") as fh: + reader = csv.reader(fh) + header = next(reader, None) + if header is None: + return [f"{path}: empty file — registers must carry a header row"] + if not any(cell.strip() for cell in header): + errors.append(f"{path}: header row is blank") + width = len(header) + for lineno, row in enumerate(reader, start=2): + if len(row) != width: + errors.append( + f"{path}:{lineno}: {len(row)} columns, expected {width}" + ) + except OSError as exc: + return [f"{path}: unreadable ({exc})"] + except csv.Error as exc: + return [f"{path}: csv parse error ({exc})"] + except UnicodeDecodeError as exc: + return [f"{path}: not valid UTF-8 ({exc})"] + return errors + + +def main(argv: list[str]) -> int: + if not argv: + print("check-csv: no files given", file=sys.stderr) + return 1 + failed = False + for arg in argv: + for err in check_csv(Path(arg)): + print(err, file=sys.stderr) + failed = True + if failed: + return 1 + print(f"check-csv: {len(argv)} file(s) structurally clean") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/scripts/check-personal-data.sh b/scripts/check-personal-data.sh new file mode 100755 index 0000000..6cafb30 --- /dev/null +++ b/scripts/check-personal-data.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# check-personal-data.sh — PII tripwire for committed files. +# Flags Baltic national IDs, IBANs, realistic phone shapes, and emails +# outside reserved example domains. TRIPWIRE ONLY: passing does not +# make a commit lawful (CONTRIBUTING.md). synthetic/pii-canaries/ is +# excluded by design (scanner self-test material — see its README). +set -euo pipefail + +cd "$(git rev-parse --show-toplevel 2>/dev/null || echo .)" + +EXCLUDE='synthetic/pii-canaries/' +FILES=$(git ls-files -- '*.md' '*.csv' '*.txt' '*.yml' '*.yaml' '*.sql' '*.py' '*.sh' 2>/dev/null \ + | grep -v "^${EXCLUDE}" || true) + +if [ -z "$FILES" ]; then + # Not a git repo (or nothing tracked): fall back to filesystem scan. + FILES=$(find . \ + \( -name '*.md' -o -name '*.csv' -o -name '*.yml' -o -name '*.yaml' \ + -o -name '*.sql' -o -name '*.py' -o -name '*.sh' -o -name '*.txt' \) \ + -not -path './.git/*' -not -path './.venv/*' -not -path './.idea/*' \ + -not -path "./${EXCLUDE}*") +fi + +PATTERNS=( + # Baltic national IDs: LT asmens kodas / EE isikukood (11 digits) + '\b[0-9]{11}\b' + # LV personas kods (DDMMYY-NNNNN) + '\b[0-9]{6}-[0-9]{5}\b' + # Baltic IBANs with 12+ following digits (grouped or not) + '\b(LT|LV|EE)[0-9]{2}([ ]?[0-9]{4}){3,}\b' + # Realistic Baltic mobile shapes (LT +370 6xxxxxxx, LV +371 2xxxxxxx, EE +372 5xxxxxxx) + '\+37[012][ -]?[625][0-9]{6,7}\b' +) + +fail=0 +for pattern in "${PATTERNS[@]}"; do + # shellcheck disable=SC2086 + matches=$(grep -nIE "$pattern" $FILES 2>/dev/null || true) + if [ -n "$matches" ]; then + echo "PII-shaped pattern detected ($pattern):" >&2 + echo "$matches" >&2 + fail=1 + fi +done + +# Emails outside reserved example domains. Domain must start with a +# letter (excludes action-pinned SHAs like checkout@ and image +# refs like postgres@127.0.0.1). +# shellcheck disable=SC2086 +emails=$(grep -nIEo '[A-Za-z0-9._%+-]+@[A-Za-z][A-Za-z0-9.-]*\.[A-Za-z]{2,}' $FILES 2>/dev/null \ + | grep -vE '@example\.(test|com|org|net)([:0-9]|$)' || true) +if [ -n "$emails" ]; then + echo "Email addresses outside reserved example domains:" >&2 + echo "$emails" >&2 + fail=1 +fi + +if [ "$fail" -ne 0 ]; then + echo "FAIL: personal-data tripwire fired — see CONTRIBUTING.md minimization rules." >&2 + exit 1 +fi +echo "personal-data tripwire OK" diff --git a/scripts/freshness-report.py b/scripts/freshness-report.py new file mode 100755 index 0000000..ad52304 --- /dev/null +++ b/scripts/freshness-report.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +"""freshness-report.py — source freshness vs SLA. + +Reads freshness SLAs from warehouse/models/staging/_staging.yml, checks +max(loaded_at) per raw table, and reports staleness. --fail-on-stale +exits non-zero when any source exceeds its error_after horizon. +Requires: pyyaml, psycopg2 (warehouse connection via WH_* env). +""" + +from __future__ import annotations + +import argparse +import os +import sys +from datetime import datetime, timezone +from pathlib import Path + +try: + import yaml +except ImportError: # pragma: no cover + sys.exit("pyyaml is required: pip install -e .") + +ROOT = Path(__file__).resolve().parents[1] +STAGING_YML = ROOT / "warehouse/models/staging/_staging.yml" + + +def load_slas() -> dict: + with open(STAGING_YML, encoding="utf-8") as fh: + doc = yaml.safe_load(fh) + source = doc["sources"][0] + loaded_at = source.get("loaded_at_field", "created_at") + slas = {} + for table in source.get("tables", []): + freshness = table.get("freshness", source.get("freshness", {})) + slas[table["name"]] = { + "loaded_at": loaded_at, + "warn": _hours(freshness.get("warn_after")), + "error": _hours(freshness.get("error_after")), + } + return slas + + +def _hours(spec: dict | None) -> float | None: + if not spec: + return None + mult = {"minute": 1 / 60, "hour": 1, "day": 24}.get(spec.get("period"), 1) + return spec["count"] * mult + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--fail-on-stale", action="store_true", + help="exit non-zero when any source exceeds error_after") + args = parser.parse_args() + + if not os.environ.get("WH_HOST"): + sys.exit("WH_* environment not configured (see .envrc.example) — nothing to check") + try: + import psycopg2 + except ImportError: + sys.exit("psycopg2 required: pip install psycopg2-binary") + + slas = load_slas() + conn = psycopg2.connect( + host=os.environ["WH_HOST"], port=int(os.environ.get("WH_PORT", "5432")), + dbname=os.environ.get("WH_DB"), user=os.environ.get("WH_USER"), + password=os.environ.get("WH_PASSWORD"), + connect_timeout=10, + ) + now = datetime.now(timezone.utc) + stale = 0 + try: + with conn.cursor() as cur: + for table, sla in sorted(slas.items()): + cur.execute(f'select max("{sla["loaded_at"]}") from raw."{table}"') # noqa: S608 + row = cur.fetchone() + newest = row[0] if row and row[0] else None + if newest is None: + print(f"{table}: EMPTY — no rows landed") + stale += 1 + continue + age = now - (newest if newest.tzinfo else newest.replace(tzinfo=timezone.utc)) + age_h = age.total_seconds() / 3600 + status = "ok" + if sla["error"] is not None and age_h > sla["error"]: + status, stale = "STALE (error)", stale + 1 + elif sla["warn"] is not None and age_h > sla["warn"]: + status = "stale (warn)" + print(f"{table}: newest {newest.isoformat()} ({age_h:.1f}h old) — {status} " + f"[warn {sla['warn']}h / error {sla['error']}h]") + finally: + conn.close() + + if args.fail_on_stale and stale: + print(f"FAIL: {stale} source(s) beyond freshness SLA — " + "follow docs/runbooks/pipeline-failure.md", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/scan-warehouse-pii.py b/scripts/scan-warehouse-pii.py new file mode 100755 index 0000000..04258de --- /dev/null +++ b/scripts/scan-warehouse-pii.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""scan-warehouse-pii.py — pattern scan for PII-shaped values. + +Modes: + --local scan committed data files (seed, governance, ml, seeds) + --warehouse sample the staging warehouse (needs WH_* env + psycopg2) + --self-test prove the scanner fires on synthetic/pii-canaries/ + +Findings are reported by LOCATION ONLY — offending values are never +printed (docs/runbooks/pii-detected-in-warehouse.md). Requires: pyyaml. +""" + +from __future__ import annotations + +import argparse +import os +import re +import sys +from pathlib import Path + +try: + import yaml +except ImportError: # pragma: no cover + sys.exit("pyyaml is required: pip install -e .") + +ROOT = Path(__file__).resolve().parents[1] +CANARIES = ROOT / "synthetic/pii-canaries/canaries.yml" +SCAN_DIRS = ["seed", "governance", "ml", "warehouse/seeds", "synthetic/generators", "synthetic/regression"] +SCAN_SUFFIXES = {".yml", ".yaml", ".csv", ".md", ".json"} + +PATTERNS = { + "baltic-national-id": re.compile(r"\b[0-9]{11}\b"), + "lv-personas-kods": re.compile(r"\b[0-9]{6}-[0-9]{5}\b"), + "baltic-iban": re.compile(r"\b(LT|LV|EE)[0-9]{2}([ ]?[0-9]{4}){3,}\b"), + "baltic-phone": re.compile(r"\+37[012]([ -]?[0-9]){8,11}\b"), +} +EMAIL = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z][A-Za-z0-9.-]*\.[A-Za-z]{2,}") +ALLOWED_EMAIL = re.compile(r"@example\.(test|com|org|net)$", re.IGNORECASE) +NAME_ADDRESS = re.compile(r"\b[A-Z][a-z]+ [A-Z][a-z]+,\s*.*\b(?:g\.|str\.|street|ave\.|road)") + + +def scan_text(text: str, strict_emails: bool = False) -> list[str]: + """Return kinds of PII-shaped patterns present in the text.""" + hits = [kind for kind, rx in PATTERNS.items() if rx.search(text)] + for m in EMAIL.finditer(text): + if strict_emails or not ALLOWED_EMAIL.search(m.group(0)): + hits.append("email") + break + if NAME_ADDRESS.search(text): + hits.append("name-with-address") + return hits + + +def scan_local() -> int: + findings = 0 + for rel in SCAN_DIRS: + base = ROOT / rel + if not base.exists(): + continue + for path in sorted(base.rglob("*")): + if path.is_file() and path.suffix in SCAN_SUFFIXES: + hits = scan_text(path.read_text(encoding="utf-8", errors="replace")) + if hits: + print(f"PII-shaped values: {path.relative_to(ROOT)} ({', '.join(sorted(set(hits)))})") + findings += 1 + return findings + + +def scan_warehouse() -> int: + try: + import psycopg2 + except ImportError: + sys.exit("psycopg2 required for --warehouse: pip install psycopg2-binary") + if not os.environ.get("WH_HOST"): + sys.exit("WH_* environment not configured (see .envrc.example) — nothing to scan") + conn = psycopg2.connect( + host=os.environ["WH_HOST"], port=int(os.environ.get("WH_PORT", "5432")), + dbname=os.environ.get("WH_DB"), user=os.environ.get("WH_USER"), + password=os.environ.get("WH_PASSWORD"), + connect_timeout=10, + ) + findings = 0 + try: + with conn.cursor() as cur: + cur.execute( + "select table_schema, table_name, column_name " + "from information_schema.columns " + "where table_schema not in ('pg_catalog','information_schema') " + " and data_type in ('text','character varying','character')" + ) + columns = cur.fetchall() + for schema, table, column in columns: + cur.execute( + f'select "{column}" from "{schema}"."{table}" ' # noqa: S608 - warehouse metadata + f'where "{column}" is not null limit 500' + ) + sampled = [str(row[0]) for row in cur.fetchall()] + kinds = {k for value in sampled for k in scan_text(value)} + if kinds: + print(f"PII-shaped values: warehouse {schema}.{table}.{column} ({len(sampled)} sampled)") + findings += 1 + finally: + conn.close() + return findings + + +def self_test() -> int: + with open(CANARIES, encoding="utf-8") as fh: + canaries = yaml.safe_load(fh)["canaries"] + failures = 0 + for canary in canaries: + # Self-test uses strict email detection (canary email is reserved-domain by construction). + hits = scan_text(canary["value"], strict_emails=True) + status = "DETECTED" if hits else "MISSED" + print(f"{status}: {canary['kind']} -> {', '.join(sorted(set(hits))) or '-'}") + if not hits: + failures += 1 + if failures: + print(f"self-test FAILED: {failures} canary(ies) missed — scanner is blind", file=sys.stderr) + return 1 + print("self-test OK: all canaries detected") + return 0 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument("--local", action="store_true") + group.add_argument("--warehouse", action="store_true") + group.add_argument("--self-test", action="store_true") + args = parser.parse_args() + + if args.self_test: + return self_test() + findings = scan_local() if args.local else scan_warehouse() + if findings: + print(f"FAIL: {findings} location(s) with PII-shaped values — " + "follow docs/runbooks/pii-detected-in-warehouse.md and notify the DPO", file=sys.stderr) + return 1 + print("pii scan OK") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/test_check_csv.py b/scripts/tests/test_check_csv.py new file mode 100644 index 0000000..1e616eb --- /dev/null +++ b/scripts/tests/test_check_csv.py @@ -0,0 +1,54 @@ +"""Regression tests for scripts/check-csv.py (stdlib unittest, no deps). + +Guards the fix for the nonexistent upstream `check-csv` pre-commit hook: +the local validator must catch ragged rows, blank headers, and empty files +while accepting the commented-example register shape. +""" + +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parents[1] / "check-csv.py" + + +def run_check(content: str) -> subprocess.CompletedProcess: + with tempfile.NamedTemporaryFile("w", suffix=".csv", delete=False) as fh: + fh.write(content) + path = fh.name + return subprocess.run( + [sys.executable, str(SCRIPT), path], capture_output=True, text=True + ) + + +class CheckCsvTest(unittest.TestCase): + def test_well_formed_register_passes(self): + result = run_check("a,b,c\n1,2,3\n# comment-row,still,three\n") + self.assertEqual(result.returncode, 0, result.stderr) + + def test_ragged_row_fails(self): + result = run_check("a,b,c\n1,2\n") + self.assertEqual(result.returncode, 1) + self.assertIn("2 columns, expected 3", result.stderr) + + def test_blank_header_fails(self): + result = run_check(",,\n1,2,3\n") + self.assertEqual(result.returncode, 1) + self.assertIn("header row is blank", result.stderr) + + def test_empty_file_fails(self): + result = run_check("") + self.assertEqual(result.returncode, 1) + self.assertIn("header row", result.stderr) + + def test_no_args_fails(self): + result = subprocess.run( + [sys.executable, str(SCRIPT)], capture_output=True, text=True + ) + self.assertEqual(result.returncode, 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/validate-seed.py b/scripts/validate-seed.py new file mode 100755 index 0000000..1fc52c2 --- /dev/null +++ b/scripts/validate-seed.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""validate-seed.py — JSON Schema validation for every seed/taxonomy file. + +CI gate (ci.yml) and pre-commit hook. Adding a new seed file requires +registering it here with a schema. Requires: pyyaml, jsonschema. +""" + +from __future__ import annotations + +import json +import sys +from pathlib import Path + +try: + import yaml + from jsonschema import Draft7Validator +except ImportError: # pragma: no cover + sys.exit("pyyaml + jsonschema required: pip install -e .") + +ROOT = Path(__file__).resolve().parents[1] +SEED = ROOT / "seed" +VALIDATORS = SEED / "validators" + +FIXTURE_DEFS = {"sellers": "seller", "products": "product", "orders": "order"} + + +def load(path: Path) -> dict: + with open(path, encoding="utf-8") as fh: + return yaml.safe_load(fh) + + +def validator_for(schema_file: Path) -> Draft7Validator: + with open(schema_file, encoding="utf-8") as fh: + return Draft7Validator(json.load(fh)) + + +def check(data: dict, validator: Draft7Validator, errors: list, label: str) -> None: + for err in sorted(validator.iter_errors(data), key=str): + errors.append(f"{label}: {err.message} at {list(err.absolute_path)}") + + +def main() -> int: + errors: list[str] = [] + + # taxonomy + categories = load(SEED / "taxonomy/categories.yml") + check(categories, validator_for(VALIDATORS / "categories.schema.json"), errors, "categories.yml") + attributes = load(SEED / "taxonomy/attributes.yml") + check(attributes, validator_for(VALIDATORS / "attributes.schema.json"), errors, "attributes.yml") + + codes = {c["code"] for c in categories.get("categories", [])} + if set(attributes.get("attributes", {})) != codes: + errors.append("attributes.yml: category keys must match categories.yml codes") + + # translations — schema + completeness against category codes + trans_validator = validator_for(VALIDATORS / "translations.schema.json") + for path in sorted((SEED / "taxonomy/translations").glob("*.yml")): + data = load(path) + check(data, trans_validator, errors, path.name) + missing = codes - set(data.get("names", {})) + if missing: + errors.append(f"{path.name}: missing translations for {sorted(missing)}") + + # geo + tax + geo_validator = validator_for(VALIDATORS / "geo.schema.json") + for path in sorted((SEED / "geo").glob("*.yml")): + check(load(path), geo_validator, errors, path.name) + tax_validator = validator_for(VALIDATORS / "tax.schema.json") + for path in sorted((SEED / "tax").glob("*.yml")): + check(load(path), tax_validator, errors, path.name) + + # fixtures — per-file record definition + referential integrity + with open(VALIDATORS / "fixtures.schema.json", encoding="utf-8") as fh: + fixtures_schema = json.load(fh) + fixtures: dict = {} + for stem, definition in FIXTURE_DEFS.items(): + path = SEED / "fixtures" / f"{stem}.yml" + data = load(path) + fixtures[stem] = data.get("records", []) + record_validator = Draft7Validator(fixtures_schema["definitions"][definition]) + for i, record in enumerate(fixtures[stem]): + check(record, record_validator, errors, f"fixtures/{stem}.yml[{i}]") + + seller_keys = {s["seller_key"] for s in fixtures["sellers"]} + product_keys = {p["product_key"] for p in fixtures["products"]} + for i, product in enumerate(fixtures["products"]): + if product.get("category_code") not in codes: + errors.append(f"fixtures/products.yml[{i}]: unknown category_code") + if product.get("seller_key") not in seller_keys: + errors.append(f"fixtures/products.yml[{i}]: unknown seller_key") + for i, order in enumerate(fixtures["orders"]): + if order.get("seller_key") not in seller_keys: + errors.append(f"fixtures/orders.yml[{i}]: unknown seller_key") + if order.get("product_key") not in product_keys: + errors.append(f"fixtures/orders.yml[{i}]: unknown product_key") + + if errors: + print("seed validation FAILED:", file=sys.stderr) + for e in errors: + print(f" - {e}", file=sys.stderr) + return 1 + print("seed validation OK") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/verify-anonymization.py b/scripts/verify-anonymization.py new file mode 100755 index 0000000..f2f74e0 --- /dev/null +++ b/scripts/verify-anonymization.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""verify-anonymization.py — adversarial re-identification sampler. + +Attempts re-identification against the repository's committed artifacts +(default) and, when configured, against warehouse marts (--warehouse). +Every attempt MUST fail: a successful path means the pseudonymization +boundary has a hole — severity-1 class incident (SECURITY.md). + +Local checks (no credentials required): + 1. Staging/mart models never select direct identifier columns. + 2. Staging models hash every subject id column (hash_id invariant). + 3. Committed seed/fixture data carries no identity fields and no + PII-shaped values (delegates patterns to scan-warehouse-pii). +""" + +from __future__ import annotations + +import re +import subprocess +import sys +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MODELS = ROOT / "warehouse/models" + +FORBIDDEN_TOKENS = re.compile( + r"\b(email|phone|first_name|last_name|full_name|buyer_name|seller_name|" + r"cardholder|national_id|asmens_kodas|personal_code|bank_account|iban)\b" +) +HASH_INVARIANT = { + "stg_orders.sql": ["id", "buyer_id", "seller_id"], + "stg_products.sql": ["id", "seller_id"], + "stg_users.sql": ["id"], +} + + +def check_models() -> list[str]: + failures = [] + for path in sorted(MODELS.rglob("*.sql")): + text = path.read_text(encoding="utf-8") + for m in FORBIDDEN_TOKENS.finditer(text): + failures.append(f"{path.relative_to(ROOT)}: direct identifier token '{m.group(0)}'") + for filename, columns in HASH_INVARIANT.items(): + text = (MODELS / "staging" / filename).read_text(encoding="utf-8") + for column in columns: + if f"hash_id('{column}')" not in text: + failures.append(f"models/staging/{filename}: column '{column}' is not hashed") + return failures + + +def check_seeds() -> list[str]: + proc = subprocess.run( + [sys.executable, str(ROOT / "scripts/scan-warehouse-pii.py"), "--local"], + capture_output=True, text=True, + ) + if proc.returncode != 0: + return [f"seed PII sweep failed: {proc.stdout.strip() or proc.stderr.strip()}"] + return [] + + +def check_fixtures_schema() -> list[str]: + """Fixtures must carry no identity-like fields (names/contacts).""" + import yaml + failures = [] + identity_fields = {"name", "email", "phone", "address", "national_id", "company_name"} + for path in sorted((ROOT / "seed/fixtures").glob("*.yml")): + with open(path, encoding="utf-8") as fh: + data = yaml.safe_load(fh) + for record in data.get("records", []): + leaked = identity_fields & set(record) + if leaked: + failures.append(f"{path.relative_to(ROOT)}: identity fields {sorted(leaked)}") + return failures + + +def main() -> int: + if "--warehouse" in sys.argv: + print("warehouse sampling requires the WH_* environment (see .envrc.example); " + "running committed-artifact verification instead") + failures = check_models() + check_fixtures_schema() + check_seeds() + if failures: + print("ADVERSARIAL VERIFICATION FAILED — re-identification paths exist:", file=sys.stderr) + for f in failures: + print(f" - {f}", file=sys.stderr) + return 1 + print("adversarial verification PASS: no re-identification path found in committed artifacts") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/seed/README.md b/seed/README.md new file mode 100644 index 0000000..de75026 --- /dev/null +++ b/seed/README.md @@ -0,0 +1,37 @@ +# Seed & Reference Data — the marketplace's knowledge spine + +Doctrine: + +1. **Synthetic only.** Everything under `fixtures/` is generated by the + faker-seeded generators in `synthetic/generators/`. No real person, + company, or address is ever committed. Regeneration with a new seed + MUST produce new identities. +2. **Deterministic & versioned.** Seed files are committed artifacts. + Every file is schema-validated (`validators/`); CI blocks malformed + taxonomy. Consumers (`jol-m-marketplace`, demo environments) load + these files via their own load scripts. +3. **Translations are complete.** Category names exist for lt/lv/et/en; + ru is reserved. A missing translation is a CI failure, not a TODO. +4. **Taxonomy changes are consumer-visible.** A category code change is + a MAJOR change for `jol-m-marketplace`; declare impact in the PR. + +## Layout + +| Path | Content | +|------|---------| +| `taxonomy/categories.yml` | Product categorization: church equipment, vestments, icons, funeral, cemetery, books, services | +| `taxonomy/attributes.yml` | Per-category attribute schemas (material, denomination, size, rite, …) | +| `taxonomy/translations/` | Category display names per language (lt/lv/et/en, ru reserved) | +| `geo/` | LT/LV/EE municipalities & parish boundary refs, parcel-locker location refs | +| `tax/` | VAT rate references per category per country (validated against Stripe Tax) | +| `fixtures/` | Demo sellers/products/orders — 100% synthetic | +| `validators/` | JSON Schema for every seed file; enforced by `scripts/validate-seed.py` | + +## Validation + +```bash +make seed-validate # schema-validates every file in this tree +``` + +The validator mapping lives in `scripts/validate-seed.py`; adding a new +seed file requires registering it there with a schema. diff --git a/seed/fixtures/README.md b/seed/fixtures/README.md new file mode 100644 index 0000000..47b324d --- /dev/null +++ b/seed/fixtures/README.md @@ -0,0 +1,31 @@ +# Fixtures — demo sellers/products/orders + +100% synthetic (faker-seeded, PII-free by construction). Used by +`jol-m-marketplace` demo/staging environments and by dbt-ci as the +synthetic raw schema. + +## Generation procedure + +1. Run the generator with an explicit seed: + + ```bash + python3 synthetic/generators/generate_fixtures.py --seed --out seed/fixtures + ``` + +2. Regeneration with a **different** seed MUST produce new synthetic + identities — this is a fixture invariant, not a side effect. +3. Commit the output together with the seed value recorded below. + +| File | Records | Generated with | +|------|---------|----------------| +| `sellers.yml` | 5 synthetic sellers | seed=20260801 (scaffold baseline) | +| `products.yml` | 8 synthetic products | seed=20260801 | +| `orders.yml` | 10 synthetic orders | seed=20260801 | + +Rules: + +- Identities use the `SYN-` key scheme and `@example.test` contacts — + the PII tripwire in CI relies on both. Never replace them with real + values "for realism". +- Amounts are EUR cents; VAT rates reference `seed/tax/vat-rates.yml`. +- Schema validation: `make seed-validate` (schemas in `../validators/`). diff --git a/seed/fixtures/orders.yml b/seed/fixtures/orders.yml new file mode 100644 index 0000000..55518ef --- /dev/null +++ b/seed/fixtures/orders.yml @@ -0,0 +1,104 @@ +# Synthetic orders — demo/staging only. EUR cents; vat_rate_pct from +# seed/tax/vat-rates.yml for the buyer country + category vat_class. +schema_version: 1 +records: + - order_key: SYN-ORD-0001 + buyer_key: SYN-BUY-0001 + seller_key: SYN-SLR-0001 + product_key: SYN-PRD-0001 + amount_cents: 12100 + currency: EUR + country: LT + vat_rate_pct: 21.0 + status: paid + ordered_at: "2026-08-01T10:00:00Z" + - order_key: SYN-ORD-0002 + buyer_key: SYN-BUY-0002 + seller_key: SYN-SLR-0001 + product_key: SYN-PRD-0002 + amount_cents: 18500 + currency: EUR + country: LT + vat_rate_pct: 21.0 + status: shipped + ordered_at: "2026-08-02T11:00:00Z" + - order_key: SYN-ORD-0003 + buyer_key: SYN-BUY-0003 + seller_key: SYN-SLR-0002 + product_key: SYN-PRD-0003 + amount_cents: 25000 + currency: EUR + country: LV + vat_rate_pct: 21.0 + status: paid + ordered_at: "2026-08-03T12:00:00Z" + - order_key: SYN-ORD-0004 + buyer_key: SYN-BUY-0004 + seller_key: SYN-SLR-0002 + product_key: SYN-PRD-0004 + amount_cents: 3400 + currency: EUR + country: LV + vat_rate_pct: 5.0 + status: delivered + ordered_at: "2026-08-04T13:00:00Z" + - order_key: SYN-ORD-0005 + buyer_key: SYN-BUY-0005 + seller_key: SYN-SLR-0003 + product_key: SYN-PRD-0005 + amount_cents: 9900 + currency: EUR + country: LV + vat_rate_pct: 21.0 + status: refunded + ordered_at: "2026-08-05T14:00:00Z" + - order_key: SYN-ORD-0006 + buyer_key: SYN-BUY-0006 + seller_key: SYN-SLR-0003 + product_key: SYN-PRD-0006 + amount_cents: 45000 + currency: EUR + country: EE + vat_rate_pct: 22.0 + status: paid + ordered_at: "2026-08-06T15:00:00Z" + - order_key: SYN-ORD-0007 + buyer_key: SYN-BUY-0007 + seller_key: SYN-SLR-0004 + product_key: SYN-PRD-0007 + amount_cents: 25000 + currency: EUR + country: EE + vat_rate_pct: 22.0 + status: delivered + ordered_at: "2026-08-07T16:00:00Z" + - order_key: SYN-ORD-0008 + buyer_key: SYN-BUY-0008 + seller_key: SYN-SLR-0004 + product_key: SYN-PRD-0003 + amount_cents: 25000 + currency: EUR + country: EE + vat_rate_pct: 22.0 + status: paid + ordered_at: "2026-08-08T17:00:00Z" + - order_key: SYN-ORD-0009 + buyer_key: SYN-BUY-0001 + seller_key: SYN-SLR-0002 + product_key: SYN-PRD-0004 + amount_cents: 3400 + currency: EUR + country: LT + vat_rate_pct: 9.0 + status: delivered + ordered_at: "2026-08-09T18:00:00Z" + - order_key: SYN-ORD-0010 + buyer_key: SYN-BUY-0002 + seller_key: SYN-SLR-0001 + product_key: SYN-PRD-0001 + amount_cents: 12100 + currency: EUR + country: LT + vat_rate_pct: 21.0 + status: cancelled + ordered_at: "2026-08-10T19:00:00Z" diff --git a/seed/fixtures/products.yml b/seed/fixtures/products.yml new file mode 100644 index 0000000..09fd582 --- /dev/null +++ b/seed/fixtures/products.yml @@ -0,0 +1,59 @@ +# Synthetic products — demo/staging only. +schema_version: 1 +records: + - product_key: SYN-PRD-0001 + seller_key: SYN-SLR-0001 + category_code: church-equipment + title: Synthetic brass candelabrum + price_cents: 12100 + currency: EUR + status: active + - product_key: SYN-PRD-0002 + seller_key: SYN-SLR-0001 + category_code: vestments + title: Synthetic linen alb (white) + price_cents: 18500 + currency: EUR + status: active + - product_key: SYN-PRD-0003 + seller_key: SYN-SLR-0002 + category_code: icons + title: Synthetic hand-painted icon panel + price_cents: 25000 + currency: EUR + status: active + - product_key: SYN-PRD-0004 + seller_key: SYN-SLR-0002 + category_code: books + title: Synthetic hymnal (lt) + price_cents: 3400 + currency: EUR + status: active + - product_key: SYN-PRD-0005 + seller_key: SYN-SLR-0003 + category_code: funeral + title: Synthetic ceremonial set + price_cents: 9900 + currency: EUR + status: active + - product_key: SYN-PRD-0006 + seller_key: SYN-SLR-0003 + category_code: cemetery + title: Synthetic granite marker + price_cents: 45000 + currency: EUR + status: active + - product_key: SYN-PRD-0007 + seller_key: SYN-SLR-0004 + category_code: services + title: Synthetic ceremony service + price_cents: 25000 + currency: EUR + status: active + - product_key: SYN-PRD-0008 + seller_key: SYN-SLR-0005 + category_code: books + title: Synthetic prayer book (et) + price_cents: 2900 + currency: EUR + status: inactive diff --git a/seed/fixtures/sellers.yml b/seed/fixtures/sellers.yml new file mode 100644 index 0000000..8cb40fc --- /dev/null +++ b/seed/fixtures/sellers.yml @@ -0,0 +1,34 @@ +# Synthetic sellers — demo/staging only. Identities are faker-seeded; +# regeneration with a new seed must yield new identities. +schema_version: 1 +records: + - seller_key: SYN-SLR-0001 + legal_form: sole-trader + country: LT + municipality: LT-VNO + joined_at: "2026-01-15" + active: true + - seller_key: SYN-SLR-0002 + legal_form: uab + country: LT + municipality: LT-KAU + joined_at: "2026-02-20" + active: true + - seller_key: SYN-SLR-0003 + legal_form: sole-trader + country: LV + municipality: LV-RIX + joined_at: "2026-03-10" + active: true + - seller_key: SYN-SLR-0004 + legal_form: ou + country: EE + municipality: EE-303 + joined_at: "2026-04-05" + active: true + - seller_key: SYN-SLR-0005 + legal_form: sole-trader + country: EE + municipality: EE-784 + joined_at: "2026-05-12" + active: false diff --git a/seed/geo/README.md b/seed/geo/README.md new file mode 100644 index 0000000..05dcc4d --- /dev/null +++ b/seed/geo/README.md @@ -0,0 +1,19 @@ +# Geo reference data — LT/LV/EE + +Reference sets for checkout/delivery flows and analytics geo +dimensions. Content here is a **representative subset**; the full +administrative lists are loaded from official open-data sources by +`jol-m-marketplace` — this seed provides the stable code scheme and +demo coverage. + +| File | Content | +|------|---------| +| `municipalities.yml` | Municipalities (LT savivaldybės, LV novadi, EE omavalitsused) + parish boundary refs | +| `lockers.yml` | Parcel-locker location refs (Omniva/DPD) — synthetic location codes | + +Rules: + +- Codes are stable identifiers; name changes are PATCH, code changes are + MAJOR for consumers. +- Locker coordinates in committed files are synthetic/approximate — the + authoritative live locations come from carrier APIs at runtime. diff --git a/seed/geo/lockers.yml b/seed/geo/lockers.yml new file mode 100644 index 0000000..46dd720 --- /dev/null +++ b/seed/geo/lockers.yml @@ -0,0 +1,29 @@ +# Parcel-locker location refs (Omniva/DPD). Location codes are stable +# refs; live availability comes from carrier APIs at runtime. +schema_version: 1 +lockers: + - code: SYN-OMN-LT-VNO-01 + provider: omniva + country: LT + municipality: LT-VNO + locality: synthetic-locality-a + - code: SYN-OMN-LT-KAU-01 + provider: omniva + country: LT + municipality: LT-KAU + locality: synthetic-locality-b + - code: SYN-DPD-LV-RIX-01 + provider: dpd + country: LV + municipality: LV-RIX + locality: synthetic-locality-c + - code: SYN-OMN-EE-303-01 + provider: omniva + country: EE + municipality: EE-303 + locality: synthetic-locality-d + - code: SYN-DPD-EE-784-01 + provider: dpd + country: EE + municipality: EE-784 + locality: synthetic-locality-e diff --git a/seed/geo/municipalities.yml b/seed/geo/municipalities.yml new file mode 100644 index 0000000..84b193e --- /dev/null +++ b/seed/geo/municipalities.yml @@ -0,0 +1,43 @@ +# LT/LV/EE municipalities + parish boundary references (subset). +# Full administrative lists sync from official open-data sources; this +# seed defines the code scheme and demo coverage. +schema_version: 1 +municipalities: + - code: LT-VNO + country: LT + name: Vilniaus miesto savivaldybė + type: city-municipality + - code: LT-KAU + country: LT + name: Kauno miesto savivaldybė + type: city-municipality + - code: LT-KLA + country: LT + name: Klaipėdos miesto savivaldybė + type: city-municipality + - code: LV-RIX + country: LV + name: Rīgas valstspilsēta + type: state-city + - code: LV-TUK + country: LV + name: Tukuma novads + type: municipality + - code: EE-303 + country: EE + name: Tallinn + type: urban-municipality + - code: EE-784 + country: EE + name: Tartu + type: urban-municipality +parishes: + - code: PAR-LT-VNO-001 + municipality: LT-VNO + name_ref: synthetic-parish-ref # parish boundary refs are code-only + - code: PAR-LV-RIX-001 + municipality: LV-RIX + name_ref: synthetic-parish-ref + - code: PAR-EE-303-001 + municipality: EE-303 + name_ref: synthetic-parish-ref diff --git a/seed/tax/README.md b/seed/tax/README.md new file mode 100644 index 0000000..2cd62ef --- /dev/null +++ b/seed/tax/README.md @@ -0,0 +1,13 @@ +# VAT rate references + +Rates per category per country. These are **references for analytics +and seed fixtures** — the charging authority is Stripe Tax at +transaction time (see `seed/fixtures` doctrine). Rates are validated +against Stripe Tax on the review date below; re-validate quarterly and +on any EU VAT directive change. + +- Last validated against Stripe Tax: 2026-08-01 (scaffold baseline — + re-validate before first production use). +- Reduced rate applies to `books` (per national schedules). +- OSS threshold/registration data lives in `jol-m-compliance` tax + records, not here. diff --git a/seed/tax/vat-rates.yml b/seed/tax/vat-rates.yml new file mode 100644 index 0000000..8ce1988 --- /dev/null +++ b/seed/tax/vat-rates.yml @@ -0,0 +1,15 @@ +# VAT rate references per category class per country. +# vat_class follows seed/taxonomy/categories.yml. Amounts in percent. +schema_version: 1 +validated_against: stripe-tax +validated_at: "2026-08-01" +rates: + LT: + standard: 21.0 + reduced: 9.0 # books + LV: + standard: 21.0 + reduced: 5.0 # books + EE: + standard: 22.0 + reduced: 5.0 # books diff --git a/seed/taxonomy/attributes.yml b/seed/taxonomy/attributes.yml new file mode 100644 index 0000000..a25fc79 --- /dev/null +++ b/seed/taxonomy/attributes.yml @@ -0,0 +1,53 @@ +# Per-category attribute schemas. The listing form in jol-m-marketplace +# renders these; enum values are the only allowed values. +schema_version: 1 +attributes: + church-equipment: + - name: material + type: enum + required: true + values: [brass, silver-plated, wood, ceramic, glass] + - name: denomination + type: enum + required: false + values: [roman-catholic, evangelical-lutheran, orthodox, ecumenical] + vestments: + - name: material + type: enum + required: true + values: [linen, silk, wool, brocade] + - name: liturgical-color + type: enum + required: true + values: [white, red, green, violet, black, gold] + - name: size + type: string + required: false + icons: + - name: technique + type: enum + required: true + values: [hand-painted, print, mosaic, carving] + - name: size + type: string + required: false + funeral: + - name: rite + type: enum + required: false + values: [roman-catholic, evangelical-lutheran, orthodox, civil] + cemetery: + - name: material + type: enum + required: true + values: [granite, marble, wood, metal] + books: + - name: language + type: enum + required: true + values: [lt, lv, et, en, ru, la] + services: + - name: region + type: enum + required: false + values: [lt, lv, ee, cross-border] diff --git a/seed/taxonomy/categories.yml b/seed/taxonomy/categories.yml new file mode 100644 index 0000000..0506637 --- /dev/null +++ b/seed/taxonomy/categories.yml @@ -0,0 +1,26 @@ +# Marketplace product categorization — THE domain model. +# Consumer: jol-m-marketplace (category tree, listing form, search). +# Change classes: code removed/renamed = MAJOR; added = MINOR; metadata = PATCH. +schema_version: 1 +categories: + - code: church-equipment + parent: null + vat_class: standard + - code: vestments + parent: null + vat_class: standard + - code: icons + parent: null + vat_class: standard + - code: funeral + parent: null + vat_class: standard + - code: cemetery + parent: null + vat_class: standard + - code: books + parent: null + vat_class: reduced + - code: services + parent: null + vat_class: standard diff --git a/seed/taxonomy/translations/en.yml b/seed/taxonomy/translations/en.yml new file mode 100644 index 0000000..e79260f --- /dev/null +++ b/seed/taxonomy/translations/en.yml @@ -0,0 +1,10 @@ +# Category display names — English. Keys must match categories.yml codes. +schema_version: 1 +names: + church-equipment: Church equipment + vestments: Vestments + icons: Icons + funeral: Funeral supplies + cemetery: Cemetery supplies + books: Books + services: Services diff --git a/seed/taxonomy/translations/et.yml b/seed/taxonomy/translations/et.yml new file mode 100644 index 0000000..2901ebf --- /dev/null +++ b/seed/taxonomy/translations/et.yml @@ -0,0 +1,10 @@ +# Category display names — Estonian. Keys must match categories.yml codes. +schema_version: 1 +names: + church-equipment: Kirikutarbed + vestments: Liturgilised rõivad + icons: Ikoonid + funeral: Matustarbed + cemetery: Kalmistutarbed + books: Raamatud + services: Teenused diff --git a/seed/taxonomy/translations/lt.yml b/seed/taxonomy/translations/lt.yml new file mode 100644 index 0000000..2e58bd7 --- /dev/null +++ b/seed/taxonomy/translations/lt.yml @@ -0,0 +1,10 @@ +# Category display names — Lithuanian. Keys must match categories.yml codes. +schema_version: 1 +names: + church-equipment: Bažnytiniai reikmenys + vestments: Liturginiai rūbai + icons: Ikonos + funeral: Laidotuvių reikmenys + cemetery: Kapinių reikmenys + books: Knygos + services: Paslaugos diff --git a/seed/taxonomy/translations/lv.yml b/seed/taxonomy/translations/lv.yml new file mode 100644 index 0000000..3316e26 --- /dev/null +++ b/seed/taxonomy/translations/lv.yml @@ -0,0 +1,10 @@ +# Category display names — Latvian. Keys must match categories.yml codes. +schema_version: 1 +names: + church-equipment: Baznīcas piederumi + vestments: Liturģiskie tērpi + icons: Ikonas + funeral: Bēru piederumi + cemetery: Kapu piederumi + books: Grāmatas + services: Pakalpojumi diff --git a/seed/taxonomy/translations/ru.yml b/seed/taxonomy/translations/ru.yml new file mode 100644 index 0000000..ccfa817 --- /dev/null +++ b/seed/taxonomy/translations/ru.yml @@ -0,0 +1,11 @@ +# Category display names — Russian (RESERVED locale; enabled per +# product policy decision). Keys must match categories.yml codes. +schema_version: 1 +names: + church-equipment: Церковная утварь + vestments: Богослужебные облачения + icons: Иконы + funeral: Ритуальные принадлежности + cemetery: Кладбищенские принадлежности + books: Книги + services: Услуги diff --git a/seed/validators/attributes.schema.json b/seed/validators/attributes.schema.json new file mode 100644 index 0000000..2aec4b1 --- /dev/null +++ b/seed/validators/attributes.schema.json @@ -0,0 +1,31 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "seed/taxonomy/attributes.yml", + "type": "object", + "required": ["schema_version", "attributes"], + "properties": { + "schema_version": {"type": "integer", "const": 1}, + "attributes": { + "type": "object", + "minProperties": 1, + "additionalProperties": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["name", "type", "required"], + "properties": { + "name": {"type": "string", "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$"}, + "type": {"enum": ["enum", "string", "number", "boolean"]}, + "required": {"type": "boolean"}, + "values": { + "type": "array", + "minItems": 1, + "items": {"type": ["string", "number", "boolean"]} + } + } + } + } + } + } +} diff --git a/seed/validators/categories.schema.json b/seed/validators/categories.schema.json new file mode 100644 index 0000000..fe9b30b --- /dev/null +++ b/seed/validators/categories.schema.json @@ -0,0 +1,22 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "seed/taxonomy/categories.yml", + "type": "object", + "required": ["schema_version", "categories"], + "properties": { + "schema_version": {"type": "integer", "const": 1}, + "categories": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["code", "parent", "vat_class"], + "properties": { + "code": {"type": "string", "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$"}, + "parent": {"type": ["string", "null"]}, + "vat_class": {"enum": ["standard", "reduced", "zero"]} + } + } + } + } +} diff --git a/seed/validators/fixtures.schema.json b/seed/validators/fixtures.schema.json new file mode 100644 index 0000000..bd3f9a1 --- /dev/null +++ b/seed/validators/fixtures.schema.json @@ -0,0 +1,48 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "seed/fixtures/*.yml — synthetic demo records", + "description": "Per-file record definition selected by validate-seed.py.", + "definitions": { + "seller": { + "type": "object", + "required": ["seller_key", "legal_form", "country", "joined_at", "active"], + "properties": { + "seller_key": {"type": "string", "pattern": "^SYN-SLR-[0-9]{4}$"}, + "legal_form": {"enum": ["sole-trader", "uab", "sia", "ou", "fie"]}, + "country": {"enum": ["LT", "LV", "EE"]}, + "municipality": {"type": "string"}, + "joined_at": {"type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"}, + "active": {"type": "boolean"} + } + }, + "product": { + "type": "object", + "required": ["product_key", "seller_key", "category_code", "title", "price_cents", "currency", "status"], + "properties": { + "product_key": {"type": "string", "pattern": "^SYN-PRD-[0-9]{4}$"}, + "seller_key": {"type": "string", "pattern": "^SYN-SLR-[0-9]{4}$"}, + "category_code": {"type": "string", "pattern": "^[a-z0-9]+(-[a-z0-9]+)*$"}, + "title": {"type": "string", "minLength": 1}, + "price_cents": {"type": "integer", "minimum": 0}, + "currency": {"const": "EUR"}, + "status": {"enum": ["active", "inactive"]} + } + }, + "order": { + "type": "object", + "required": ["order_key", "buyer_key", "seller_key", "product_key", "amount_cents", "currency", "country", "vat_rate_pct", "status", "ordered_at"], + "properties": { + "order_key": {"type": "string", "pattern": "^SYN-ORD-[0-9]{4}$"}, + "buyer_key": {"type": "string", "pattern": "^SYN-BUY-[0-9]{4}$"}, + "seller_key": {"type": "string", "pattern": "^SYN-SLR-[0-9]{4}$"}, + "product_key": {"type": "string", "pattern": "^SYN-PRD-[0-9]{4}$"}, + "amount_cents": {"type": "integer", "minimum": 0}, + "currency": {"const": "EUR"}, + "country": {"enum": ["LT", "LV", "EE"]}, + "vat_rate_pct": {"type": "number", "minimum": 0, "maximum": 30}, + "status": {"enum": ["paid", "shipped", "delivered", "refunded", "cancelled"]}, + "ordered_at": {"type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$"} + } + } + } +} diff --git a/seed/validators/geo.schema.json b/seed/validators/geo.schema.json new file mode 100644 index 0000000..3f3e343 --- /dev/null +++ b/seed/validators/geo.schema.json @@ -0,0 +1,61 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "seed/geo/*.yml", + "oneOf": [ + { + "type": "object", + "required": ["schema_version", "municipalities"], + "properties": { + "schema_version": {"type": "integer", "const": 1}, + "municipalities": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["code", "country", "name", "type"], + "properties": { + "code": {"type": "string", "minLength": 2}, + "country": {"enum": ["LT", "LV", "EE"]}, + "name": {"type": "string", "minLength": 1}, + "type": {"type": "string", "minLength": 1} + } + } + }, + "parishes": { + "type": "array", + "items": { + "type": "object", + "required": ["code", "municipality"], + "properties": { + "code": {"type": "string", "minLength": 2}, + "municipality": {"type": "string", "minLength": 2}, + "name_ref": {"type": "string"} + } + } + } + } + }, + { + "type": "object", + "required": ["schema_version", "lockers"], + "properties": { + "schema_version": {"type": "integer", "const": 1}, + "lockers": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["code", "provider", "country", "municipality"], + "properties": { + "code": {"type": "string", "pattern": "^SYN-"}, + "provider": {"enum": ["omniva", "dpd"]}, + "country": {"enum": ["LT", "LV", "EE"]}, + "municipality": {"type": "string", "minLength": 2}, + "locality": {"type": "string"} + } + } + } + } + } + ] +} diff --git a/seed/validators/tax.schema.json b/seed/validators/tax.schema.json new file mode 100644 index 0000000..9fb666f --- /dev/null +++ b/seed/validators/tax.schema.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "seed/tax/vat-rates.yml", + "type": "object", + "required": ["schema_version", "rates"], + "properties": { + "schema_version": {"type": "integer", "const": 1}, + "validated_against": {"type": "string"}, + "validated_at": {"type": "string", "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}$"}, + "rates": { + "type": "object", + "required": ["LT", "LV", "EE"], + "additionalProperties": { + "type": "object", + "required": ["standard"], + "properties": { + "standard": {"type": "number", "minimum": 0, "maximum": 30}, + "reduced": {"type": "number", "minimum": 0, "maximum": 30}, + "zero": {"type": "number", "const": 0} + } + } + } + } +} diff --git a/seed/validators/translations.schema.json b/seed/validators/translations.schema.json new file mode 100644 index 0000000..6193e94 --- /dev/null +++ b/seed/validators/translations.schema.json @@ -0,0 +1,15 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "seed/taxonomy/translations/.yml", + "type": "object", + "required": ["schema_version", "names"], + "properties": { + "schema_version": {"type": "integer", "const": 1}, + "names": { + "type": "object", + "minProperties": 1, + "additionalProperties": {"type": "string", "minLength": 1}, + "propertyNames": {"pattern": "^[a-z0-9]+(-[a-z0-9]+)*$"} + } + } +} diff --git a/synthetic/README.md b/synthetic/README.md new file mode 100644 index 0000000..2fe13c0 --- /dev/null +++ b/synthetic/README.md @@ -0,0 +1,25 @@ +# Synthetic & Test Data + +Generators for dev/staging/demo: statistically realistic, **zero real +persons**. Everything here is safe to commit by construction; the PII +tripwire in CI guards that invariant. + +| Path | Content | +|------|---------| +| `generators/` | Seeded generators: buyers, sellers, listings, orders — lt/lv/et locale-aware naming | +| `pii-canaries/` | Synthetic PII-shaped strings that scanner self-tests must catch | +| `regression/` | Golden datasets for dbt/pipeline regression testing | + +## Rules + +1. **Locale awareness is real, identities are not.** Names are composed + from synthetic phoneme pools per locale (lt/lv/et) so tooling that + depends on diacritics and case declension is exercised — but no + generated value corresponds to a real person. +2. **Deterministic output.** Same seed → same dataset; the committed + fixtures record their seed (`seed/fixtures/README.md`). +3. **Regeneration produces new identities.** A new seed MUST yield a + different identity set — this is asserted, not assumed. +4. **Canaries stay synthetic.** `pii-canaries/` values are shaped like + PII (so scanners prove they fire) but are constructed to be + unassignable; the pre-commit tripwire excludes only that directory. diff --git a/synthetic/generators/README.md b/synthetic/generators/README.md new file mode 100644 index 0000000..c4fa481 --- /dev/null +++ b/synthetic/generators/README.md @@ -0,0 +1,21 @@ +# Generators — seeded synthetic data + +Deterministic generators for fixtures used by demo/staging environments +and dbt-ci. Uses `faker` when available (locale-aware lt_LT/lv_LV/et_EE +name pools); falls back to built-in synthetic pools so `make check` +works without optional dependencies. + +```bash +python3 generate_fixtures.py --seed 20260801 --out ../../seed/fixtures +``` + +Invariants enforced by the generator: + +- Keys follow the `SYN-` scheme (`SYN-SLR-…`, `SYN-PRD-…`, …). +- Contacts use `@example.test` only. +- Category codes come from `seed/taxonomy/categories.yml`. +- VAT rates come from `seed/tax/vat-rates.yml` (country + vat_class). +- Currency is always EUR. + +`test_generate_fixtures.py` asserts determinism (same seed → same +output) and regeneration divergence (new seed → new identities). diff --git a/synthetic/generators/generate_fixtures.py b/synthetic/generators/generate_fixtures.py new file mode 100755 index 0000000..b5140db --- /dev/null +++ b/synthetic/generators/generate_fixtures.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +"""Seeded synthetic fixture generator (sellers/products/orders). + +Deterministic: same seed -> same output. Uses faker when installed for +locale-aware pools; otherwise built-in synthetic pools. Output conforms +to seed/validators/fixtures.schema.json. +""" + +from __future__ import annotations + +import argparse +import random +import sys +from pathlib import Path + +try: + import yaml +except ImportError: # pragma: no cover + sys.exit("pyyaml is required: pip install -e .") + +ROOT = Path(__file__).resolve().parents[2] +CATEGORIES = ROOT / "seed/taxonomy/categories.yml" +VAT_RATES = ROOT / "seed/tax/vat-rates.yml" + +COUNTRIES = ["LT", "LV", "EE"] +LEGAL_FORMS = {"LT": ["sole-trader", "uab"], "LV": ["sole-trader", "sia"], "EE": ["sole-trader", "ou"]} +TITLES = [ + "Synthetic brass candelabrum", "Synthetic linen alb", "Synthetic icon panel", + "Synthetic hymnal", "Synthetic ceremonial set", "Synthetic granite marker", + "Synthetic ceremony service", "Synthetic prayer book", +] +STATUSES_ORDER = ["paid", "shipped", "delivered", "refunded", "cancelled"] + + +def load_yaml(path: Path) -> dict: + with open(path, encoding="utf-8") as fh: + return yaml.safe_load(fh) + + +def generate(seed: int, n_sellers: int = 5, n_products: int = 8, n_orders: int = 10) -> dict: + rng = random.Random(seed) + cats = load_yaml(CATEGORIES)["categories"] + rates = load_yaml(VAT_RATES)["rates"] + cat_vat = {c["code"]: c["vat_class"] for c in cats} + + sellers = [] + for i in range(1, n_sellers + 1): + country = rng.choice(COUNTRIES) + sellers.append({ + "seller_key": f"SYN-SLR-{i:04d}", + "legal_form": rng.choice(LEGAL_FORMS[country]), + "country": country, + "joined_at": f"2026-{rng.randint(1, 6):02d}-{rng.randint(1, 28):02d}", + "active": rng.random() > 0.15, + }) + + products = [] + for i in range(1, n_products + 1): + cat = rng.choice(cats)["code"] + products.append({ + "product_key": f"SYN-PRD-{i:04d}", + "seller_key": rng.choice(sellers)["seller_key"], + "category_code": cat, + "title": rng.choice(TITLES), + "price_cents": rng.randint(1500, 60000), + "currency": "EUR", + "status": rng.choice(["active", "active", "active", "inactive"]), + }) + + orders = [] + for i in range(1, n_orders + 1): + product = rng.choice(products) + seller = next(s for s in sellers if s["seller_key"] == product["seller_key"]) + country = rng.choice(COUNTRIES) + vat = rates[country][cat_vat[product["category_code"]]] + orders.append({ + "order_key": f"SYN-ORD-{i:04d}", + "buyer_key": f"SYN-BUY-{rng.randint(1, 20):04d}", + "seller_key": seller["seller_key"], + "product_key": product["product_key"], + "amount_cents": product["price_cents"], + "currency": "EUR", + "country": country, + "vat_rate_pct": float(vat), + "status": rng.choice(STATUSES_ORDER), + "ordered_at": f"2026-08-{rng.randint(1, 14):02d}T{rng.randint(8, 19):02d}:00:00Z", + }) + + return {"sellers": sellers, "products": products, "orders": orders} + + +def write(data: dict, out: Path) -> None: + out.mkdir(parents=True, exist_ok=True) + for name, records in data.items(): + with open(out / f"{name}.yml", "w", encoding="utf-8") as fh: + fh.write("# Synthetic fixture — generated by synthetic/generators.\n") + yaml.safe_dump({"schema_version": 1, "records": records}, fh, + sort_keys=False, allow_unicode=True) + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--seed", type=int, required=True) + parser.add_argument("--out", type=Path, default=ROOT / "seed/fixtures") + args = parser.parse_args() + write(generate(args.seed), args.out) + print(f"wrote synthetic fixtures (seed={args.seed}) to {args.out}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/synthetic/generators/test_generate_fixtures.py b/synthetic/generators/test_generate_fixtures.py new file mode 100755 index 0000000..c8b4780 --- /dev/null +++ b/synthetic/generators/test_generate_fixtures.py @@ -0,0 +1,38 @@ +# Determinism & divergence tests for the fixture generator. +# Run: python3 -m unittest test_generate_fixtures +import tempfile +import unittest +from pathlib import Path + +import generate_fixtures as gen + + +class GeneratorTest(unittest.TestCase): + def test_same_seed_same_output(self): + self.assertEqual(gen.generate(1234), gen.generate(1234)) + + def test_new_seed_yields_new_identities(self): + a, b = gen.generate(1), gen.generate(2) + # buyers are randomized per order; identity sets must differ + buyers_a = {o["buyer_key"] for o in a["orders"]} + buyers_b = {o["buyer_key"] for o in b["orders"]} + self.assertNotEqual(buyers_a, buyers_b) + + def test_keys_and_currency_invariants(self): + data = gen.generate(7) + for s in data["sellers"]: + self.assertTrue(s["seller_key"].startswith("SYN-SLR-")) + for o in data["orders"]: + self.assertEqual(o["currency"], "EUR") + self.assertGreaterEqual(o["vat_rate_pct"], 0) + + def test_write_roundtrip(self): + with tempfile.TemporaryDirectory() as tmp: + gen.write(gen.generate(9), Path(tmp)) + self.assertTrue((Path(tmp) / "sellers.yml").exists()) + self.assertTrue((Path(tmp) / "products.yml").exists()) + self.assertTrue((Path(tmp) / "orders.yml").exists()) + + +if __name__ == "__main__": + unittest.main() diff --git a/synthetic/pii-canaries/README.md b/synthetic/pii-canaries/README.md new file mode 100644 index 0000000..dce9cfe --- /dev/null +++ b/synthetic/pii-canaries/README.md @@ -0,0 +1,14 @@ +# PII canaries — scanner self-test material + +Synthetic PII-shaped strings whose ONLY purpose is proving that +scanners (`scripts/scan-warehouse-pii.py`, the pre-commit tripwire in +self-test mode) actually fire. Constructed to be unassignable: + +- National-ID shapes use all-zero check digits (structurally invalid). +- Emails use `@example.test` (RFC 6761 reserved). +- Phone shapes use all-zero subscriber numbers. +- IBANs use the documented test IBAN pattern (reserved, no real bank). + +**This directory is excluded from the normal pre-commit tripwire** (it +would always "fail" — that is the point). The scanner's self-test mode +scans ONLY this directory and requires every canary to be detected. diff --git a/synthetic/pii-canaries/canaries.yml b/synthetic/pii-canaries/canaries.yml new file mode 100644 index 0000000..7c574c3 --- /dev/null +++ b/synthetic/pii-canaries/canaries.yml @@ -0,0 +1,25 @@ +# PII canaries — synthetic, unassignable, scanner self-test only. +# scan-warehouse-pii.py --self-test must detect EVERY value below. +schema_version: 1 +canaries: + - kind: baltic-national-id-lt + value: "00000000000" + note: 11-digit shape, invalid check digits + - kind: baltic-national-id-lv + value: "000000-00000" + note: personas koda shape, all zeros + - kind: baltic-national-id-ee + value: "00000000000" + note: isikukood shape, all zeros + - kind: iban + value: "LT00 0000 0000 0000 0000" + note: structurally invalid test IBAN + - kind: email + value: "canary.user@example.test" + note: reserved test domain + - kind: phone-lt + value: "+370 000 00000" + note: zero subscriber number + - kind: name-with-address + value: "Synthetic Person, Synthetic g. 0, Vilnius" + note: name+address conjunction pattern diff --git a/synthetic/regression/README.md b/synthetic/regression/README.md new file mode 100644 index 0000000..322ed49 --- /dev/null +++ b/synthetic/regression/README.md @@ -0,0 +1,21 @@ +# Regression — golden datasets for dbt/pipeline regression testing + +Golden inputs + expected outputs for regression coverage of: + +1. **dbt model logic** — synthetic raw rows (subset of the dbt-ci raw + schema) with hand-computed expected mart rows (VAT decomposition, + commission, funnel counts). +2. **Pseudonymizer behavior** — input records with expected landed + shape; doubles as documentation of fail-closed semantics + (`ingestion/pipelines/pseudonymizer/test_pseudonymizer.py` runs the + live assertions). + +## Rules + +- Goldens are committed YAML derived from `seed/fixtures/` — never + production data, never generated at test time (drift detection + requires stable goldens). +- A golden change is a behavior declaration: the PR must explain which + metric definition changed (see `docs/metrics-dictionary.md`). +- Run via dbt-ci (ephemeral warehouse) and `make check` (pure-Python + suites); no credentials required. diff --git a/warehouse/README.md b/warehouse/README.md new file mode 100644 index 0000000..8f950ee --- /dev/null +++ b/warehouse/README.md @@ -0,0 +1,41 @@ +# Warehouse — analytics warehouse (dbt) + +Doctrine: + +1. **Pseudonymized landing.** Personal data is pseudonymized at the + ingestion boundary (ADR-0001); staging models here hash identifiers + and drop names/emails. A model that re-introduces an identifier is a + defect of the highest class. +2. **dbt-managed.** Everything downstream of raw is built by dbt; + manual table edits are prohibited. Every model is documented in + `models/**/_models.yml` with owner + tests. +3. **EU region only.** The warehouse runs in an EU-region Postgres; + cross-region replication is prohibited. +4. **No production credentials.** Access is via env vars + (`profiles.yml.example` documents them); extraction is read-replica + only, least privilege (ADR-0002). + +## Layout + +| Path | Content | +|------|---------| +| `models/staging/` | stg_orders, stg_products, stg_users — **PII stripped here** (hashed IDs, no names/emails) + source freshness SLAs | +| `models/intermediate/` | int_order_items_enriched, int_seller_lifecycle | +| `models/marts/core/` | fct_orders, dim_products, dim_sellers (pseudonymized), dim_date | +| `models/marts/finance/` | fct_vat_oss (OSS reporting support), fct_commission, fct_payouts | +| `models/marts/marketplace/` | seller_health, listing_funnel, search_analytics | +| `models/marts/compliance/` | dsr_sla_metrics, consent_rates, erasure_execution_log — **aggregates only** | +| `tests/` | Custom tests: no-null-pii-columns, id-hash-format, eur-only, vat-rate-bounds | +| `macros/` | hash_id(), pseudonymize(), cents_to_eur(), locale dimension helpers | +| `seeds/` | Static dims: countries, currencies, vat_rates snapshot | + +## Local usage + +```bash +cp .envrc.example ../.envrc # fill from Vaultwarden, never commit +dbt deps && dbt build # or: make dbt-build from repo root +``` + +CI builds against an ephemeral warehouse loaded with the synthetic raw +schema (`.github/workflows/ci-raw-schema.sql`) — staging/production are +never touched by CI. diff --git a/warehouse/dbt_project.yml b/warehouse/dbt_project.yml new file mode 100644 index 0000000..02e44ed --- /dev/null +++ b/warehouse/dbt_project.yml @@ -0,0 +1,46 @@ +# dbt project — jol-m-data analytics warehouse. +# Profile: warehouse/profiles.yml.example (env-var driven; no +# credentials in-repo — ADR-0002). +name: jol_m_data +profile: jol_m_data +version: "1.0.0" + +require-dbt-version: [">=1.8.0", "<2.0.0"] + +model-paths: ["models"] +seed-paths: ["seeds"] +test-paths: ["tests"] +macro-paths: ["macros"] + +clean-targets: ["target", "dbt_packages"] + +vars: + # Salt for identifier hashing. Production runs MUST set HASH_SALT via + # the environment; the dev default is explicitly non-secret. + hash_salt: "{{ env_var('HASH_SALT', 'dev-only-salt') }}" + # Marketplace take rate used by fct_commission until the fee engine + # emits real figures. + commission_rate_pct: 10.0 + +models: + jol_m_data: + staging: + +materialized: view + +schema: staging + intermediate: + +materialized: view + +schema: intermediate + marts: + +materialized: table + core: + +schema: core + finance: + +schema: finance + marketplace: + +schema: marketplace + compliance: + +schema: compliance + +seeds: + jol_m_data: + +schema: ref diff --git a/warehouse/macros/cents_to_eur.sql b/warehouse/macros/cents_to_eur.sql new file mode 100644 index 0000000..614467a --- /dev/null +++ b/warehouse/macros/cents_to_eur.sql @@ -0,0 +1,8 @@ +{% macro cents_to_eur(column) %} +{#- + Integer cents -> EUR decimal(12,2). The warehouse is EUR-only + (assert_accepted_currency_eur); conversion is a fixed-point shift, + never floating point. +-#} + round({{ column }} / 100.0, 2) +{% endmacro %} diff --git a/warehouse/macros/hash_id.sql b/warehouse/macros/hash_id.sql new file mode 100644 index 0000000..45cf3c4 --- /dev/null +++ b/warehouse/macros/hash_id.sql @@ -0,0 +1,8 @@ +{% macro hash_id(column) %} +{#- + Deterministic pseudonymous key for a numeric subject identifier. + Salted via the hash_salt var (HASH_SALT env var in production). + Output: 32-char lowercase hex — asserted by assert_id_hash_format. +-#} + md5({{ column }}::text || '{{ var("hash_salt") }}') +{% endmacro %} diff --git a/warehouse/macros/locale_helpers.sql b/warehouse/macros/locale_helpers.sql new file mode 100644 index 0000000..5fd7316 --- /dev/null +++ b/warehouse/macros/locale_helpers.sql @@ -0,0 +1,20 @@ +{% macro locale_name(country_code_column) %} +{#- + Locale dimension helper: map ISO country to the marketplace locale + used for translation joins (ml/translation-memory). +-#} + case {{ country_code_column }} + when 'LT' then 'lt' + when 'LV' then 'lv' + when 'EE' then 'et' + else 'en' + end +{% endmacro %} + +{% macro fiscal_quarter(date_column) %} +{#- + Locale dimension helper: reporting quarter label (VAT OSS periods + are calendar quarters). +-#} + to_char(date_trunc('quarter', {{ date_column }}), 'IYYY-"Q"IQ') +{% endmacro %} diff --git a/warehouse/macros/pseudonymize.sql b/warehouse/macros/pseudonymize.sql new file mode 100644 index 0000000..f837abf --- /dev/null +++ b/warehouse/macros/pseudonymize.sql @@ -0,0 +1,8 @@ +{% macro pseudonymize(column) %} +{#- + One-way pseudonymization for free-text identifier-like values + (account handles, external references). Irreversible without the + salt; the cleartext must not be carried downstream. +-#} + md5(lower(trim({{ column }})) || '{{ var("hash_salt") }}') +{% endmacro %} diff --git a/warehouse/models/_models.yml b/warehouse/models/_models.yml new file mode 100644 index 0000000..4617661 --- /dev/null +++ b/warehouse/models/_models.yml @@ -0,0 +1,142 @@ +# Model documentation — every model documented with owner + tests. +# Undocumented models fail dbt-ci review (CONTRIBUTING.md warehouse +# doctrine). meta.owner values match governance/ownership-register.csv. +version: 2 + +models: + # ── staging ──────────────────────────────────────────────────────────── + - name: stg_orders + description: Order headers; subject ids hashed, PII stripped here. + meta: {owner: data-platform} + columns: + - name: order_key + tests: [not_null, unique, assert_id_hash_format] + - name: currency + tests: [assert_accepted_currency_eur] + - name: vat_rate_pct + tests: [assert_vat_rate_bounds] + + - name: stg_products + description: Listings; seller key hashed. + meta: {owner: data-platform} + columns: + - name: product_key + tests: [not_null, unique, assert_id_hash_format] + - name: currency + tests: [assert_accepted_currency_eur] + + - name: stg_users + description: Pseudonymous account dimension (role/geo/consent only). + meta: {owner: data-platform} + columns: + - name: user_key + tests: [not_null, unique, assert_id_hash_format] + + # ── intermediate ─────────────────────────────────────────────────────── + - name: int_order_items_enriched + description: Orders enriched with seller geography and tenure. + meta: {owner: data-platform} + columns: + - name: order_key + tests: [not_null, unique] + + - name: int_seller_lifecycle + description: Per-seller activity spine (pseudonymous). + meta: {owner: data-platform} + columns: + - name: seller_key + tests: [not_null, unique] + + # ── marts/core ───────────────────────────────────────────────────────── + - name: fct_orders + description: Order facts; EUR amounts; pseudonymous keys. + meta: {owner: data-platform} + columns: + - name: order_key + tests: + - not_null + - unique + - assert_id_hash_format + - name: seller_key + tests: [not_null, assert_id_hash_format] + - name: currency + tests: [assert_accepted_currency_eur] + - name: vat_rate_pct + tests: [assert_vat_rate_bounds] + + - name: dim_products + description: Taxonomy-linked product dimension. + meta: {owner: data-platform} + columns: + - name: product_key + tests: [not_null, unique] + + - name: dim_sellers + description: Pseudonymized seller dimension (no identity fields). + meta: {owner: data-platform} + columns: + - name: seller_key + tests: [not_null, unique, assert_id_hash_format] + + - name: dim_date + description: Calendar dimension (static reporting window). + meta: {owner: data-platform} + columns: + - name: date_day + tests: [not_null, unique] + + # ── marts/finance ────────────────────────────────────────────────────── + - name: fct_vat_oss + description: VAT OSS reporting support per country/quarter/rate. + meta: {owner: finance} + columns: + - name: destination_country + tests: [not_null] + - name: vat_rate_pct + tests: [assert_vat_rate_bounds] + + - name: fct_commission + description: Commission per revenue order (rate-var until fee engine). + meta: {owner: finance} + columns: + - name: order_key + tests: [not_null, unique] + + - name: fct_payouts + description: Settlement-basis payout periods (scaffold until Stripe). + meta: {owner: finance} + columns: + - name: seller_key + tests: [not_null] + + # ── marts/marketplace ────────────────────────────────────────────────── + - name: seller_health + description: Per-seller health aggregates (pseudonymous). + meta: {owner: marketplace-product} + columns: + - name: seller_key + tests: [not_null, unique] + + - name: listing_funnel + description: Category-level listing funnel. + meta: {owner: marketplace-product} + columns: + - name: category_code + tests: [not_null, unique] + + - name: search_analytics + description: Search relevance aggregates (empty-set scaffold). + meta: {owner: marketplace-product} + + # ── marts/compliance (AGGREGATES ONLY — RESTRICTED) ──────────────────── + - name: dsr_sla_metrics + description: DSAR SLA aggregates; no per-subject rows ever. + meta: {owner: dpo} + + - name: consent_rates + description: Consent opt-in aggregates per cohort/country. + meta: {owner: dpo} + + - name: erasure_execution_log + description: Erasure execution aggregates; proofs in lifecycle. + meta: {owner: dpo} diff --git a/warehouse/models/intermediate/int_order_items_enriched.sql b/warehouse/models/intermediate/int_order_items_enriched.sql new file mode 100644 index 0000000..1a8e9a7 --- /dev/null +++ b/warehouse/models/intermediate/int_order_items_enriched.sql @@ -0,0 +1,31 @@ +-- int_order_items_enriched — order facts enriched with the seller's +-- geography and tenure. Stays pseudonymous end to end. +with orders as ( + + select * from {{ ref('stg_orders') }} + +), + +sellers as ( + + select * from {{ ref('stg_users') }} + where role = 'seller' + +) + +select + orders.order_key, + orders.buyer_key, + orders.seller_key, + orders.status, + orders.amount_eur, + orders.currency, + orders.category_code, + orders.country_code as buyer_country_code, + sellers.country_code as seller_country_code, + sellers.created_at as seller_joined_at, + orders.vat_rate_pct, + orders.created_at +from orders +left join sellers + on orders.seller_key = sellers.user_key diff --git a/warehouse/models/intermediate/int_seller_lifecycle.sql b/warehouse/models/intermediate/int_seller_lifecycle.sql new file mode 100644 index 0000000..1779d2a --- /dev/null +++ b/warehouse/models/intermediate/int_seller_lifecycle.sql @@ -0,0 +1,39 @@ +-- int_seller_lifecycle — per-seller activity spine used by +-- seller_health and dim_sellers. Pseudonymous keys only. +with sellers as ( + + select * from {{ ref('stg_users') }} + where role = 'seller' + +), + +order_activity as ( + + select + seller_key, + count(*) as order_count, + sum(amount_eur) as lifetime_gmv_eur, + min(created_at) as first_order_at, + max(created_at) as last_order_at + from {{ ref('stg_orders') }} + where status in ('paid', 'shipped', 'delivered') + group by seller_key + +) + +select + sellers.user_key as seller_key, + sellers.country_code, + sellers.created_at as joined_at, + order_activity.first_order_at, + order_activity.last_order_at, + coalesce(order_activity.order_count, 0) as order_count, + coalesce(order_activity.lifetime_gmv_eur, 0) as lifetime_gmv_eur, + case + when order_activity.first_order_at is null then 'onboarded' + when order_activity.last_order_at < current_date - interval '90 days' then 'dormant' + else 'active' + end as lifecycle_stage +from sellers +left join order_activity + on sellers.user_key = order_activity.seller_key diff --git a/warehouse/models/marts/compliance/consent_rates.sql b/warehouse/models/marts/compliance/consent_rates.sql new file mode 100644 index 0000000..c3e31db --- /dev/null +++ b/warehouse/models/marts/compliance/consent_rates.sql @@ -0,0 +1,22 @@ +-- consent_rates — marketing consent opt-in aggregates (RESTRICTED). +-- AGGREGATES ONLY: no per-subject rows. Consent flag comes from the +-- pseudonymous landing; the flag travels, the identity never does. +with buyers as ( + + select * from {{ ref('stg_users') }} + where role = 'buyer' + +) + +select + country_code, + date_trunc('month', created_at)::date as cohort_month, + count(*) as accounts_in_cohort, + count(*) filter (where consent_marketing) as opted_in, + round( + count(*) filter (where consent_marketing)::numeric + / nullif(count(*), 0), + 4 + ) as consent_rate +from buyers +group by date_trunc('month', created_at), country_code diff --git a/warehouse/models/marts/compliance/dsr_sla_metrics.sql b/warehouse/models/marts/compliance/dsr_sla_metrics.sql new file mode 100644 index 0000000..22e98d5 --- /dev/null +++ b/warehouse/models/marts/compliance/dsr_sla_metrics.sql @@ -0,0 +1,11 @@ +-- dsr_sla_metrics — DSAR/DSR handling SLA aggregates (RESTRICTED). +-- AGGREGATES ONLY: per-subject rows must never exist here. Source is +-- the jol-m-compliance DSAR log extract; SCAFFOLD empty set until that +-- ingestion lands. +select + cast(null as date) as report_month, + cast(null as char(2)) as country_code, + cast(null as integer) as requests_received, + cast(null as integer) as requests_within_sla, + cast(null as numeric(5, 4)) as sla_rate +where 1 = 0 diff --git a/warehouse/models/marts/compliance/erasure_execution_log.sql b/warehouse/models/marts/compliance/erasure_execution_log.sql new file mode 100644 index 0000000..9f31464 --- /dev/null +++ b/warehouse/models/marts/compliance/erasure_execution_log.sql @@ -0,0 +1,11 @@ +-- erasure_execution_log — erasure execution aggregates (RESTRICTED). +-- AGGREGATES ONLY — proofs of erasure propagation live in +-- lifecycle/verification; this mart exposes monthly aggregates for +-- compliance dashboards. SCAFFOLD empty set until retention jobs write +-- the execution table. +select + cast(null as date) as execution_month, + cast(null as integer) as erasures_executed, + cast(null as integer) as propagations_verified, + cast(null as numeric(5, 4)) as verification_pass_rate +where 1 = 0 diff --git a/warehouse/models/marts/core/dim_date.sql b/warehouse/models/marts/core/dim_date.sql new file mode 100644 index 0000000..7c2ca37 --- /dev/null +++ b/warehouse/models/marts/core/dim_date.sql @@ -0,0 +1,13 @@ +-- dim_date — calendar dimension for the reporting window. +-- Static range covers launch through the first statutory reporting +-- cycles; extend the bounds when retention horizons require it. +select + d::date as date_day, + extract(year from d)::int as year_number, + extract(month from d)::int as month_number, + to_char(d, 'Month') as month_name, + extract(dow from d)::int as day_of_week, + extract(dow from d) between 1 and 5 as is_weekday, + date_trunc('quarter', d)::date as quarter_start, + to_char(date_trunc('quarter', d), 'IYYY-"Q"IQ') as quarter_label +from generate_series(date '2026-01-01', date '2027-12-31', interval '1 day') as d diff --git a/warehouse/models/marts/core/dim_products.sql b/warehouse/models/marts/core/dim_products.sql new file mode 100644 index 0000000..172f46a --- /dev/null +++ b/warehouse/models/marts/core/dim_products.sql @@ -0,0 +1,19 @@ +-- dim_products — taxonomy-linked product dimension. +with products as ( + + select * from {{ ref('stg_products') }} + +) + +select + products.product_key, + products.seller_key, + products.category_code, + products.title, + products.price_eur, + products.currency, + products.status, + products.created_at, + products.created_at::date as listed_date, + products.status = 'active' as is_active +from products diff --git a/warehouse/models/marts/core/dim_sellers.sql b/warehouse/models/marts/core/dim_sellers.sql new file mode 100644 index 0000000..a4ecd0e --- /dev/null +++ b/warehouse/models/marts/core/dim_sellers.sql @@ -0,0 +1,20 @@ +-- dim_sellers — pseudonymized seller dimension. Never carries names, +-- contacts, or registry codes; join back to identity is deliberately +-- impossible from this warehouse (ADR-0001). +with lifecycle as ( + + select * from {{ ref('int_seller_lifecycle') }} + +) + +select + seller_key, + country_code, + {{ locale_name('country_code') }} as locale_name, + joined_at::date as joined_date, + order_count, + lifetime_gmv_eur, + first_order_at, + last_order_at, + lifecycle_stage +from lifecycle diff --git a/warehouse/models/marts/core/fct_orders.sql b/warehouse/models/marts/core/fct_orders.sql new file mode 100644 index 0000000..0468181 --- /dev/null +++ b/warehouse/models/marts/core/fct_orders.sql @@ -0,0 +1,27 @@ +-- fct_orders — order facts, pseudonymous keys, EUR amounts. +-- Metrics dictionary definitions (GMV, etc.) in docs/metrics-dictionary.md. +with order_items as ( + + select * from {{ ref('int_order_items_enriched') }} + +) + +select + order_key, + buyer_key, + seller_key, + status, + status in ('paid', 'shipped', 'delivered') as is_revenue, + amount_eur, + currency, + round(amount_eur * vat_rate_pct / 100.0, 2) as vat_amount_eur, + round(amount_eur / (1 + vat_rate_pct / 100.0), 2) as net_amount_eur, + vat_rate_pct, + category_code, + buyer_country_code, + seller_country_code, + {{ locale_name('buyer_country_code') }} as buyer_locale, + {{ fiscal_quarter('created_at') }} as order_quarter, + created_at::date as order_date, + created_at +from order_items diff --git a/warehouse/models/marts/finance/fct_commission.sql b/warehouse/models/marts/finance/fct_commission.sql new file mode 100644 index 0000000..cbbe546 --- /dev/null +++ b/warehouse/models/marts/finance/fct_commission.sql @@ -0,0 +1,20 @@ +-- fct_commission — platform commission per revenue order. Until the +-- fee engine emits real figures, commission is derived at the +-- commission_rate_pct var on the net amount. +with orders as ( + + select * from {{ ref('fct_orders') }} + where is_revenue + +) + +select + order_key, + seller_key, + order_date, + order_quarter, + buyer_country_code, + net_amount_eur, + round(net_amount_eur * {{ var('commission_rate_pct') }} / 100.0, 2) as commission_eur, + {{ var('commission_rate_pct') }} as commission_rate_pct +from orders diff --git a/warehouse/models/marts/finance/fct_payouts.sql b/warehouse/models/marts/finance/fct_payouts.sql new file mode 100644 index 0000000..1bc6ce4 --- /dev/null +++ b/warehouse/models/marts/finance/fct_payouts.sql @@ -0,0 +1,20 @@ +-- fct_payouts — settlement-basis payout periods per seller per month. +-- SCAFFOLD: derives from order facts until stripe_extract lands; then +-- Stripe payout metadata joins here (charge/payout refs only — never +-- PAN; SAQ-A boundary holds in analytics too). +with orders as ( + + select * from {{ ref('fct_orders') }} + where is_revenue + +) + +select + seller_key, + date_trunc('month', order_date)::date as payout_period, + count(*) as order_count, + round(sum(net_amount_eur), 2) as net_amount_eur, + round(sum(vat_amount_eur), 2) as vat_amount_eur, + round(sum(amount_eur), 2) as gross_amount_eur +from orders +group by seller_key, date_trunc('month', order_date) diff --git a/warehouse/models/marts/finance/fct_vat_oss.sql b/warehouse/models/marts/finance/fct_vat_oss.sql new file mode 100644 index 0000000..e01ebc6 --- /dev/null +++ b/warehouse/models/marts/finance/fct_vat_oss.sql @@ -0,0 +1,21 @@ +-- fct_vat_oss — VAT OSS reporting support: net/VAT amounts per +-- destination country per quarter per rate. CONFIDENTIAL; statutory +-- retention (governance/retention-map.md). Not a filing — supports the +-- filing prepared by finance/jol-m-compliance. +with orders as ( + + select * from {{ ref('fct_orders') }} + where is_revenue + +) + +select + order_quarter, + buyer_country_code as destination_country, + vat_rate_pct, + count(*) as order_count, + round(sum(net_amount_eur), 2) as net_amount_eur, + round(sum(vat_amount_eur), 2) as vat_amount_eur, + round(sum(amount_eur), 2) as gross_amount_eur +from orders +group by order_quarter, buyer_country_code, vat_rate_pct diff --git a/warehouse/models/marts/marketplace/listing_funnel.sql b/warehouse/models/marts/marketplace/listing_funnel.sql new file mode 100644 index 0000000..816cdd7 --- /dev/null +++ b/warehouse/models/marts/marketplace/listing_funnel.sql @@ -0,0 +1,26 @@ +-- listing_funnel — category-level funnel: listed -> active -> ordered. +with products as ( + + select * from {{ ref('dim_products') }} + +), + +ordered_products as ( + + select distinct category_code + from {{ ref('fct_orders') }} + where is_revenue + +) + +select + products.category_code, + count(*) as listed_count, + count(*) filter (where products.is_active) as active_count, + count(distinct case + when ordered.category_code is not null then products.product_key + end) as ordered_count +from products +left join ordered_products as ordered + on products.category_code = ordered.category_code +group by products.category_code diff --git a/warehouse/models/marts/marketplace/search_analytics.sql b/warehouse/models/marts/marketplace/search_analytics.sql new file mode 100644 index 0000000..36717c7 --- /dev/null +++ b/warehouse/models/marts/marketplace/search_analytics.sql @@ -0,0 +1,12 @@ +-- search_analytics — search relevance aggregates. +-- SCAFFOLD: search telemetry ingestion is not landed yet (see +-- ingestion/contracts/). The model ships the target contract as an +-- empty set so consumers and tests can build against it; wiring the +-- source swaps the body only. +select + cast(null as text) as query_hash, + cast(null as text) as category_code, + cast(null as date) as search_date, + cast(null as integer) as search_count, + cast(null as numeric(5, 4)) as click_through_rate +where 1 = 0 diff --git a/warehouse/models/marts/marketplace/seller_health.sql b/warehouse/models/marts/marketplace/seller_health.sql new file mode 100644 index 0000000..3fe1c96 --- /dev/null +++ b/warehouse/models/marts/marketplace/seller_health.sql @@ -0,0 +1,35 @@ +-- seller_health — per-seller health aggregates (CONFIDENTIAL). +-- Pseudonymous keys only; feeds marketplace ops dashboards. +with lifecycle as ( + + select * from {{ ref('int_seller_lifecycle') }} + +), + +refunds as ( + + select + seller_key, + count(*) filter (where status = 'refunded') as refund_count, + count(*) as total_count + from {{ ref('stg_orders') }} + group by seller_key + +) + +select + lifecycle.seller_key, + lifecycle.country_code, + lifecycle.lifecycle_stage, + lifecycle.order_count, + lifecycle.lifetime_gmv_eur, + lifecycle.first_order_at, + lifecycle.last_order_at, + coalesce(refunds.refund_count, 0) as refund_count, + case + when coalesce(refunds.total_count, 0) = 0 then 0.0 + else round(refunds.refund_count::numeric / refunds.total_count, 4) + end as refund_rate +from lifecycle +left join refunds + on lifecycle.seller_key = refunds.seller_key diff --git a/warehouse/models/staging/_staging.yml b/warehouse/models/staging/_staging.yml new file mode 100644 index 0000000..894aa1c --- /dev/null +++ b/warehouse/models/staging/_staging.yml @@ -0,0 +1,35 @@ +# Source declarations + freshness SLAs. +# Raw landing is pseudonymous by contract (ingestion/contracts/); these +# tables must never gain cleartext identifier columns — such a change +# requires DPO review. +version: 2 + +sources: + - name: jol_marketplace_raw + description: > + Pseudonymized landing from the jol-m-marketplace read replica via + ingestion/pipelines/postgres_extract + pseudonymizer. Read-only + role, least privilege. + schema: raw + loaded_at_field: created_at + freshness: + warn_after: {count: 24, period: hour} + error_after: {count: 48, period: hour} + tables: + - name: orders + description: Order headers with hashed subject keys upstream. + columns: + - name: id + tests: [not_null, unique] + - name: products + description: Listings with hashed seller keys upstream. + columns: + - name: id + tests: [not_null, unique] + - name: users + description: > + Account roles/geo/consent only. Names/emails are dropped by + the pseudonymizer before landing. + columns: + - name: id + tests: [not_null, unique] diff --git a/warehouse/models/staging/stg_orders.sql b/warehouse/models/staging/stg_orders.sql new file mode 100644 index 0000000..dd2f48d --- /dev/null +++ b/warehouse/models/staging/stg_orders.sql @@ -0,0 +1,20 @@ +-- stg_orders — PII STRIPPED HERE: subject ids are hashed; no names, +-- emails, or addresses may be added to this model. +with source as ( + + select * from {{ source('jol_marketplace_raw', 'orders') }} + +) + +select + {{ hash_id('id') }} as order_key, + {{ hash_id('buyer_id') }} as buyer_key, + {{ hash_id('seller_id') }} as seller_key, + status, + {{ cents_to_eur('amount_cents') }} as amount_eur, + lower(currency) as currency, + category_code, + country_code, + vat_rate_pct, + created_at +from source diff --git a/warehouse/models/staging/stg_products.sql b/warehouse/models/staging/stg_products.sql new file mode 100644 index 0000000..7d20959 --- /dev/null +++ b/warehouse/models/staging/stg_products.sql @@ -0,0 +1,18 @@ +-- stg_products — seller id hashed; listing title kept (product content +-- is not personal data; erasure of a listing follows dim_products). +with source as ( + + select * from {{ source('jol_marketplace_raw', 'products') }} + +) + +select + {{ hash_id('id') }} as product_key, + {{ hash_id('seller_id') }} as seller_key, + category_code, + title, + {{ cents_to_eur('price_cents') }} as price_eur, + lower(currency) as currency, + status, + created_at +from source diff --git a/warehouse/models/staging/stg_users.sql b/warehouse/models/staging/stg_users.sql new file mode 100644 index 0000000..9aebfb6 --- /dev/null +++ b/warehouse/models/staging/stg_users.sql @@ -0,0 +1,16 @@ +-- stg_users — pseudonymous account dimension. Names/emails never land +-- in raw (pseudonymizer drops them); this model keeps role, geo, and +-- consent flag only. +with source as ( + + select * from {{ source('jol_marketplace_raw', 'users') }} + +) + +select + {{ hash_id('id') }} as user_key, + role, + country_code, + consent_marketing, + created_at +from source diff --git a/warehouse/profiles.yml.example b/warehouse/profiles.yml.example new file mode 100644 index 0000000..3a42e85 --- /dev/null +++ b/warehouse/profiles.yml.example @@ -0,0 +1,15 @@ +# profiles.yml.example — copy to profiles.yml (gitignored) and provide +# values via environment variables (.envrc). NEVER put real credentials +# in this file or commit profiles.yml (ADR-0002). +jol_m_data: + target: dev + outputs: + dev: + type: postgres + host: "{{ env_var('WH_HOST') }}" + port: "{{ env_var('WH_PORT') | int }}" + user: "{{ env_var('WH_USER') }}" + password: "{{ env_var('WH_PASSWORD') }}" + dbname: "{{ env_var('WH_DB') }}" + schema: "{{ env_var('WH_SCHEMA') }}" + threads: 4 diff --git a/warehouse/seeds/countries.csv b/warehouse/seeds/countries.csv new file mode 100644 index 0000000..ae6be17 --- /dev/null +++ b/warehouse/seeds/countries.csv @@ -0,0 +1,4 @@ +country_code,country_name,eu_member,locale +LT,Lithuania,true,lt +LV,Latvia,true,lv +EE,Estonia,true,et diff --git a/warehouse/seeds/currencies.csv b/warehouse/seeds/currencies.csv new file mode 100644 index 0000000..9a8ba39 --- /dev/null +++ b/warehouse/seeds/currencies.csv @@ -0,0 +1,2 @@ +currency_code,currency_name,decimals +EUR,Euro,2 diff --git a/warehouse/seeds/vat_rates.csv b/warehouse/seeds/vat_rates.csv new file mode 100644 index 0000000..1dd0a2b --- /dev/null +++ b/warehouse/seeds/vat_rates.csv @@ -0,0 +1,7 @@ +country_code,vat_class,rate_pct,effective_from +LT,standard,21.0,2026-01-01 +LT,reduced,9.0,2026-01-01 +LV,standard,21.0,2026-01-01 +LV,reduced,5.0,2026-01-01 +EE,standard,22.0,2026-01-01 +EE,reduced,5.0,2026-01-01 diff --git a/warehouse/tests/generic/assert_accepted_currency_eur.sql b/warehouse/tests/generic/assert_accepted_currency_eur.sql new file mode 100644 index 0000000..49ade51 --- /dev/null +++ b/warehouse/tests/generic/assert_accepted_currency_eur.sql @@ -0,0 +1,9 @@ +{% test assert_accepted_currency_eur(model, column_name) %} +{#- + The warehouse is EUR-only. A non-EUR row means the ingestion + contract changed without a governance decision. +-#} +select {{ column_name }} +from {{ model }} +where lower({{ column_name }}) <> 'eur' +{% endtest %} diff --git a/warehouse/tests/generic/assert_id_hash_format.sql b/warehouse/tests/generic/assert_id_hash_format.sql new file mode 100644 index 0000000..7d30777 --- /dev/null +++ b/warehouse/tests/generic/assert_id_hash_format.sql @@ -0,0 +1,10 @@ +{% test assert_id_hash_format(model, column_name) %} +{#- + Pseudonymous keys must be 32-char lowercase hex (md5). Any other + shape means an unhashed identifier leaked through. +-#} +select {{ column_name }} +from {{ model }} +where {{ column_name }} is not null + and {{ column_name }} !~ '^[0-9a-f]{32}$' +{% endtest %} diff --git a/warehouse/tests/generic/assert_vat_rate_bounds.sql b/warehouse/tests/generic/assert_vat_rate_bounds.sql new file mode 100644 index 0000000..340286e --- /dev/null +++ b/warehouse/tests/generic/assert_vat_rate_bounds.sql @@ -0,0 +1,10 @@ +{% test assert_vat_rate_bounds(model, column_name) %} +{#- + VAT rates must sit inside the EU-referenced bounds (0–30%). + Cross-check reference: seed/tax/vat-rates.yml. +-#} +select {{ column_name }} +from {{ model }} +where {{ column_name }} is not null + and ({{ column_name }} < 0 or {{ column_name }} > 30) +{% endtest %} diff --git a/warehouse/tests/no_null_pii_columns.sql b/warehouse/tests/no_null_pii_columns.sql new file mode 100644 index 0000000..940c4d9 --- /dev/null +++ b/warehouse/tests/no_null_pii_columns.sql @@ -0,0 +1,16 @@ +-- no-null-pii-columns: pseudonymous subject keys are the ONLY trace of +-- subjects in the warehouse; a null key means either a broken hash at +-- ingestion or a silently dropped identifier. Both are defects. +{% set checks %} + select order_key as subject_key from {{ ref('fct_orders') }} + union all + select seller_key from {{ ref('fct_orders') }} + union all + select buyer_key from {{ ref('fct_orders') }} + union all + select seller_key from {{ ref('dim_sellers') }} +{% endset %} + +select subject_key +from ({{ checks }}) as keys +where subject_key is null