Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 22 additions & 0 deletions .editorconfig
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# EditorConfig — consistent diffs matter for auditable pipeline changes.
root = true

[*]
charset = utf-8
end_of_line = lf
insert_final_newline = true
trim_trailing_whitespace = true
indent_style = space
indent_size = 2

[Makefile]
indent_style = tab

[*.md]
trim_trailing_whitespace = false

[*.py]
indent_size = 4

[*.sql]
indent_size = 2
21 changes: 21 additions & 0 deletions .envrc.example
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
# .envrc.example — operator environment template (direnv-compatible).
# Copy to .envrc and fill in from Vaultwarden. .envrc is gitignored.
# NEVER commit real values; NEVER paste tokens into issue trackers or chat.
# ADR-0002: analytics never holds production credentials — these point at
# the staging/dev warehouse and scoped read-replica extract roles only.

# ── Analytics warehouse (dbt profile: warehouse/profiles.yml.example) ─────
export WH_HOST=""
export WH_PORT="5432"
export WH_DB="analytics_dev"
export WH_SCHEMA="dbt"
export WH_USER=""
export WH_PASSWORD=""

# ── Extraction (read-replica only, least privilege — ADR-0002) ────────────
export EXTRACT_PG_DSN="" # read-only role on the read replica
export EXTRACT_STRIPE_KEY="" # restricted key: charges/payouts read only # gitleaks:allow

# ── Quality & freshness gates ─────────────────────────────────────────────
export GE_CONTEXT_ROOT="quality"
# export SLACK_WEBHOOK_URL="" # freshness-monitor alerts (optional)
19 changes: 19 additions & 0 deletions .gitattributes
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
# .gitattributes — diff semantics for the data platform.
# Data artifacts get whole-line semantics; binaries never fake text diffs.

* text=auto eol=lf

# Registers, seeds and translation memories are data, not prose
*.csv text eol=lf
*.yml text eol=lf
*.yaml text eol=lf

# Binary artifacts — no textual diff, no merge guessing
*.png binary
*.jpg binary
*.jpeg binary
*.zip binary

# Language identification for review tooling
*.md linguist-documentation
*.sql linguist-language SQL
31 changes: 31 additions & 0 deletions .github/CODEOWNERS
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
# CODEOWNERS — routes reviews; the approving-review COUNT is enforced by
# branch protection / rulesets (see CONTRIBUTING.md, risk classes).
# CODEOWNERS alone cannot require N approvers — do not rely on it for that.

# Default: data platform operators
* @journeyoflife-org/data-operators

# Catalog & classification — the governance record (§3 DPO gate)
/governance/ @journeyoflife-org/data-operators @journeyoflife-org/compliance

# Anything touching personal-data machinery — DPO required (§3)
/ingestion/pipelines/pseudonymizer/ @journeyoflife-org/data-operators @journeyoflife-org/compliance
/warehouse/models/marts/compliance/ @journeyoflife-org/data-operators @journeyoflife-org/compliance
/lifecycle/ @journeyoflife-org/data-operators @journeyoflife-org/compliance
/governance/classification.md @journeyoflife-org/compliance
/governance/retention-map.md @journeyoflife-org/compliance

# Taxonomy is the marketplace domain model — product owner in the loop
/seed/taxonomy/ @journeyoflife-org/data-operators @journeyoflife-org/marketplace-product
/seed/tax/ @journeyoflife-org/data-operators @journeyoflife-org/finance

# Finance marts
/warehouse/models/marts/finance/ @journeyoflife-org/data-operators @journeyoflife-org/finance

# Automation touching pipelines needs security in the loop
/.github/workflows/ @journeyoflife-org/data-operators @journeyoflife-org/security
/scripts/ @journeyoflife-org/data-operators @journeyoflife-org/security

# PII scanning & erasure verification — DPO + security
/scripts/scan-warehouse-pii.py @journeyoflife-org/data-operators @journeyoflife-org/compliance
/scripts/verify-anonymization.py @journeyoflife-org/data-operators @journeyoflife-org/compliance
48 changes: 48 additions & 0 deletions .github/ISSUE_TEMPLATE/bug_report.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,48 @@
name: Bug report
description: Defect in a pipeline, model, seed file, or this repo's automation (NOT an incident — use SECURITY.md for those)
labels: ["bug"]
body:
- type: textarea
id: behavior
attributes:
label: Observed vs expected behavior
description: What is wrong, what should be true. Do NOT paste personal data or credentials into this issue.
validations:
required: true
- type: dropdown
id: area
attributes:
label: Affected area
options:
- warehouse-models
- ingestion
- quality
- seed-taxonomy
- governance
- lifecycle
- ml
- ci-cd
- scripts
- docs
- other
validations:
required: true
- type: dropdown
id: severity
attributes:
label: Severity
description: If personal data may have leaked into the warehouse, STOP and follow SECURITY.md instead.
options:
- minor (cosmetic/dashboards)
- moderate (wrong metric)
- major (wrong money/VAT figure)
- critical (pseudonymization or retention defect)
validations:
required: true
- type: textarea
id: repro
attributes:
label: Reproduction / evidence
description: Pipeline/model name, failing test or expectation, date window affected.
validations:
required: true
51 changes: 51 additions & 0 deletions .github/ISSUE_TEMPLATE/dataset_request.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
name: Dataset request
description: Register a new dataset — forces purpose, classification, and retention declaration before any code
labels: ["dataset", "governance"]
body:
- type: input
id: dataset
attributes:
label: Dataset name
description: Stable identifier, e.g. fct_payouts or seed_geo_lockers.
validations:
required: true
- type: textarea
id: purpose
attributes:
label: Purpose (RoPA linkage)
description: >
Why does this dataset exist and which RoPA purpose in
jol-m-compliance covers it? If no personal data is involved,
state that explicitly.
validations:
required: true
- type: dropdown
id: classification
attributes:
label: Classification tier
description: Per governance/classification.md.
options: [PUBLIC, INTERNAL, CONFIDENTIAL, RESTRICTED]
validations:
required: true
- type: dropdown
id: retention
attributes:
label: Retention class
description: Per governance/retention-map.md; "none" only for synthetic/reference data.
options: [short-term (≤ 90 days), operational (≤ 2 years), statutory (≤ 10 years), indefinite-reference, none (synthetic)]
validations:
required: true
- type: textarea
id: ownership
attributes:
label: Owner & steward
description: Business owner and technical steward names/teams — orphan datasets are blocked by catalog-lint.
validations:
required: true
- type: textarea
id: sources
attributes:
label: Sources & lineage
description: Upstream systems and fields; which identifiers are present and how they are pseudonymized.
validations:
required: true
34 changes: 34 additions & 0 deletions .github/ISSUE_TEMPLATE/feature_request.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
name: Feature request
description: New capability for the data platform or its automation
labels: ["enhancement"]
body:
- type: textarea
id: capability
attributes:
label: Capability
description: What should the data platform/repository be able to do, and why now?
validations:
required: true
- type: textarea
id: driver
attributes:
label: Business / regulatory driver
description: Which need drives this (VAT-OSS reporting, seller health, DSA transparency, retention duty, cost)?
validations:
required: true
- type: dropdown
id: area
attributes:
label: Target area
options:
- warehouse-models
- ingestion
- quality
- seed-taxonomy
- governance
- lifecycle
- ml
- automation
- other
validations:
required: true
50 changes: 50 additions & 0 deletions .github/ISSUE_TEMPLATE/pii_incident.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
name: PII incident
description: Personal data detected in analytics or committed data — DPO notification. NEVER paste the data itself.
labels: ["pii", "incident", "dpo"]
body:
- type: markdown
attributes:
value: |
**Stop:** do not include the personal data itself, record
contents, or more rows than strictly needed to locate the issue.
This issue carries triage metadata only. Follow
SECURITY.md / docs/runbooks/pii-detected-in-warehouse.md.
- type: dropdown
id: severity
attributes:
label: Severity
options:
- low (synthetic-looking false positive to verify)
- medium (PII-shaped values in a non-restricted dataset)
- high (cleartext identifiers downstream of ingestion)
- critical (direct identifiers in marts or committed to git)
validations:
required: true
- type: input
id: location
attributes:
label: Where detected
description: Table/model/file path and column — no row contents.
validations:
required: true
- type: input
id: detected_at
attributes:
label: Detected at (UTC)
description: ISO 8601 timestamp.
validations:
required: true
- type: dropdown
id: dpo_notified
attributes:
label: DPO notified
options: ["yes", "no — please page"]
validations:
required: true
- type: textarea
id: containment
attributes:
label: Containment actions taken
description: Quarantine, pipeline pause, purge request — what has already been done.
validations:
required: false
30 changes: 30 additions & 0 deletions .github/PULL_REQUEST_TEMPLATE.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
## Change summary

<!-- One paragraph: what changes, why, link to the intake issue. -->

## Dataset checklist (mandatory — CONTRIBUTING.md)

- Data classification: [ ] PUBLIC [ ] INTERNAL [ ] CONFIDENTIAL [ ] RESTRICTED
- RoPA purpose reference: <!-- jol-m-compliance RoPA entry id, or "n/a (no personal data)" -->
- Retention class: <!-- per governance/retention-map.md, or "n/a" -->
- Catalog entry + ownership row added for new datasets: [ ] yes / n/a
- No personal data committed (fixtures are synthetic): [ ] confirmed
- No warehouse credentials or connection strings introduced: [ ] confirmed

## Warehouse impact

- Models added/changed: <!-- names, or "none" -->
- Staging still pseudonymous (hashed IDs, no names/emails): [ ] confirmed / n/a
- dbt tests cover the change (unique/not_null/relationships + custom): [ ] yes / n/a
- Backfill required: [ ] yes / no — <!-- window & owner -->

## Seed impact

- Taxonomy change class: [ ] MAJOR (breaking for consumers) [ ] MINOR [ ] PATCH [ ] n/a
- Translations complete for lt/lv/et/en: [ ] yes / n/a
- Fixture regeneration documented (new identities on reseed): [ ] yes / n/a

## Compliance notes

<!-- GDPR / VAT-OSS / DSA analytics relevance, DPO consulted (yes/no),
or "none". -->
20 changes: 20 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# Dependabot — supply-chain hygiene for dependency ecosystems present in
# this repo. SOC 2 CC7.1: vulnerabilities in tooling are monitored.
version: 2
updates:
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "weekly"
groups:
actions:
patterns: ["*"]
commit-message:
prefix: "chore(deps)"

- package-ecosystem: "pip"
directory: "/"
schedule:
interval: "monthly"
commit-message:
prefix: "chore(deps)"
58 changes: 58 additions & 0 deletions .github/workflows/ci-raw-schema.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
-- ci-raw-schema.sql — synthetic source stand-in for dbt-ci ONLY.
-- Mirrors ingestion/contracts/* field allow-lists so staging models can
-- build in CI without ever touching the real read replica. Values are
-- synthetic literals; keep them that way (PII scan runs on this file).

create schema if not exists raw;

create table raw.orders (
id bigint primary key,
buyer_id bigint not null,
seller_id bigint not null,
status text not null,
amount_cents integer not null,
currency char(3) not null,
category_code text not null,
country_code char(2) not null,
vat_rate_pct numeric(5, 2) not null,
created_at timestamptz not null
);

create table raw.products (
id bigint primary key,
seller_id bigint not null,
category_code text not null,
title text not null,
price_cents integer not null,
currency char(3) not null,
status text not null,
created_at timestamptz not null
);

create table raw.users (
id bigint primary key,
role text not null,
country_code char(2) not null,
consent_marketing boolean not null,
created_at timestamptz not null
);

insert into raw.orders values
(1, 101, 201, 'paid', 12100, 'EUR', 'vestments', 'LT', 21.00, '2026-08-01 10:00:00+00'),
(2, 102, 201, 'paid', 5900, 'EUR', 'icons', 'LV', 21.00, '2026-08-02 11:00:00+00'),
(3, 103, 202, 'shipped', 3400, 'EUR', 'books', 'EE', 22.00, '2026-08-03 12:00:00+00'),
(4, 104, 202, 'refunded', 9900, 'EUR', 'funeral', 'LT', 21.00, '2026-08-04 13:00:00+00'),
(5, 105, 203, 'paid', 25000, 'EUR', 'services', 'LT', 21.00, '2026-08-05 14:00:00+00');

insert into raw.products values
(1001, 201, 'vestments', 'Synthetic chasuble', 12100, 'EUR', 'active', '2026-07-01 09:00:00+00'),
(1002, 201, 'icons', 'Synthetic icon panel', 5900, 'EUR', 'active', '2026-07-02 09:00:00+00'),
(1003, 202, 'books', 'Synthetic hymnal', 3400, 'EUR', 'active', '2026-07-03 09:00:00+00'),
(1004, 203, 'services', 'Synthetic ceremony svc', 25000, 'EUR', 'inactive', '2026-07-04 09:00:00+00');

insert into raw.users values
(101, 'buyer', 'LT', true, '2026-01-10 08:00:00+00'),
(102, 'buyer', 'LV', false, '2026-02-11 08:00:00+00'),
(201, 'seller', 'LT', true, '2026-03-12 08:00:00+00'),
(202, 'seller', 'EE', false, '2026-04-13 08:00:00+00'),
(203, 'seller', 'LT', true, '2026-05-14 08:00:00+00');
Loading
Loading