From 9ee98f7bf7301c7d12134e65b739204d7522afe9 Mon Sep 17 00:00:00 2001 From: OmerBaddour Date: Mon, 3 Aug 2026 13:04:18 -0400 Subject: [PATCH] feat: give each example project a __routing__.py PR #17 moved routing into a project __routing__.py. The example projects had none, thus they did not show the feature and `clair validate` had no rules to apply. Each project now holds a routing table with a `dev` entry and a `prod` entry. example_from_init holds the scaffold template, byte for byte, because that project shows the output of `clair init`. The READMEs also named `~/.clair/profiles.yml` and a `--profile` option. Both are gone from the CLI. The READMEs now name `~/.clair/environments.yml` and `--env`. Co-Authored-By: Claude Opus 5 --- README.md | 10 ++++- example_projects/example_1/README.md | 31 +++++++++++++- example_projects/example_1/__routing__.py | 42 +++++++++++++++++++ example_projects/example_2/README.md | 31 +++++++++++++- example_projects/example_2/__routing__.py | 42 +++++++++++++++++++ example_projects/example_3/README.md | 33 +++++++++++++-- example_projects/example_3/__routing__.py | 42 +++++++++++++++++++ example_projects/example_4/README.md | 31 +++++++++++++- example_projects/example_4/__routing__.py | 42 +++++++++++++++++++ .../example_from_init/__routing__.py | 41 ++++++++++++++++++ 10 files changed, 335 insertions(+), 10 deletions(-) create mode 100644 example_projects/example_1/__routing__.py create mode 100644 example_projects/example_2/__routing__.py create mode 100644 example_projects/example_3/__routing__.py create mode 100644 example_projects/example_4/__routing__.py create mode 100644 example_projects/example_from_init/__routing__.py diff --git a/README.md b/README.md index f914840..ea9fc75 100644 --- a/README.md +++ b/README.md @@ -476,5 +476,13 @@ Example projects are included under `example_projects/`: | `example_1` | A minimal 4-Trouve events pipeline with VARIANT flattening | | `example_2` | A 50-Trouve e-commerce warehouse across 4 layers (source → refined → derived → reports) | | `example_3` | Incremental APPEND and UPSERT strategies | +| `example_4` | A `PandasTrouve` that gives a DataFrame instead of SQL | +| `example_from_init` | The files that `clair init` writes | -Each includes a `setup.sql` to create and seed the source tables and a `verify.sql` to inspect the results. +Each project holds a `__routing__.py` with a `dev` entry and a `prod` entry. The `dev` entry +reads `CLAIR_USER`, thus each person writes to a separate database. Run +`clair validate --project example_projects/example_1` to apply the rules without a Snowflake +connection. + +The README of each project holds the SQL that creates and seeds the source tables, and the SQL +that inspects the results. diff --git a/example_projects/example_1/README.md b/example_projects/example_1/README.md index 084cf10..da30188 100644 --- a/example_projects/example_1/README.md +++ b/example_projects/example_1/README.md @@ -17,7 +17,7 @@ Lineage: `source.events` → `refined.events` → `derived.daily_event_counts` You need a Snowflake account with: -- A profile configured at `~/.clair/profiles.yml` (the `local` profile is used below) +- An environment configured at `~/.clair/environments.yml` (the examples below use `dev`) - The source table `example_1_database.source.events` created and populated ### Install clair @@ -55,6 +55,33 @@ from values as t(event_id, user_id, event_type, occurred_at, properties); ``` +## Routing + +`__routing__.py` at the root of this project holds the routing rules. It has two entries: + +| Environment | Physical write target | +|---|---| +| `dev` | `example_1_database_` | +| `prod` | `example_1_database` — the logical names | + +The `dev` entry reads the `CLAIR_USER` environment variable, thus each person writes to a +separate database. Set it before you run the `dev` environment: + +```bash +export CLAIR_USER=alice +``` + +`clair validate` applies the rules to every Trouve and needs no Snowflake connection: + +```bash +clair validate --project example_projects/example_1 +clair validate --project example_projects/example_1 --env prod +``` + +SOURCE Trouves never route. The `source` schema keeps its logical name in every environment. + +See the [routing guide](../../site_docs/docs/guides/routing.md) for the full rules. + ## Running the example From the project root (`clair/`): @@ -64,7 +91,7 @@ From the project root (`clair/`): clair compile --project example_projects/example_1 # Run (executes against Snowflake) -clair run --project example_projects/example_1 --profile local +clair run --project example_projects/example_1 --env dev ``` After running, you should see three new tables in Snowflake: diff --git a/example_projects/example_1/__routing__.py b/example_projects/example_1/__routing__.py new file mode 100644 index 0000000..45d4493 --- /dev/null +++ b/example_projects/example_1/__routing__.py @@ -0,0 +1,42 @@ +"""Clair routing -- gives each environment its physical write target. + +Each entry names one environment. The name matches a top-level key in +~/.clair/environments.yml. The route method accepts the logical TrouveAddress +and gives the physical TrouveAddress. SOURCE Trouves never route. + +Commit this file. It holds no credentials. +Run `clair validate --project example_projects/example_1` to apply the entries. +""" + +import os + +from clair import RoutingEntry, RoutingTable, TrouveAddress + + +class DeveloperRouting(RoutingEntry): + """Each person writes to a separate database. + + With CLAIR_USER=alice, example_1_database.refined.events becomes + example_1_database_ALICE.refined.events. + """ + + environment_name: str = "dev" + user_variable: str = "CLAIR_USER" + + def route(self, trouve_address: TrouveAddress) -> TrouveAddress: + user_name = os.environ[self.user_variable].upper() + return trouve_address.model_copy( + update={"database_name": f"{trouve_address.database_name}_{user_name}"} + ) + + +class ProductionRouting(RoutingEntry): + """Production writes to the logical names, so the address stays the same.""" + + environment_name: str = "prod" + + def route(self, trouve_address: TrouveAddress) -> TrouveAddress: + return trouve_address + + +routing = RoutingTable(entries=[DeveloperRouting(), ProductionRouting()]) diff --git a/example_projects/example_2/README.md b/example_projects/example_2/README.md index b01736a..ad7bdfd 100644 --- a/example_projects/example_2/README.md +++ b/example_projects/example_2/README.md @@ -19,7 +19,7 @@ Lineage: `source.*` → `refined.*` → `derived.*` → `reports.*` You need a Snowflake account with: -- A profile configured at `~/.clair/profiles.yml` (the `local` profile is used below) +- An environment configured at `~/.clair/environments.yml` (the examples below use `dev`) - All ten source tables created and seeded (SQL below) ### Install clair @@ -192,6 +192,33 @@ as t(return_id, order_item_id, user_id, reason, status, created_at, refund_amoun --- +## Routing + +`__routing__.py` at the root of this project holds the routing rules. It has two entries: + +| Environment | Physical write target | +|---|---| +| `dev` | `example_2_database_` | +| `prod` | `example_2_database` — the logical names | + +The `dev` entry reads the `CLAIR_USER` environment variable, thus each person writes to a +separate database. Set it before you run the `dev` environment: + +```bash +export CLAIR_USER=alice +``` + +`clair validate` applies the rules to every Trouve and needs no Snowflake connection: + +```bash +clair validate --project example_projects/example_2 +clair validate --project example_projects/example_2 --env prod +``` + +SOURCE Trouves never route. The `source` schema keeps its logical name in every environment. + +See the [routing guide](../../site_docs/docs/guides/routing.md) for the full rules. + ## Running the example From the project root (`clair/`): @@ -201,7 +228,7 @@ From the project root (`clair/`): clair compile --project example_projects/example_2 # Run (executes against Snowflake) -clair run --project example_projects/example_2 --profile local +clair run --project example_projects/example_2 --env dev ``` After running, 40 new tables will be created across three schemas. A few interesting ones to verify: diff --git a/example_projects/example_2/__routing__.py b/example_projects/example_2/__routing__.py new file mode 100644 index 0000000..793cccc --- /dev/null +++ b/example_projects/example_2/__routing__.py @@ -0,0 +1,42 @@ +"""Clair routing -- gives each environment its physical write target. + +Each entry names one environment. The name matches a top-level key in +~/.clair/environments.yml. The route method accepts the logical TrouveAddress +and gives the physical TrouveAddress. SOURCE Trouves never route. + +Commit this file. It holds no credentials. +Run `clair validate --project example_projects/example_2` to apply the entries. +""" + +import os + +from clair import RoutingEntry, RoutingTable, TrouveAddress + + +class DeveloperRouting(RoutingEntry): + """Each person writes to a separate database. + + With CLAIR_USER=alice, example_2_database.refined.orders becomes + example_2_database_ALICE.refined.orders. + """ + + environment_name: str = "dev" + user_variable: str = "CLAIR_USER" + + def route(self, trouve_address: TrouveAddress) -> TrouveAddress: + user_name = os.environ[self.user_variable].upper() + return trouve_address.model_copy( + update={"database_name": f"{trouve_address.database_name}_{user_name}"} + ) + + +class ProductionRouting(RoutingEntry): + """Production writes to the logical names, so the address stays the same.""" + + environment_name: str = "prod" + + def route(self, trouve_address: TrouveAddress) -> TrouveAddress: + return trouve_address + + +routing = RoutingTable(entries=[DeveloperRouting(), ProductionRouting()]) diff --git a/example_projects/example_3/README.md b/example_projects/example_3/README.md index 7b7b382..6f605c1 100644 --- a/example_projects/example_3/README.md +++ b/example_projects/example_3/README.md @@ -21,7 +21,7 @@ source.orders (SOURCE) ## Prerequisites -- Snowflake account with a profile at `~/.clair/profiles.yml` (examples below use `local`) +- Snowflake account with an environment at `~/.clair/environments.yml` (examples below use `dev`) - clair installed: from the repo root run `uv sync && source .venv/bin/activate` ## Setup: create the source table @@ -44,6 +44,33 @@ from values as t(order_id, customer_id, order_status, amount, created_at, updated_at); ``` +## Routing + +`__routing__.py` at the root of this project holds the routing rules. It has two entries: + +| Environment | Physical write target | +|---|---| +| `dev` | `example_3_database_` | +| `prod` | `example_3_database` — the logical names | + +The `dev` entry reads the `CLAIR_USER` environment variable, thus each person writes to a +separate database. Set it before you run the `dev` environment: + +```bash +export CLAIR_USER=alice +``` + +`clair validate` applies the rules to every Trouve and needs no Snowflake connection: + +```bash +clair validate --project example_projects/example_3 +clair validate --project example_projects/example_3 --env prod +``` + +SOURCE Trouves never route. The `source` schema keeps its logical name in every environment. + +See the [routing guide](../../site_docs/docs/guides/routing.md) for the full rules. + ## Running the example All commands run from the repo root (`clair/`). @@ -63,7 +90,7 @@ clair compile --project example_projects/example_3 --run-mode incremental Creates all tables from scratch. Use this to initialise the derived tables before testing incremental. ```bash -clair run --project example_projects/example_3 --profile local +clair run --project example_projects/example_3 --env dev ``` Verify: @@ -86,7 +113,7 @@ insert into example_3_database.source.orders values ### 4. Incremental run ```bash -clair run --project example_projects/example_3 --profile local --run-mode incremental +clair run --project example_projects/example_3 --env dev --run-mode incremental ``` Expected behaviour: diff --git a/example_projects/example_3/__routing__.py b/example_projects/example_3/__routing__.py new file mode 100644 index 0000000..64415ca --- /dev/null +++ b/example_projects/example_3/__routing__.py @@ -0,0 +1,42 @@ +"""Clair routing -- gives each environment its physical write target. + +Each entry names one environment. The name matches a top-level key in +~/.clair/environments.yml. The route method accepts the logical TrouveAddress +and gives the physical TrouveAddress. SOURCE Trouves never route. + +Commit this file. It holds no credentials. +Run `clair validate --project example_projects/example_3` to apply the entries. +""" + +import os + +from clair import RoutingEntry, RoutingTable, TrouveAddress + + +class DeveloperRouting(RoutingEntry): + """Each person writes to a separate database. + + With CLAIR_USER=alice, example_3_database.refined.orders becomes + example_3_database_ALICE.refined.orders. + """ + + environment_name: str = "dev" + user_variable: str = "CLAIR_USER" + + def route(self, trouve_address: TrouveAddress) -> TrouveAddress: + user_name = os.environ[self.user_variable].upper() + return trouve_address.model_copy( + update={"database_name": f"{trouve_address.database_name}_{user_name}"} + ) + + +class ProductionRouting(RoutingEntry): + """Production writes to the logical names, so the address stays the same.""" + + environment_name: str = "prod" + + def route(self, trouve_address: TrouveAddress) -> TrouveAddress: + return trouve_address + + +routing = RoutingTable(entries=[DeveloperRouting(), ProductionRouting()]) diff --git a/example_projects/example_4/README.md b/example_projects/example_4/README.md index 992e70a..e9d1586 100644 --- a/example_projects/example_4/README.md +++ b/example_projects/example_4/README.md @@ -17,7 +17,7 @@ Lineage: `source.events` → `refined.events` → `derived.daily_event_counts` You need a Snowflake account with: -- A profile configured at `~/.clair/profiles.yml` (the `local` profile is used below) +- An environment configured at `~/.clair/environments.yml` (the examples below use `dev`) - The source table `example_4_database.source.events` created and populated ### Install clair @@ -55,6 +55,33 @@ from values as t(event_id, user_id, event_type, occurred_at, properties); ``` +## Routing + +`__routing__.py` at the root of this project holds the routing rules. It has two entries: + +| Environment | Physical write target | +|---|---| +| `dev` | `example_4_database_` | +| `prod` | `example_4_database` — the logical names | + +The `dev` entry reads the `CLAIR_USER` environment variable, thus each person writes to a +separate database. Set it before you run the `dev` environment: + +```bash +export CLAIR_USER=alice +``` + +`clair validate` applies the rules to every Trouve and needs no Snowflake connection: + +```bash +clair validate --project example_projects/example_4 +clair validate --project example_projects/example_4 --env prod +``` + +SOURCE Trouves never route. The `source` schema keeps its logical name in every environment. + +See the [routing guide](../../site_docs/docs/guides/routing.md) for the full rules. + ## Running the example From the project root (`clair/`): @@ -64,7 +91,7 @@ From the project root (`clair/`): clair compile --project example_projects/example_4 # Run (executes against Snowflake) -clair run --project example_projects/example_4 --profile dev +clair run --project example_projects/example_4 --env dev ``` After running, you should see three new tables in Snowflake: diff --git a/example_projects/example_4/__routing__.py b/example_projects/example_4/__routing__.py new file mode 100644 index 0000000..ecb98b1 --- /dev/null +++ b/example_projects/example_4/__routing__.py @@ -0,0 +1,42 @@ +"""Clair routing -- gives each environment its physical write target. + +Each entry names one environment. The name matches a top-level key in +~/.clair/environments.yml. The route method accepts the logical TrouveAddress +and gives the physical TrouveAddress. SOURCE Trouves never route. + +Commit this file. It holds no credentials. +Run `clair validate --project example_projects/example_4` to apply the entries. +""" + +import os + +from clair import RoutingEntry, RoutingTable, TrouveAddress + + +class DeveloperRouting(RoutingEntry): + """Each person writes to a separate database. + + With CLAIR_USER=alice, example_4_database.refined.events becomes + example_4_database_ALICE.refined.events. + """ + + environment_name: str = "dev" + user_variable: str = "CLAIR_USER" + + def route(self, trouve_address: TrouveAddress) -> TrouveAddress: + user_name = os.environ[self.user_variable].upper() + return trouve_address.model_copy( + update={"database_name": f"{trouve_address.database_name}_{user_name}"} + ) + + +class ProductionRouting(RoutingEntry): + """Production writes to the logical names, so the address stays the same.""" + + environment_name: str = "prod" + + def route(self, trouve_address: TrouveAddress) -> TrouveAddress: + return trouve_address + + +routing = RoutingTable(entries=[DeveloperRouting(), ProductionRouting()]) diff --git a/example_projects/example_from_init/__routing__.py b/example_projects/example_from_init/__routing__.py new file mode 100644 index 0000000..7d2c378 --- /dev/null +++ b/example_projects/example_from_init/__routing__.py @@ -0,0 +1,41 @@ +"""Clair routing -- gives each environment its physical write target. + +Each entry names one environment. The name matches a top-level key in +~/.clair/environments.yml. The route method accepts the logical TrouveAddress +and gives the physical TrouveAddress. SOURCE Trouves never route. + +Commit this file. It holds no credentials. +Run `clair validate` to apply the entries to every Trouve in the project. +""" + +import os + +from clair import RoutingEntry, RoutingTable, TrouveAddress + + +class DeveloperRouting(RoutingEntry): + """Each person writes to a separate database. + + Set CLAIR_USER to your name before you run clair. + """ + + environment_name: str = "dev" + user_variable: str = "CLAIR_USER" + + def route(self, trouve_address: TrouveAddress) -> TrouveAddress: + user_name = os.environ[self.user_variable].upper() + return trouve_address.model_copy( + update={"database_name": f"{trouve_address.database_name}_{user_name}"} + ) + + +class ProductionRouting(RoutingEntry): + """Production writes to the logical names, so the address stays the same.""" + + environment_name: str = "prod" + + def route(self, trouve_address: TrouveAddress) -> TrouveAddress: + return trouve_address + + +routing = RoutingTable(entries=[DeveloperRouting(), ProductionRouting()])