From f3975bce6bb1e1bef0e5adfe9dae9d862975dd33 Mon Sep 17 00:00:00 2001 From: Joypal Navaneeth Date: Tue, 1 Sep 2026 13:59:43 +0530 Subject: [PATCH] docs: architecture/sequence/module diagrams, real CLI screenshots, badges, CI --- .github/workflows/tests.yml | 23 +++ README.md | 196 +++++++++++++------------- docs/ADDING_A_VENDOR.md | 112 +++++++++++++++ docs/ARCHITECTURE.md | 226 ++++++++++++++++++++++++++++++ docs/assets/cli-discover.svg | 106 ++++++++++++++ docs/assets/cli-enrich.svg | 94 +++++++++++++ docs/assets/cli-push-crm.svg | 94 +++++++++++++ docs/assets/cli-status.svg | 84 +++++++++++ scripts/generate_readme_assets.py | 77 ++++++++++ 9 files changed, 916 insertions(+), 96 deletions(-) create mode 100644 .github/workflows/tests.yml create mode 100644 docs/ADDING_A_VENDOR.md create mode 100644 docs/ARCHITECTURE.md create mode 100644 docs/assets/cli-discover.svg create mode 100644 docs/assets/cli-enrich.svg create mode 100644 docs/assets/cli-push-crm.svg create mode 100644 docs/assets/cli-status.svg create mode 100644 scripts/generate_readme_assets.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..4f213bf --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,23 @@ +name: Tests + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install project (with dev dependencies) + run: pip install -e ".[dev]" + + - name: Run tests + run: pytest -v diff --git a/README.md b/README.md index 5a5cf65..5a09b40 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,9 @@ # Data Enrichment System +[![Tests](https://github.com/codeapplied/data-enrichment-system/actions/workflows/tests.yml/badge.svg)](https://github.com/codeapplied/data-enrichment-system/actions/workflows/tests.yml) +[![Python 3.11+](https://img.shields.io/badge/python-3.11%2B-blue.svg)](pyproject.toml) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE) + A company-to-decision-maker contact enrichment pipeline — rebuilt as a generalized, open-source system. @@ -8,115 +12,85 @@ generic, sanitized logic only. No employer-specific data, workflows, or branding ## Status -🚧 Early development. The originally-scoped pipeline is fully built end to -end: domain discovery, contact enrichment, and CRM push, all wired into -the CLI/DB pipeline and unit-tested. See +The originally-scoped pipeline is complete end to end: domain discovery +(confidence-gated), contact enrichment (department-priority ranked), and +CRM push (3-phase find-or-create), all wired into the CLI/DB pipeline, a +CI-tested suite (46 tests — run `pytest`), and a real (not templated) +Pipedrive client alongside network-free sandbox backends for every +external integration point. See [open issues](https://github.com/codeapplied/data-enrichment-system/issues) for what's next (a real CSV/Excel import path, reconciliation checks, and -wiring in real vendors in place of the sandbox backends). +wiring in real search/enrichment vendors in place of their sandboxes). -Try it end to end with zero setup beyond `uv venv && uv pip install -e .`: +## Architecture +```mermaid +flowchart LR + A["Raw record"] --> B["Domain discovery"] + B --> C{"Confidence gate"} + C -->|"high"| D["Contact enrichment"] + C -->|"else"| E["needs_review
(parked)"] + D --> F{"Contacts found?"} + F -->|"yes"| G["CRM push
org -> contact -> lead"] + F -->|"no"| E ``` -dataenrich init -dataenrich seed-demo -dataenrich discover --apply # resolves high-confidence orgs, parks the rest -dataenrich enrich --apply # ranks+stores contacts for domain-resolved orgs -dataenrich push-crm --apply # organization -> contact -> lead, find-or-create -dataenrich status -``` + +Every stage logs a `SyncLog` row, which `dataenrich status` reads. Full +breakdown, sequence diagram, module responsibilities, and the reasoning +behind the key design decisions (the confidence gate, field-level +authority, true dry-run safety, department-priority ranking, the +overwrite-protection gate): [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md). + +## In action + +Real output from the actual CLI (sandbox data, default rules) — not +mockups. Regenerate these with `python scripts/generate_readme_assets.py` +after any CLI change. + +**`dataenrich discover --apply`** — resolves 2 of 5 sample organizations at +high confidence, parks the other 3 (one low-confidence single-source match +with an aggregator false-positive correctly excluded, one +agreement-without-confirmation, one fully unresolved): + +![dataenrich discover --apply output](docs/assets/cli-discover.svg) + +**`dataenrich enrich --apply`** — ranks and stores contacts for both +domain-resolved organizations: + +![dataenrich enrich --apply output](docs/assets/cli-enrich.svg) + +**`dataenrich push-crm`** (plan-only) — the first organization matches the +sandbox's pre-seeded CRM record and reuses its ID; the second is a genuine +`(new)`, and no CRM write happens either way in dry-run: + +![dataenrich push-crm output](docs/assets/cli-push-crm.svg) + +**`dataenrich status`** — every stage's `SyncLog` row: + +![dataenrich status output](docs/assets/cli-status.svg) ## Domain discovery -`src/dataenrich/discovery/` — the confidence-gated engine that turns a raw -company/project/address record into a resolved domain, or parks it for -manual review: - -- `query.py` — builds two deliberately different query shapes per company - (not a naive repeat); numbered/shell-entity names (e.g. "1234567 Ontario - Inc.") get a project-name/address-first query instead of a name-first one, - since their own name is nearly useless for search. -- `confidence.py` — the actual gate: two independent search signals - agreeing *and* a live domain confirmation is "high" (only this tier - proceeds automatically); agreement without confirmation is "medium"; - a single confirmed signal is "low"; anything else is "unresolved" and - parked for manual review, never silently dropped or silently pushed - downstream. -- `exclusion.py` — a living blocklist (aggregators, directories, news - sites, generic tokens) applied before a result ever becomes a vote. -- `base.py` — the `SearchClient` interface and `discover_domain()` - orchestrator that ties the above together. -- `sandbox_client.py` — the default demo backend: bundled fixture data, - zero network calls, zero API keys, proves the whole pipeline end-to-end. -- `real_search_client_template.py` + `http_head_check.py` — template for - wiring in a real search vendor plus a working, real (not sandboxed) HTTP - HEAD confirmation check. - -Run `pytest` to see the confidence gate exercised against 5 scenarios -(clean agreement, a shell-entity name, an aggregator false-positive -correctly excluded, agreement-without-confirmation, and an all-excluded -case) — fully offline, no network or API keys required. +`src/dataenrich/discovery/` — confidence-gated: two independent search +signals agreeing *and* a live domain confirmation is "high" (the only tier +that proceeds automatically); numbered/shell-entity company names (e.g. +"1234567 Ontario Inc.") get a project/address-first query instead of a +name-first one. Ships with a network-free sandbox backend by default — +see [ARCHITECTURE.md](docs/ARCHITECTURE.md) for the full module breakdown. ## Contact enrichment -`src/dataenrich/enrichment/` — turns a resolved domain into ranked, -stored contacts: - -- `base.py` — the `ContactEnrichmentClient` interface and `RawContact`/ - `EnrichmentResult` types. -- `ranking.py` — re-sorts contacts by department relevance instead of - the vendor's default seniority-first order. A department not in the - configured priority list sorts last; ties keep the vendor's original - order (stable sort). -- `sandbox_client.py` — the default demo backend: bundled fixture data, - including one domain with zero contacts on purpose (a "thin" result — - a correct domain the vendor just has nothing indexed for, a different - failure mode from a wrong domain). -- `real_client_template.py` — template for wiring in a real vendor - (e.g. a Hunter.io-style domain-based contact API). +`src/dataenrich/enrichment/` — turns a resolved domain into ranked, stored +contacts, re-sorted by department relevance instead of the vendor's +default seniority-first order (a real, measured fix in the reference +system this project is modeled on). ## CRM push `src/dataenrich/crm/` — three ordered find-or-create phases (organization -→ contact → lead), so re-running is always safe and nothing gets -duplicated: - -- `base.py` — the `CRMClient` interface. Organizations dedup on - normalized website (not name — names vary too much in formatting); - contacts dedup on email; leads dedup on (organization, title). -- `overwrite_gate.py` — the shared "diff against the live value, only - auto-approve the safe cases" gate every write funnels through, rather - than each call site trusting its own judgment: empty-in-CRM or - identical values auto-write, a real conflict is reported and skipped, - never silently overwritten. -- `sandbox_client.py` — the default demo backend: an in-memory fake CRM, - pre-seeded with one "existing" organization so dedup against - *pre-existing* CRM data is actually exercised, not just dedup within a - single run. -- `pipedrive_client.py` — a real, working Pipedrive REST API v1 client - (not just a template — the same pattern already proven in the sibling - [tender-tracking-system](https://github.com/codeapplied/tender-tracking-system) - repo). Honest limitation: Pipedrive has no built-in "website" field on - organizations, so org lookup falls back to name-based search; contact - lookup by email has no such limitation. No live Pipedrive account was - available to test against in this session — verified via mocked-HTTP- - shape tests only. - -A lead is modeled per *project*, not per company — a single developer can -have multiple concurrently active projects, each needing independent -outreach tracking. Dry-run is genuinely safe here too: no `create_*` -method is ever called on the CRM client in plan-only mode, not even -against the sandbox's in-memory fake. - -## Architecture - -Raw records (company name, project name, address) → domain discovery (confidence-gated, -cross-checked across independent signals) → only high-confidence domains proceed -automatically, everything else is parked for manual review → contact enrichment against -a domain-based third-party API → contacts re-ranked by department relevance instead of -the vendor's default seniority-first sort → push to CRM (organization → contact → lead, -each phase gated on the previous phase's log). Every stage is logged to `SyncLog`, which -the ops CLI reads for pipeline health. +→ contact → lead), each deduplicated (website / email / (org, title)) so +re-running is always safe. A lead is modeled per *project*, not per +company. Includes a real, working Pipedrive client, not just a template. ## Setup @@ -128,6 +102,16 @@ cp config/rules.example.yaml config/rules.yaml dataenrich init ``` +Try it end to end with just the sandbox backends, no credentials needed: + +``` +dataenrich seed-demo +dataenrich discover --apply +dataenrich enrich --apply +dataenrich push-crm --apply +dataenrich status +``` + ## CLI - `dataenrich init` — create the database @@ -135,8 +119,28 @@ dataenrich init - `dataenrich rules` — show the loaded department-priority and domain-exclusion rules - `dataenrich seed-demo` — load the bundled sandbox fixture as pending organizations (demo/test data only — there's no CSV/Excel import for your own data yet) - `dataenrich discover [--apply]` — run domain discovery for pending organizations. Plan-only by default. Only "high" confidence proceeds automatically (status → `domain_resolved`); everything else is parked (`needs_review`) — field-level authority means a later run never touches a non-`pending` organization again, so a human's manual correction always sticks -- `dataenrich enrich [--apply]` — run contact enrichment for domain-resolved organizations. Plan-only by default. Contacts are ranked by department priority and stored (top 5); a domain that resolves but returns zero contacts is parked back to `needs_review` rather than marked "enriched" with nothing in it — field-level authority applies here too (only `domain_resolved` rows are ever touched) -- `dataenrich push-crm [--apply]` — push enriched organizations to the CRM: organization → contact → lead, each a find-or-create. Plan-only by default (no CRM writes at all, not even to the sandbox's in-memory fake). Field-level authority again — only `enriched` rows are ever touched +- `dataenrich enrich [--apply]` — run contact enrichment for domain-resolved organizations. Plan-only by default. Contacts are ranked by department priority and stored (top 5); a domain that resolves but returns zero contacts is parked back to `needs_review` rather than marked "enriched" with nothing in it +- `dataenrich push-crm [--apply]` — push enriched organizations to the CRM: organization → contact → lead, each a find-or-create. Plan-only by default — no CRM writes at all, not even to the sandbox's in-memory fake + +## Testing + +``` +uv pip install -e ".[dev]" +pytest +``` + +46 tests covering: the confidence gate against all 4 tiers, query +construction (including the numbered-entity anchor selection), exclusion +filtering, department-priority ranking, the overwrite-protection gate, a +real (mocked-HTTP) Pipedrive client, and every pipeline stage end-to-end +against a real temp SQLite DB — including field-level authority (a second +run never reprocesses an already-advanced row) and true dry-run safety +(no `create_*` call ever reaches a client, sandbox or real, in plan-only +mode). + +## Adding a real vendor + +See [docs/ADDING_A_VENDOR.md](docs/ADDING_A_VENDOR.md). ## License diff --git a/docs/ADDING_A_VENDOR.md b/docs/ADDING_A_VENDOR.md new file mode 100644 index 0000000..a609885 --- /dev/null +++ b/docs/ADDING_A_VENDOR.md @@ -0,0 +1,112 @@ +# Adding a real vendor + +Every external integration point in this project ships with a working, +network-free sandbox backend as the default. Swapping in a real vendor +means implementing the same interface and pointing `pipeline/*.py`'s +`_default_client()` at it instead — orchestration, the confidence gate, +ranking, and dedup logic don't change. + +## 1. Search vendor (domain discovery) + +Copy `discovery/real_search_client_template.py`, implement `search()` +against your chosen search API (Google Custom Search JSON API, Brave +Search API, SerpAPI, or similar): + +```python +# src/dataenrich/discovery/my_search_vendor.py +from .base import SearchClient, SearchResult + +class MySearchVendorClient(SearchClient): + name = "my-search-vendor" + + def __init__(self, api_key: str) -> None: + self.api_key = api_key + + def search(self, query: str) -> list[SearchResult]: + # call the vendor's API, return ranked result URLs + ... +``` + +Run **two independent instances** (different vendors, or one real vendor +plus the sandbox as a sanity check) — `discover_domain()`'s confidence +gate is only meaningful when it's cross-checking genuinely independent +signals, not the same vendor called twice. Pair it with +`http_head_check.py` (already real, no vendor needed) as the `head_check` +argument. + +Wire it into `pipeline/discover.py`'s `_default_clients()`: + +```python +def _default_clients() -> list[SearchClient]: + return [MySearchVendorClient(settings.search_api_key_primary), AnotherVendorClient(...)] +``` + +## 2. Contact-enrichment vendor + +Copy `enrichment/real_client_template.py`, implement `enrich()`: + +```python +# src/dataenrich/enrichment/my_enrichment_vendor.py +from .base import ContactEnrichmentClient, EnrichmentResult, RawContact + +class MyEnrichmentVendorClient(ContactEnrichmentClient): + name = "my-enrichment-vendor" + + def __init__(self, api_key: str) -> None: + self.api_key = api_key + + def enrich(self, domain: str) -> EnrichmentResult: + # call the vendor's domain-based contact lookup API + ... +``` + +Two things worth keeping from the reference system's own experience (see +the module's own docstring): log the *raw* contact count separately from +the count actually stored, and don't trust the vendor's default sort — +`enrichment/ranking.py`'s department-priority re-ranking runs on whatever +`enrich()` returns regardless of vendor. + +Wire it into `pipeline/enrich.py`'s `_default_client()`. + +## 3. CRM + +`crm/pipedrive_client.py` is already a real, working Pipedrive client — +not a template. If Pipedrive is your CRM, set `PIPEDRIVE_API_TOKEN` and +`PIPEDRIVE_DOMAIN` in `.env` and swap `pipeline/push_crm.py`'s +`_default_client()`: + +```python +def _default_client() -> CRMClient: + from ..crm.pipedrive_client import PipedriveClient + return PipedriveClient(settings.pipedrive_api_token, settings.pipedrive_domain) +``` + +For a different CRM, implement the `CRMClient` interface (`crm/base.py`): +`find_organization_by_website`/`create_organization`, +`find_contact_by_email`/`create_contact`, `find_lead`/`create_lead`. Keep +organization dedup on a normalized website field if your CRM has one — +`pipedrive_client.py`'s own docstring documents why it had to fall back to +name-based search instead (Pipedrive has no native website field), which +is a real, non-generic limitation worth avoiding if your CRM does better. + +Route every write through `crm/overwrite_gate.py`'s `decide_write()` +before updating a field on an already-found record — this project's +"surface a conflict, never silently overwrite" principle applies to a real +CRM exactly the way it does to the sandbox one. + +## 4. Test it + +``` +dataenrich seed-demo # or your own data, once a real import path exists +dataenrich discover # plan-only — see what it would resolve, no writes +dataenrich discover --apply +dataenrich enrich --apply +dataenrich push-crm # plan-only — no CRM writes at all, not even to sandbox +dataenrich push-crm --apply +dataenrich status +``` + +Write unit tests for a real vendor's request/response shape following +`tests/test_pipedrive_client.py`'s mocked-HTTP pattern (`unittest.mock.patch` +on `requests.Session.request`) — no live credentials needed to verify the +shape of what your client sends and how it parses what comes back. diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..b8d9473 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,226 @@ +# Architecture + +## Data flow + +```mermaid +flowchart TD + A["Raw record
company name, project name, address"] --> B["Domain discovery
discover_domain()"] + B --> C{"Confidence gate"} + C -->|"high"| D["status: domain_resolved"] + C -->|"medium / low / unresolved"| E["status: needs_review
parked, candidate domain kept if any"] + D --> F["Contact enrichment
enrich() + rank_contacts()"] + F --> G{"Raw contacts found?"} + G -->|"yes"| H["status: enriched
top-5 ranked contacts stored"] + G -->|"no (thin result)"| E + H --> I["CRM push
org -> contact -> lead, find-or-create"] + I --> J["status: pushed"] + E -.human resolves,
promotes back to pending.-> B +``` + +Every pipeline stage writes a `SyncLog` row (`domain_discovery` / +`contact_enrichment` / `crm_push`) — what `dataenrich status` reads. +`needs_review` is a dead end for automation by design: nothing in this +codebase ever re-promotes a parked organization on its own. A human +resolves it (confirms/corrects the domain, or accepts a thin result) and +moves it back to `pending`/`domain_resolved` themselves. + +## Sequence: one full pipeline run + +```mermaid +sequenceDiagram + participant CLI as dataenrich + participant Discovery as discover_domain() + participant Enrichment as enrich() + rank_contacts() + participant CRM as CRMClient + participant DB + + CLI->>DB: query status == pending + loop each organization + CLI->>Discovery: discover_domain(name, project, address) + Discovery-->>CLI: domain, confidence, method + alt confidence == high + CLI->>DB: domain_resolved + else + CLI->>DB: needs_review + end + end + CLI->>DB: write SyncLog(domain_discovery) + + CLI->>DB: query status == domain_resolved + loop each organization + CLI->>Enrichment: enrich(domain) + Enrichment-->>CLI: raw contacts + CLI->>Enrichment: rank_contacts(raw, department_priority) + alt raw_count > 0 + CLI->>DB: store top-5 contacts, status: enriched + else + CLI->>DB: needs_review (thin result) + end + end + CLI->>DB: write SyncLog(contact_enrichment) + + CLI->>DB: query status == enriched + loop each organization + CLI->>CRM: find_organization_by_website(domain) + alt not found + CLI->>CRM: create_organization(name, domain) + end + CLI->>CRM: find_contact_by_email(primary.email) + alt not found + CLI->>CRM: create_contact(name, email, org_id) + end + CLI->>CRM: find_lead(org_id, title) + alt not found + CLI->>CRM: create_lead(org_id, contact_id, title, note) + end + CLI->>DB: store Lead, status: pushed + end + CLI->>DB: write SyncLog(crm_push) +``` + +## Components + +| Module | Responsibility | +|---|---| +| `discovery/query.py` | Builds two deliberately different search-query shapes per company; numbered/shell-entity names get a project/address-first query | +| `discovery/confidence.py` | The confidence gate — resolves votes + head-confirmation into high/medium/low/unresolved | +| `discovery/exclusion.py` | Living domain/keyword blocklist, applied before a result becomes a vote | +| `discovery/base.py` | `SearchClient` interface + `discover_domain()` orchestrator | +| `discovery/sandbox_client.py` | Default demo search backend — bundled fixture data, zero network | +| `discovery/real_search_client_template.py`, `http_head_check.py` | Template for a real search vendor + a real, working HTTP HEAD confirmation check | +| `enrichment/base.py` | `ContactEnrichmentClient` interface + `RawContact`/`EnrichmentResult` types | +| `enrichment/ranking.py` | Re-sorts contacts by department priority instead of the vendor's default seniority-first order | +| `enrichment/sandbox_client.py` | Default demo enrichment backend, including one domain seeded with zero contacts (a "thin" result) | +| `enrichment/real_client_template.py` | Template for a real contact-enrichment vendor | +| `crm/base.py` | `CRMClient` interface — org dedup by website, contact by email, lead by (org, title) | +| `crm/overwrite_gate.py` | Shared "diff live value, auto-approve only safe cases" gate | +| `crm/sandbox_client.py` | Default demo CRM backend, pre-seeded with one existing org (proves dedup against pre-existing data, not just within-run) | +| `crm/pipedrive_client.py` | A real, working Pipedrive REST v1 client (not a template) | +| `pipeline/discover.py` | Orchestrates domain discovery for `pending` organizations | +| `pipeline/enrich.py` | Orchestrates contact enrichment for `domain_resolved` organizations | +| `pipeline/push_crm.py` | Orchestrates the 3-phase CRM push for `enriched` organizations | +| `storage/models.py` | `Organization`, `Contact`, `Lead`, `SyncLog` SQLAlchemy models | +| `storage/db.py` | Engine/session setup (SQLite) | +| `config.py` | `.env` + `rules.yaml` loading | +| `cli.py` | Typer CLI wiring everything above together | + +Vendor clients (`*/sandbox_client.py`, `*/pipedrive_client.py`, +`*/real_*_template.py`) contain no DB or business logic — they're thin +backends implementing a shared interface. Orchestration (which rows to +touch, what the confidence/ranking/dedup rules mean for status) lives in +`pipeline/*.py`. This split is what makes the confidence gate, ranking +logic, and dedup rules independently unit-testable without any network +access, and what makes swapping a sandbox backend for a real vendor a +one-line change in `pipeline/*.py`'s `_default_client()`. + +```mermaid +flowchart TB + CLI["cli.py"] + + subgraph Pipeline["pipeline/ (orchestration)"] + discover["discover.py"] + enrich["enrich.py"] + push_crm["push_crm.py"] + end + + subgraph Discovery["discovery/"] + d_base["base.py"] + d_backends["sandbox_client.py,
real_search_client_template.py"] + end + + subgraph Enrichment["enrichment/"] + e_base["base.py"] + e_backends["sandbox_client.py,
real_client_template.py"] + end + + subgraph CRM["crm/"] + c_base["base.py"] + c_gate["overwrite_gate.py"] + c_backends["sandbox_client.py,
pipedrive_client.py"] + end + + subgraph Storage["storage/"] + models["models.py"] + db["db.py"] + end + + CLI --> Pipeline + discover --> Discovery + enrich --> Enrichment + push_crm --> CRM + Pipeline --> Storage +``` + +## Design decisions + +**Confidence is a hard architectural gate, not a score shown to a human.** +Only "high" confidence (two independent search signals agree, and the +domain is live-confirmed) proceeds automatically. Everything else — a +single confirmed signal, agreement without confirmation, total +disagreement — is parked (`needs_review`) rather than pushed downstream +with a caveat. This was the single highest-leverage design decision in the +reference system this project is modeled on: roughly half of all +formally-reviewed flagged bad leads traced to exactly the failure mode this +gate exists to prevent (see the private design notes this rebuild was +informed by — not part of this repo). + +**Field-level authority, not blanket automation.** Every pipeline stage +only ever reads organizations at the specific status it's responsible for +(`pending` for discovery, `domain_resolved` for enrichment, `enriched` for +CRM push). An organization already parked or already moved past a stage is +never silently re-processed on a later run — a human (or the next stage) +has to move it forward explicitly. This is enforced by construction (a +`WHERE status == X` query), not by convention. + +**Plan-only by default, `--apply` to write for real — and dry-run means +zero external mutation, not just zero DB writes.** `discover`, `enrich`, +and `push-crm` all default to computing and printing what *would* happen. +`push_crm.py` goes further than a simple DB-write skip: in plan-only mode, +no `create_*` method is ever called on the CRM client at all — not even +against the sandbox's in-memory fake — because a naive "skip only the DB +write" dry-run would still silently mutate a stateful client if the same +instance were reused across calls. + +**Department-priority re-ranking overrides the vendor's default sort.** +Contact-enrichment vendors typically sort by seniority (C-suite first). +The reference system's own real outcomes showed the actual decision-makers +for this industry sit in development/construction/property-management +functions — the ranking module exists specifically to correct that +mismatch, not to second-guess the vendor's data itself. + +**A lead is modeled per project, not per company.** A single developer can +have multiple concurrently active projects, each needing independent +outreach tracking. Since an `Organization` row here already represents one +raw project/prospect record, one `Lead` per `Organization` row already *is* +one lead per project — CRM organization-level dedup (by website) happens +at push time via find-or-create, without needing a separate +company/developer table. + +**A shared overwrite-protection gate, not per-call-site judgment.** +`overwrite_gate.decide_write()` is a small pure function every CRM-write +path is meant to funnel through: empty-in-CRM or identical values +auto-write, a real conflict is reported and skipped rather than silently +overwritten. Centralizing this once prevents the class of bug where one +write path's conflict-handling silently drifts from another's. + +**Sandbox-first, real vendor as a documented follow-up.** Every external +integration point (search, contact enrichment, CRM) ships with a working, +network-free sandbox backend as the default, plus either a documented +template (`real_search_client_template.py`, `real_client_template.py`) or +a real-but-untested client (`pipedrive_client.py`) for the real thing. This +mirrors the honest-limitation pattern used throughout the sibling +[tender-tracking-system](https://github.com/codeapplied/tender-tracking-system) +repo: verified via mocked-HTTP-shape tests where no live account was +available, never claimed as more than that. + +**Naive UTC everywhere, not timezone-aware datetimes.** SQLite doesn't +preserve timezone info across a save/reload — a timezone-aware value read +back from the DB would never equal a freshly-computed one, even when the +underlying instant is identical. `storage/models.py`'s `utcnow()` stays +naive-UTC everywhere, the same fix already validated in the sibling repo +after it hit this as a real bug. + +## Adding a new vendor + +See [ADDING_A_VENDOR.md](ADDING_A_VENDOR.md) — covers search, contact +enrichment, and CRM extension points. diff --git a/docs/assets/cli-discover.svg b/docs/assets/cli-discover.svg new file mode 100644 index 0000000..f51ede2 --- /dev/null +++ b/docs/assets/cli-discover.svg @@ -0,0 +1,106 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + dataenrich discover --apply + + + + + + + + + + APPLIED +                                Domain Discovery Results                                  ++---------------------------------------------------------------------------------------+ +|Company                           |Domain                             |Confidence| +|------------------------------------+-------------------------------------+------------| +|Sample Riverside Developments Inc.|sampleriversidedevelopments.example|high      | +|1234567 Ontario Inc.              |maplegroveresidences.example       |high      | +|Sample Harbor Properties Ltd.     |sampleharborproperties.example     |low       | +|Sample Quiet Construction Co.     |samplequietconstruction.example    |medium    | +|Sample Ghost Holdings LLC         |-                                  |unresolved| ++---------------------------------------------------------------------------------------+ +Processed: 5  High-confidence: 2  Parked for review: 3  Errors: 0 + + + + diff --git a/docs/assets/cli-enrich.svg b/docs/assets/cli-enrich.svg new file mode 100644 index 0000000..979995c --- /dev/null +++ b/docs/assets/cli-enrich.svg @@ -0,0 +1,94 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + dataenrich enrich --apply + + + + + + + + + + APPLIED +                      Contact Enrichment Results                       ++--------------------------------------------------------------------+ +|Company                           |Raw contacts found|Selected| +|------------------------------------+--------------------+----------| +|Sample Riverside Developments Inc.|3                 |3       | +|1234567 Ontario Inc.              |2                 |2       | ++--------------------------------------------------------------------+ +Processed: 2  Enriched: 2  Thin (zero contacts found): 0  Errors: 0 + + + + diff --git a/docs/assets/cli-push-crm.svg b/docs/assets/cli-push-crm.svg new file mode 100644 index 0000000..324af8f --- /dev/null +++ b/docs/assets/cli-push-crm.svg @@ -0,0 +1,94 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + dataenrich push-crm + + + + + + + + + + PLAN ONLY (dry-run) — pass --apply to write for real +                          CRM Push Results                            ++-------------------------------------------------------------------+ +|Company                           |CRM Org ID    |CRM Lead ID| +|------------------------------------+----------------+-------------| +|Sample Riverside Developments Inc.|crm-org-seed-1|(new)      | +|1234567 Ontario Inc.              |(new)         |(new)      | ++-------------------------------------------------------------------+ +Processed: 2  Pushed: 2  Errors: 0 + + + + diff --git a/docs/assets/cli-status.svg b/docs/assets/cli-status.svg new file mode 100644 index 0000000..9c4cbe1 --- /dev/null +++ b/docs/assets/cli-status.svg @@ -0,0 +1,84 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + dataenrich status + + + + + + + + + +                                        Recent Pipeline Runs                                         ++-------------------------------------------------------------------------------------------------+ +|Stage             |Started                   |Status |Processed|New|Updated|Flagged| +|--------------------+----------------------------+---------+-----------+-----+---------+---------| +|contact_enrichment|2026-09-01 08:28:47.334418|success|2        |2  |0      |0      | +|domain_discovery  |2026-09-01 08:28:47.313140|success|5        |2  |0      |3      | ++-------------------------------------------------------------------------------------------------+ + + + + diff --git a/scripts/generate_readme_assets.py b/scripts/generate_readme_assets.py new file mode 100644 index 0000000..75b291c --- /dev/null +++ b/scripts/generate_readme_assets.py @@ -0,0 +1,77 @@ +"""Regenerate the CLI-output SVGs embedded in the README. + +Runs the real CLI code paths (not a mockup) against a throwaway temp DB +populated by `seed-demo`, capturing Rich's actual rendered output (colors, +tables, box-drawing) to SVG. Re-run this after any change to the CLI's +table layouts so the README images stay accurate. + +Usage: python scripts/generate_readme_assets.py +""" + +import os +import shutil +import tempfile +from pathlib import Path + +from rich.console import Console + +REPO_ROOT = Path(__file__).parent.parent +ASSETS_DIR = REPO_ROOT / "docs" / "assets" + + +def main() -> None: + tmp_dir = Path(tempfile.mkdtemp(prefix="dataenrich-assets-")) + original_cwd = Path.cwd() + + try: + (tmp_dir / "config").mkdir() + shutil.copy(REPO_ROOT / "config" / "rules.example.yaml", tmp_dir / "config" / "rules.yaml") + + os.chdir(tmp_dir) + + import dataenrich.cli as cli_module + from dataenrich.config import settings as demo_settings + + demo_settings.db_path = str(tmp_dir / "demo.db") + demo_settings.pipedrive_api_token = None + demo_settings.pipedrive_domain = None + + ASSETS_DIR.mkdir(parents=True, exist_ok=True) + + cli_module.init() + cli_module.seed_demo() + + # --- dataenrich discover --apply --------------------------------- + console = Console(record=True, width=100) + cli_module.console = console + cli_module.discover(apply=True) + console.save_svg(str(ASSETS_DIR / "cli-discover.svg"), title="dataenrich discover --apply") + + # --- dataenrich enrich --apply ------------------------------------ + console = Console(record=True, width=100) + cli_module.console = console + cli_module.enrich(apply=True) + console.save_svg(str(ASSETS_DIR / "cli-enrich.svg"), title="dataenrich enrich --apply") + + # --- dataenrich push-crm (plan-only, shows dedup against the ------ + # --- sandbox's pre-seeded org) ------------------------------------- + console = Console(record=True, width=100) + cli_module.console = console + cli_module.push_crm(apply=False) + console.save_svg(str(ASSETS_DIR / "cli-push-crm.svg"), title="dataenrich push-crm") + + # --- dataenrich status --------------------------------------------- + console = Console(record=True, width=100) + cli_module.console = console + cli_module.status() + console.save_svg(str(ASSETS_DIR / "cli-status.svg"), title="dataenrich status") + + print(f"Wrote SVGs to {ASSETS_DIR}") + + finally: + os.chdir(original_cwd) + shutil.rmtree(tmp_dir, ignore_errors=True) + + +if __name__ == "__main__": + main()