From 9d83dc7a99da3deec896ef58dc29280cf7b7dd12 Mon Sep 17 00:00:00 2001 From: Oleg Popov Date: Wed, 29 Apr 2026 23:55:05 +0200 Subject: [PATCH 1/8] docs: enforce sql isolation and transport registry in AGENTS.md --- AGENTS.MD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.MD b/AGENTS.MD index 92c91b5..a6a2ae1 100644 --- a/AGENTS.MD +++ b/AGENTS.MD @@ -13,7 +13,7 @@ Parqcast is strictly layered. Agents must NEVER introduce imports that violate t - **`packages/parqcast-collectors/`**: Executes fast raw SQL queries using `psycopg2` to build arrow arrays. **Constraint:** Must NEVER import `odoo` ORM models. - **`packages/parqcast-ingesters/`**: Pulls data via Odoo ORM to push planning decisions back. This is the ONLY python package (outside the addon itself) permitted to use dynamic Odoo typing. - **`packages/parqcast-transport-http/`**: Handles HTTP uploads of generated chunks. -- **`packages/parqcast/`**: The Odoo addon. It bridges the environment with the workspace packages. **Constraint:** Must pass Odoo Community Association (OCA) linting via `bash scripts/oca_lint.sh`. +- **`packages/parqcast/`**: The Odoo addon. It bridges the environment with the workspace packages. **Constraint:** Must pass Odoo Community Association (OCA) linting via `bash scripts/oca_lint.sh`. Must **NEVER** contain inline raw SQL queries (`cr.execute`); all tracking SQL is delegated to `packages/parqcast-core`. Transports must be decoupled and registered via `parqcast/utils/transport_registry.py`. ## 3. Odoo Versioning (PEP 695 Generics) Parqcast certifies per Odoo major. From abfc6c6bba347ab0bbb8343b184050e66326a0d0 Mon Sep 17 00:00:00 2001 From: Oleg Popov Date: Fri, 1 May 2026 22:49:47 +0200 Subject: [PATCH 2/8] feat: export company country_id for mapping --- .../parqcast-collectors/src/parqcast/collectors/company.py | 3 ++- packages/parqcast-core/src/parqcast/schemas/outbound.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/parqcast-collectors/src/parqcast/collectors/company.py b/packages/parqcast-collectors/src/parqcast/collectors/company.py index 0140096..218d74b 100644 --- a/packages/parqcast-collectors/src/parqcast/collectors/company.py +++ b/packages/parqcast-collectors/src/parqcast/collectors/company.py @@ -14,7 +14,8 @@ def get_sql(self): rc.id, rc.name, rc.currency_id, rc.parent_id, COALESCE(rc.security_lead, 0), - rc.active + rc.active, + rc.country_id FROM res_company rc """, None, diff --git a/packages/parqcast-core/src/parqcast/schemas/outbound.py b/packages/parqcast-core/src/parqcast/schemas/outbound.py index 40dd18f..cb1f75c 100644 --- a/packages/parqcast-core/src/parqcast/schemas/outbound.py +++ b/packages/parqcast-core/src/parqcast/schemas/outbound.py @@ -436,6 +436,7 @@ ("_odoo_parent_id", OdooId), ("security_lead", OdooFloat), ("active", OdooBoolean), + ("_odoo_country_id", OdooId), ] ) From 044f25522ecf7e4675e2526403a05a30dc2106f9 Mon Sep 17 00:00:00 2001 From: Oleg Popov Date: Sat, 2 May 2026 09:47:19 +0200 Subject: [PATCH 3/8] docs: expand data dictionary schemas and clarify i18n --- AGENTS.MD | 4 +++ docs/DATA_DICTIONARY.md | 79 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+) create mode 100644 docs/DATA_DICTIONARY.md diff --git a/AGENTS.MD b/AGENTS.MD index a6a2ae1..1fc043b 100644 --- a/AGENTS.MD +++ b/AGENTS.MD @@ -24,3 +24,7 @@ Parqcast certifies per Odoo major. ## 4. Testing - Use `uv run pytest tests/ -x -q` to verify logic. - Do not run `tests/test_live_db.py` without the `PARQCAST_TEST_DB` environment variable set. + +## 5. Data Architecture & Schemas +When answering questions about what data is available, writing SQL extractors, or modifying PyArrow schemas, you MUST first read `docs/DATA_DICTIONARY.md`. +* All outbound schemas are defined and strictly typed in `packages/parqcast-core/src/parqcast/schemas/outbound.py`. diff --git a/docs/DATA_DICTIONARY.md b/docs/DATA_DICTIONARY.md new file mode 100644 index 0000000..5d86c3c --- /dev/null +++ b/docs/DATA_DICTIONARY.md @@ -0,0 +1,79 @@ +# Parqcast Data Dictionary + +## Overview +This document serves as the single source of truth for the data architecture within the `parqcast` repository. Parqcast extracts operational data from Odoo, serializes it into Apache Parquet format via PyArrow, and transmits it for external forecasting, predictive replenishment, and supply chain planning. + +**Critical Note for AI Agents & Developers:** +Data is **NOT** extracted via the standard Odoo ORM (`env['...'].search_read()`). Instead, it is extracted using extremely fast, raw PostgreSQL queries defined in `packages/parqcast-collectors/`. The extracted data strictly conforms to the PyArrow schemas defined in `packages/parqcast-core/src/parqcast/schemas/outbound.py`. + +When answering questions about available data or modifying the pipeline, you MUST refer to `outbound.py` as the ultimate authority on what data columns are currently supported. + +--- + +## Available Data Entities + +Below is a categorized list of the primary data entities currently extracted by Parqcast. + +### 1. Manufacturing & Production +- **`MRP_PRODUCTION`** (`mrp_production`): Core manufacturing orders. Extracts state, priorities, and crucial scheduling dates to feed production planning algorithms. + - **Schema Fields:** `_odoo_mo_id`, `name`, `_odoo_product_id`, `product_name`, `_odoo_bom_id`, `product_qty`, `qty_producing`, `_odoo_uom_id`, `state`, `priority`, `reservation_state`, `date_start`, `date_finished`, `date_deadline`, `_odoo_location_src_id`, `_odoo_location_dest_id`, `_odoo_picking_type_id`, `origin`, `_odoo_company_id`. +- **`MRP_WORKORDER`** (`mrp_workorder`): Specific routing operations tied to an MO. Extracts durations and costs to model workcenter capacity. + - **Schema Fields:** `_odoo_wo_id`, `_odoo_mo_id`, `display_name`, `_odoo_workcenter_id`, `workcenter_name`, `_odoo_operation_id`, `state`, `date_start`, `date_finished`, `duration_expected`, `duration_unit`, `is_user_working`, `employee_costs_hour`. +- **`BOM`** (`mrp_bom`): Bills of Materials. Essential for recursive material planning. + - **Schema Fields:** `_odoo_bom_id`, `_odoo_product_tmpl_id`, `_odoo_product_id`, `product_name`, `bom_type`, `product_qty`, `_odoo_uom_id`, `uom_name`, `product_qty_multiple`, `produce_delay`, `days_to_prepare_mo`, `sequence`, `code`, `_odoo_company_id`. +- **`BOM_LINES`** (`mrp_bom_line`): Component-level BOM requirements. + - **Schema Fields:** `_odoo_bom_line_id`, `_odoo_bom_id`, `_odoo_product_id`, `product_name`, `product_qty`, `_odoo_uom_id`, `uom_name`, `_odoo_operation_id`, `operation_name`, `bom_product_template_attribute_value_ids`. +- **`BOM_OPERATIONS`** (`mrp_routing_workcenter`): Standard cycle times. + - **Schema Fields:** `_odoo_operation_id`, `_odoo_bom_id`, `name`, `sequence`, `_odoo_workcenter_id`, `workcenter_name`, `time_cycle`, `workcenter_quantity`, `search_mode`, `_odoo_skill_id`, `skill_name`, `post_operation_time`. +- **`WORKCENTER`** (`mrp_workcenter`): Captures default capacity and costs to constrain production schedules. + - **Schema Fields:** `_odoo_workcenter_id`, `name`, `_odoo_resource_id`, `default_capacity`, `time_efficiency`, `is_tool`, `is_constrained`, `_odoo_owner_id`, `owner_name`, `_odoo_calendar_id`, `calendar_name`, `post_operation_time`, `employee_costs_hour`, `_odoo_company_id`. + +### 2. Inventory & Stock +- **`STOCK_QUANT`** (`stock_quant`): Live, real-time snapshot of inventory. Distinguishes between physical and reserved quantity. + - **Schema Fields:** `_odoo_quant_id`, `_odoo_product_id`, `product_name`, `product_code`, `_odoo_location_id`, `location_name`, `location_usage`, `_odoo_warehouse_id`, `warehouse_code`, `quantity`, `reserved_quantity`, `_odoo_lot_id`, `lot_name`, `expiry_date`, `_odoo_package_id`, `package_name`, `_odoo_owner_id`, `owner_name`, `in_date`, `_odoo_company_id`. +- **`STOCK_MOVE`** (`stock_move`): Planned or pending inventory movements. + - **Schema Fields:** `_odoo_move_id`, `_odoo_product_id`, `product_name`, `product_uom_qty`, `quantity`, `_odoo_uom_id`, `state`, `priority`, `_odoo_picking_id`, `picking_name`, `_odoo_location_id`, `_odoo_location_dest_id`, `_odoo_warehouse_id`, `_odoo_production_id`, `_odoo_sale_line_id`, `_odoo_purchase_line_id`, `origin`, `date`, `date_deadline`, `delay_alert_date`, `procure_method`, `move_orig_ids`, `move_dest_ids`, `is_subcontract`. +- **`STOCK_MOVE_LINE`** (`stock_move_line`): Granular execution of moves. Tracks the exact lot and expiration. + - **Schema Fields:** `_odoo_move_line_id`, `_odoo_move_id`, `_odoo_picking_id`, `_odoo_product_id`, `_odoo_product_uom_id`, `quantity`, `_odoo_lot_id`, `lot_name`, `_odoo_package_id`, `_odoo_result_package_id`, `_odoo_owner_id`, `_odoo_location_id`, `_odoo_location_dest_id`, `state`, `date`, `expiration_date`, `_odoo_company_id`. +- **`STOCK_LOCATION`** (`stock_location`): The spatial hierarchy, noting location usage and removal strategy. + - **Schema Fields:** `_odoo_location_id`, `name`, `complete_name`, `usage`, `_odoo_parent_id`, `_odoo_warehouse_id`, `warehouse_code`, `removal_strategy`, `_odoo_storage_category_id`, `storage_category_name`, `active`, `_odoo_company_id`. +- **`STOCK_PICKING`** (`stock_picking`): The overarching transfer document (receipts/deliveries). + - **Schema Fields:** `_odoo_picking_id`, `name`, `state`, `picking_type_code`, `_odoo_location_id`, `_odoo_location_dest_id`, `scheduled_date`, `date_deadline`, `date_done`, `origin`, `_odoo_partner_id`, `_odoo_backorder_id`, `_odoo_sale_id`, `priority`, `move_type`, `_odoo_company_id`. +- **`ORDERPOINT`** (`stock_warehouse_orderpoint`): Configured reordering rules. + - **Schema Fields:** `_odoo_orderpoint_id`, `_odoo_product_id`, `product_name`, `_odoo_warehouse_id`, `warehouse_code`, `_odoo_location_id`, `location_name`, `product_min_qty`, `product_max_qty`, `trigger`, `active`, `snoozed_until`, `_odoo_route_id`, `route_name`, `deadline_date`, `_odoo_replenishment_uom_id`, `qty_to_order_computed`, `qty_to_order_manual`, `_odoo_supplier_id`, `_odoo_company_id`. + +### 3. Sales & Purchasing +- **`SALE_ORDER`** (`sale_order`): Customer orders. + - **Schema Fields:** `_odoo_so_id`, `name`, `state`, `_odoo_partner_id`, `date_order`, `commitment_date`, `_odoo_warehouse_id`, `warehouse_code`, `picking_policy`, `amount_untaxed`, `amount_total`, `_odoo_currency_id`, `invoice_status`, `delivery_status`, `_odoo_company_id`. +- **`SALE_ORDER_LINE`** (`sale_order_line`): Specific order lines tracking requested vs delivered quantities. + - **Schema Fields:** `_odoo_sol_id`, `_odoo_so_id`, `so_name`, `_odoo_product_id`, `product_name`, `product_uom_qty`, `qty_delivered`, `qty_invoiced`, `_odoo_uom_id`, `uom_name`, `price_unit`, `discount`, `price_subtotal`, `customer_lead`, `_odoo_partner_id`, `partner_name`, `is_company`, `commitment_date`, `date_order`, `so_state`, `sol_state`, `_odoo_warehouse_id`, `warehouse_code`, `picking_policy`. +- **`PURCHASE_ORDER`** (`purchase_order`): Vendor orders showing status and planned dates. + - **Schema Fields:** `_odoo_po_id`, `name`, `state`, `_odoo_partner_id`, `date_order`, `date_approve`, `date_planned`, `receipt_status`, `invoice_status`, `amount_untaxed`, `amount_total`, `_odoo_currency_id`, `_odoo_picking_type_id`, `_odoo_warehouse_id`, `origin`, `priority`, `_odoo_company_id`. +- **`PURCHASE_ORDER_LINE`** (`purchase_order_line`): Purchased quantities and unit prices. + - **Schema Fields:** `_odoo_pol_id`, `_odoo_po_id`, `po_name`, `po_state`, `receipt_status`, `_odoo_product_id`, `product_name`, `product_qty`, `qty_received`, `_odoo_uom_id`, `uom_name`, `price_unit`, `_odoo_currency_id`, `date_order`, `date_planned`, `_odoo_partner_id`, `partner_name`, `_odoo_warehouse_id`, `origin`, `_odoo_sale_line_id`, `_odoo_orderpoint_id`, `_odoo_company_id`. +- **`POS_ORDER` & `POS_ORDER_LINE`** (`pos_order`): Point of Sale transactions for retail forecasting. + - **POS_ORDER Schema Fields:** `_odoo_pos_order_id`, `name`, `state`, `date_order`, `_odoo_partner_id`, `_odoo_session_id`, `_odoo_config_id`, `amount_total`, `amount_tax`, `amount_paid`, `is_refund`, `_odoo_company_id`. + - **POS_ORDER_LINE Schema Fields:** `_odoo_pos_line_id`, `_odoo_pos_order_id`, `_odoo_product_id`, `full_product_name`, `qty`, `price_unit`, `price_subtotal`, `price_subtotal_incl`, `discount`, `_odoo_company_id`. + +### 4. Master Data +- **`PRODUCT`** (`product_product` / `product_template`): Core item master. Extracts critical rules to determine how items are procured. + - **Schema Fields:** `_odoo_product_id`, `_odoo_template_id`, `name`, `name_lang2`, `name_lang3`, `default_code`, `barcode`, `display_name`, `_odoo_uom_id`, `uom_name`, `uom_factor`, `list_price`, `standard_price`, `weight`, `volume`, `_odoo_categ_id`, `category_name`, `purchase_ok`, `sale_ok`, `is_storable`, `active`, `product_type`, `description_sale`, `invoice_policy`, `purchase_method`, `use_expiration_date`, `service_tracking`, `sale_delay`, `route_ids`, `route_names`, `expiration_time`, `tracking`, `_odoo_company_id`. +- **`PRODUCT_SUPPLIERINFO`** (`product_supplierinfo`): Vendor pricing and constraints. Critical for replenishment planners. + - **Schema Fields:** `_odoo_supplierinfo_id`, `_odoo_template_id`, `_odoo_product_id`, `_odoo_partner_id`, `partner_name`, `delay`, `min_qty`, `price`, `_odoo_currency_id`, `currency_name`, `sequence`, `date_start`, `date_end`, `batching_window`, `is_subcontractor`. +- **`PARTNER`** (`res_partner`): Suppliers and customers. + - **Schema Fields:** `_odoo_partner_id`, `name`, `ref`, `is_company`, `_odoo_parent_id`, `parent_name`, `_odoo_country_id`, `customer_rank`, `supplier_rank`, `active`, `_odoo_company_id`. +- **`COMPANY` & `CURRENCY`** (`res_company`, `res_currency`): Context variables like `security_lead` and exchange rates. + - **COMPANY Schema Fields:** `_odoo_company_id`, `name`, `_odoo_currency_id`, `_odoo_parent_id`, `security_lead`, `active`, `_odoo_country_id`. + - **CURRENCY Schema Fields:** `_odoo_currency_id`, `name`, `symbol`, `decimal_places`, `rounding`, `active`, `_odoo_rate_id`, `rate`, `rate_date`, `_odoo_rate_company_id`. + +--- + +## Technical Nuances + +1. **Internationalization (i18n):** Queries are dynamically parameterized to extract translatable JSONB fields in the company's language. The `parqcast-collectors` system injects the primary language via `self.caps.lang(1)` into the raw SQL (e.g., `name->>'{lang1}'`). `en_US` is only used as a final fallback if the environment lacks language capabilities entirely. This guarantees that LLMs and external systems receive the correctly localized entity names natively. +2. **Timestamps:** All dates and datetimes are exported as UTC to ensure cross-system consistency. +3. **Odoo IDs:** Primary and foreign keys are explicitly typed and prefixed with `_odoo_` in the Parquet schema (e.g., `_odoo_product_id`) to differentiate them from downstream system IDs. +4. **Boolean & Numeric Types:** We strictly map Odoo's internal types to optimized PyArrow equivalents (e.g., `OdooMonetary`, `OdooQuantity`, `OdooId`) to compress file sizes and maximize memory efficiency. + +--- +*For the exact, up-to-date column definitions, always inspect `outbound.py`.* From 1f355d1081e9f75d22db76d77fad2d1e3e38559f Mon Sep 17 00:00:00 2001 From: Oleg Popov Date: Sat, 2 May 2026 09:52:19 +0200 Subject: [PATCH 4/8] docs: add inbound decisions schema for ingesters --- docs/DATA_DICTIONARY.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/DATA_DICTIONARY.md b/docs/DATA_DICTIONARY.md index 5d86c3c..f3442bc 100644 --- a/docs/DATA_DICTIONARY.md +++ b/docs/DATA_DICTIONARY.md @@ -66,6 +66,12 @@ Below is a categorized list of the primary data entities currently extracted by - **COMPANY Schema Fields:** `_odoo_company_id`, `name`, `_odoo_currency_id`, `_odoo_parent_id`, `security_lead`, `active`, `_odoo_country_id`. - **CURRENCY Schema Fields:** `_odoo_currency_id`, `name`, `symbol`, `decimal_places`, `rounding`, `active`, `_odoo_rate_id`, `rate`, `rate_date`, `_odoo_rate_company_id`. +### 5. Inbound Actions & Decisions (The Ingesters) +While the entities above describe the data being extracted *out* of Odoo via `parqcast-collectors` (using raw SQL), the external planner also pushes actionable decisions *back into* Odoo. This data is handled by `parqcast-ingesters`, which is the ONLY package explicitly permitted to use the standard Odoo ORM (e.g., `env['purchase.order'].create(...)`) to safely trigger Odoo's internal workflows. + +- **`DECISIONS`** (`inbound.py`): The prescriptive actions prescribed by the external AI/planner (e.g., creating a Purchase Order, updating an Orderpoint, scheduling Manufacturing). + - **Schema Fields:** `decision_id`, `decision_type`, `status`, `item_name`, `_odoo_product_id`, `_odoo_uom_id`, `location_name`, `_odoo_location_id`, `quantity`, `start_date`, `end_date`, `supplier_name`, `_odoo_supplier_id`, `_odoo_bom_id`, `origin_location`, `destination_location`, `parent_reference`, `workcenter_name`, `_odoo_workcenter_id`, `batch`, `remark`, `min_quantity`, `max_quantity`. + --- ## Technical Nuances From 3aac84d8b6594c526d2f09e9d2f1930f8ef8b33c Mon Sep 17 00:00:00 2001 From: Oleg Popov Date: Sat, 2 May 2026 10:03:14 +0200 Subject: [PATCH 5/8] feat(ingesters): add LEAD_TIME prescriptive decision actor --- AGENTS.MD | 2 +- README.md | 2 +- docs/DATA_DICTIONARY.md | 2 +- .../src/parqcast/schemas/inbound.py | 2 + .../src/parqcast/schemas/odoo_types.py | 4 ++ .../src/parqcast/ingesters/v18/__init__.py | 3 ++ .../ingesters/v18/supplierinfo_actor.py | 52 +++++++++++++++++++ .../src/parqcast/ingesters/v19/__init__.py | 3 ++ .../ingesters/v19/supplierinfo_actor.py | 52 +++++++++++++++++++ 9 files changed, 119 insertions(+), 3 deletions(-) create mode 100644 packages/parqcast-ingesters/src/parqcast/ingesters/v18/supplierinfo_actor.py create mode 100644 packages/parqcast-ingesters/src/parqcast/ingesters/v19/supplierinfo_actor.py diff --git a/AGENTS.MD b/AGENTS.MD index 1fc043b..8d82ba6 100644 --- a/AGENTS.MD +++ b/AGENTS.MD @@ -11,7 +11,7 @@ This document defines the constraints, architecture boundaries, and tooling inst Parqcast is strictly layered. Agents must NEVER introduce imports that violate these boundaries: - **`packages/parqcast-core/`**: Defines data schemas, pyarrow constructs, and protocol interfaces. **Constraint:** Must NEVER import from `odoo`, `psycopg2`, or any other workspace package. - **`packages/parqcast-collectors/`**: Executes fast raw SQL queries using `psycopg2` to build arrow arrays. **Constraint:** Must NEVER import `odoo` ORM models. -- **`packages/parqcast-ingesters/`**: Pulls data via Odoo ORM to push planning decisions back. This is the ONLY python package (outside the addon itself) permitted to use dynamic Odoo typing. +- **`packages/parqcast-ingesters/`**: Receives decisions matching `inbound.py` schemas and pushes them into Odoo via the ORM (e.g., updating vendor lead times or creating POs). This is the ONLY python package (outside the addon itself) permitted to use dynamic Odoo typing. - **`packages/parqcast-transport-http/`**: Handles HTTP uploads of generated chunks. - **`packages/parqcast/`**: The Odoo addon. It bridges the environment with the workspace packages. **Constraint:** Must pass Odoo Community Association (OCA) linting via `bash scripts/oca_lint.sh`. Must **NEVER** contain inline raw SQL queries (`cr.execute`); all tracking SQL is delegated to `packages/parqcast-core`. Transports must be decoupled and registered via `parqcast/utils/transport_registry.py`. diff --git a/README.md b/README.md index 0102ff1..4622beb 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ This repo is a `uv` workspace with four Python packages plus the Odoo addon: |------|---------| | `packages/parqcast-core/` | Schemas, transport, tracking, capabilities, manifest, protocols (pyarrow only) | | `packages/parqcast-collectors/` | SQL-based data extractors + time-budgeted orchestrator | -| `packages/parqcast-ingesters/` | ORM-based writers (push planning decisions back into Odoo) | +| `packages/parqcast-ingesters/` | ORM-based writers (push prescriptive ML decisions like POs and dynamic lead times back into Odoo) | | `packages/parqcast-transport-http/` | HTTP upload transport | | `packages/parqcast/` | The Odoo 19 addon (settings UI, cron job, env adapter) — see [its README](packages/parqcast/README.rst) | diff --git a/docs/DATA_DICTIONARY.md b/docs/DATA_DICTIONARY.md index f3442bc..7d55e7d 100644 --- a/docs/DATA_DICTIONARY.md +++ b/docs/DATA_DICTIONARY.md @@ -70,7 +70,7 @@ Below is a categorized list of the primary data entities currently extracted by While the entities above describe the data being extracted *out* of Odoo via `parqcast-collectors` (using raw SQL), the external planner also pushes actionable decisions *back into* Odoo. This data is handled by `parqcast-ingesters`, which is the ONLY package explicitly permitted to use the standard Odoo ORM (e.g., `env['purchase.order'].create(...)`) to safely trigger Odoo's internal workflows. - **`DECISIONS`** (`inbound.py`): The prescriptive actions prescribed by the external AI/planner (e.g., creating a Purchase Order, updating an Orderpoint, scheduling Manufacturing). - - **Schema Fields:** `decision_id`, `decision_type`, `status`, `item_name`, `_odoo_product_id`, `_odoo_uom_id`, `location_name`, `_odoo_location_id`, `quantity`, `start_date`, `end_date`, `supplier_name`, `_odoo_supplier_id`, `_odoo_bom_id`, `origin_location`, `destination_location`, `parent_reference`, `workcenter_name`, `_odoo_workcenter_id`, `batch`, `remark`, `min_quantity`, `max_quantity`. + - **Schema Fields:** `decision_id`, `decision_type`, `status`, `item_name`, `_odoo_product_id`, `_odoo_uom_id`, `location_name`, `_odoo_location_id`, `quantity`, `start_date`, `end_date`, `supplier_name`, `_odoo_supplier_id`, `_odoo_bom_id`, `origin_location`, `destination_location`, `parent_reference`, `workcenter_name`, `_odoo_workcenter_id`, `batch`, `remark`, `min_quantity`, `max_quantity`, `delay`. --- diff --git a/packages/parqcast-core/src/parqcast/schemas/inbound.py b/packages/parqcast-core/src/parqcast/schemas/inbound.py index 4f20a0b..a57012f 100644 --- a/packages/parqcast-core/src/parqcast/schemas/inbound.py +++ b/packages/parqcast-core/src/parqcast/schemas/inbound.py @@ -4,6 +4,7 @@ OdooCode, OdooDatetime, OdooId, + OdooInteger, OdooName, OdooQuantity, OdooState, @@ -35,5 +36,6 @@ ("remark", OdooText), ("min_quantity", OdooQuantity), ("max_quantity", OdooQuantity), + ("delay", OdooInteger), ] ) diff --git a/packages/parqcast-core/src/parqcast/schemas/odoo_types.py b/packages/parqcast-core/src/parqcast/schemas/odoo_types.py index 1278a2e..c9c1c9f 100644 --- a/packages/parqcast-core/src/parqcast/schemas/odoo_types.py +++ b/packages/parqcast-core/src/parqcast/schemas/odoo_types.py @@ -19,6 +19,10 @@ # --- Odoo Integer fields --- +OdooInteger = pa.int64() +"""Generic integer field (e.g. delay, sequence lengths). +Maps to: fields.Integer""" + OdooId = pa.int64() """Many2one FK, Integer PK, or any Odoo integer field. Maps to: fields.Many2one, fields.Integer, fields.Id""" diff --git a/packages/parqcast-ingesters/src/parqcast/ingesters/v18/__init__.py b/packages/parqcast-ingesters/src/parqcast/ingesters/v18/__init__.py index d50ae83..e867e44 100644 --- a/packages/parqcast-ingesters/src/parqcast/ingesters/v18/__init__.py +++ b/packages/parqcast-ingesters/src/parqcast/ingesters/v18/__init__.py @@ -13,6 +13,7 @@ from .production_actor import ProductionActorV18 from .purchase_actor import PurchaseActorV18 from .reschedule_actor import RescheduleActorV18 +from .supplierinfo_actor import SupplierinfoActorV18 append_to_bundle( "18", @@ -22,6 +23,7 @@ DistributionActorV18, RescheduleActorV18, OrderpointActorV18, + SupplierinfoActorV18, ), ) @@ -31,4 +33,5 @@ "ProductionActorV18", "PurchaseActorV18", "RescheduleActorV18", + "SupplierinfoActorV18", ] diff --git a/packages/parqcast-ingesters/src/parqcast/ingesters/v18/supplierinfo_actor.py b/packages/parqcast-ingesters/src/parqcast/ingesters/v18/supplierinfo_actor.py new file mode 100644 index 0000000..099522d --- /dev/null +++ b/packages/parqcast-ingesters/src/parqcast/ingesters/v18/supplierinfo_actor.py @@ -0,0 +1,52 @@ +import pyarrow as pa + +from parqcast.core.protocols import OdooEnvironment +from parqcast.core.version import V18 + +from ..base import BaseIngester, IngestResult + + +class SupplierinfoActorV18(BaseIngester[V18]): + """Update vendor lead times (delay) based on ML empirical calculations.""" + + decision_type = "LEAD_TIME" + + def apply(self, decisions: pa.Table, env: OdooEnvironment) -> IngestResult: + df = decisions.to_pydict() + updated = 0 + errors = 0 + + for i in range(decisions.num_rows): + supplier_id = df["_odoo_supplier_id"][i] + product_id = df["_odoo_product_id"][i] + delay = df["delay"][i] + + if not supplier_id or not product_id or delay is None: + errors += 1 + continue + + product = env["product.product"].browse(product_id) + if not product.exists(): + errors += 1 + continue + + # Find the exact supplier pricelist link + supplier_info = env["product.supplierinfo"].search( + [ + ("partner_id", "=", supplier_id), + ("product_tmpl_id", "=", product.product_tmpl_id.id), + ], + limit=1, + ) + + if supplier_info: + supplier_info.write({"delay": delay}) + updated += 1 + else: + errors += 1 # We don't create new supplierinfo here, only update existing + + return IngestResult(updated=updated, errors=errors) + + def cleanup_previous(self, env: OdooEnvironment, company_id: int) -> int: + # We don't cleanup past empirical lead times, they are a permanent state update + return 0 diff --git a/packages/parqcast-ingesters/src/parqcast/ingesters/v19/__init__.py b/packages/parqcast-ingesters/src/parqcast/ingesters/v19/__init__.py index 5333890..0520b5d 100644 --- a/packages/parqcast-ingesters/src/parqcast/ingesters/v19/__init__.py +++ b/packages/parqcast-ingesters/src/parqcast/ingesters/v19/__init__.py @@ -13,6 +13,7 @@ from .production_actor import ProductionActorV19 from .purchase_actor import PurchaseActorV19 from .reschedule_actor import RescheduleActorV19 +from .supplierinfo_actor import SupplierinfoActorV19 append_to_bundle( "19", @@ -22,6 +23,7 @@ DistributionActorV19, RescheduleActorV19, OrderpointActorV19, + SupplierinfoActorV19, ), ) @@ -31,4 +33,5 @@ "ProductionActorV19", "PurchaseActorV19", "RescheduleActorV19", + "SupplierinfoActorV19", ] diff --git a/packages/parqcast-ingesters/src/parqcast/ingesters/v19/supplierinfo_actor.py b/packages/parqcast-ingesters/src/parqcast/ingesters/v19/supplierinfo_actor.py new file mode 100644 index 0000000..d3723f3 --- /dev/null +++ b/packages/parqcast-ingesters/src/parqcast/ingesters/v19/supplierinfo_actor.py @@ -0,0 +1,52 @@ +import pyarrow as pa + +from parqcast.core.protocols import OdooEnvironment +from parqcast.core.version import V19 + +from ..base import BaseIngester, IngestResult + + +class SupplierinfoActorV19(BaseIngester[V19]): + """Update vendor lead times (delay) based on ML empirical calculations.""" + + decision_type = "LEAD_TIME" + + def apply(self, decisions: pa.Table, env: OdooEnvironment) -> IngestResult: + df = decisions.to_pydict() + updated = 0 + errors = 0 + + for i in range(decisions.num_rows): + supplier_id = df["_odoo_supplier_id"][i] + product_id = df["_odoo_product_id"][i] + delay = df["delay"][i] + + if not supplier_id or not product_id or delay is None: + errors += 1 + continue + + product = env["product.product"].browse(product_id) + if not product.exists(): + errors += 1 + continue + + # Find the exact supplier pricelist link + supplier_info = env["product.supplierinfo"].search( + [ + ("partner_id", "=", supplier_id), + ("product_tmpl_id", "=", product.product_tmpl_id.id), + ], + limit=1, + ) + + if supplier_info: + supplier_info.write({"delay": delay}) + updated += 1 + else: + errors += 1 # We don't create new supplierinfo here, only update existing + + return IngestResult(updated=updated, errors=errors) + + def cleanup_previous(self, env: OdooEnvironment, company_id: int) -> int: + # We don't cleanup past empirical lead times, they are a permanent state update + return 0 From c562acd67116c9b725fc973b9c946a9b1d277e6f Mon Sep 17 00:00:00 2001 From: Oleg Popov Date: Sat, 2 May 2026 10:05:09 +0200 Subject: [PATCH 6/8] docs: add GEMINI.md context file --- GEMINI.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 GEMINI.md diff --git a/GEMINI.md b/GEMINI.md new file mode 100644 index 0000000..5f92def --- /dev/null +++ b/GEMINI.md @@ -0,0 +1,19 @@ +# Gemini Code Assist Context (Parqcast) + +This file provides workspace context and system instructions specifically tailored for Google Gemini Code Assist and Gemini-powered agents (like Antigravity) working in the `parqcast` repository. + +## 1. Universal Project Rules +Gemini must strictly adhere to the universal architectural boundaries defined in **`AGENTS.MD`**. +* **Do not duplicate those rules here.** Always cross-reference `AGENTS.MD` for the strict package isolation rules (e.g., why `parqcast-core` cannot import `odoo`). + +## 2. Gemini Context Retrieval Instructions +When asked to write code, debug, or answer architectural questions, Gemini should pull the following files into its context window: +* **Schema Inquiries:** If the user asks about available data, columns, or ML decisions, immediately reference `docs/DATA_DICTIONARY.md` and `packages/parqcast-core/src/parqcast/schemas/outbound.py`. +* **Action/Ingester Logic:** If writing code that pushes decisions back to Odoo, reference `packages/parqcast-core/src/parqcast/schemas/inbound.py`. + +## 3. Gemini Coding Style & Toolchain +* **Package Manager:** We strictly use `uv`. When suggesting terminal commands to the user, always format them as `uv run ` (e.g., `uv run ruff check .`, `uv run pytest`). Do not suggest `pip install`. +* **Strict Pyright:** Gemini must write Python code that satisfies `pyright` strict mode. + * Avoid `Any` and implicit `Unknown` types. + * Utilize PEP 695 generics (`type V18`, `type V19`) exactly as shown in the project stubs to enforce Odoo major version safety at compile time. +* **Odoo ORM Restriction:** When generating code, remember that only `parqcast-ingesters` is permitted to invoke the Odoo `env` object dynamically. `parqcast-collectors` must always use raw PostgreSQL queries via `psycopg2`. From 490e1db0e1b395ecb997065a54bcdc3154ce169e Mon Sep 17 00:00:00 2001 From: Oleg Popov Date: Sat, 2 May 2026 10:05:47 +0200 Subject: [PATCH 7/8] docs: revert AGENTS.MD to preserve generic rules --- AGENTS.MD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/AGENTS.MD b/AGENTS.MD index 8d82ba6..1fc043b 100644 --- a/AGENTS.MD +++ b/AGENTS.MD @@ -11,7 +11,7 @@ This document defines the constraints, architecture boundaries, and tooling inst Parqcast is strictly layered. Agents must NEVER introduce imports that violate these boundaries: - **`packages/parqcast-core/`**: Defines data schemas, pyarrow constructs, and protocol interfaces. **Constraint:** Must NEVER import from `odoo`, `psycopg2`, or any other workspace package. - **`packages/parqcast-collectors/`**: Executes fast raw SQL queries using `psycopg2` to build arrow arrays. **Constraint:** Must NEVER import `odoo` ORM models. -- **`packages/parqcast-ingesters/`**: Receives decisions matching `inbound.py` schemas and pushes them into Odoo via the ORM (e.g., updating vendor lead times or creating POs). This is the ONLY python package (outside the addon itself) permitted to use dynamic Odoo typing. +- **`packages/parqcast-ingesters/`**: Pulls data via Odoo ORM to push planning decisions back. This is the ONLY python package (outside the addon itself) permitted to use dynamic Odoo typing. - **`packages/parqcast-transport-http/`**: Handles HTTP uploads of generated chunks. - **`packages/parqcast/`**: The Odoo addon. It bridges the environment with the workspace packages. **Constraint:** Must pass Odoo Community Association (OCA) linting via `bash scripts/oca_lint.sh`. Must **NEVER** contain inline raw SQL queries (`cr.execute`); all tracking SQL is delegated to `packages/parqcast-core`. Transports must be decoupled and registered via `parqcast/utils/transport_registry.py`. From b6adf3acede6f4a3e691fb3618abbceea9323624 Mon Sep 17 00:00:00 2001 From: Oleg Popov Date: Sat, 2 May 2026 10:16:57 +0200 Subject: [PATCH 8/8] feat: add temporal tracking to inbound schema and outbound manifest --- .idea/.gitignore | 10 ++ .idea/misc.xml | 7 + .idea/modules.xml | 8 + .idea/parqcast.iml | 9 ++ .idea/vcs.xml | 6 + docs/edge-cases-review.md | 109 +++++++++++++ docs/settings-improvements.md | 153 ++++++++++++++++++ .../src/parqcast/orchestrator/__init__.py | 1 + .../src/parqcast/core/manifest.py | 8 +- .../src/parqcast/core/tracking.py | 19 ++- .../src/parqcast/schemas/inbound.py | 2 + 11 files changed, 325 insertions(+), 7 deletions(-) create mode 100644 .idea/.gitignore create mode 100644 .idea/misc.xml create mode 100644 .idea/modules.xml create mode 100644 .idea/parqcast.iml create mode 100644 .idea/vcs.xml create mode 100644 docs/edge-cases-review.md create mode 100644 docs/settings-improvements.md diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..ab1f416 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,10 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Ignored default folder with query files +/queries/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..d4f91c7 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..e4d4fa2 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/parqcast.iml b/.idea/parqcast.iml new file mode 100644 index 0000000..d6ebd48 --- /dev/null +++ b/.idea/parqcast.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/docs/edge-cases-review.md b/docs/edge-cases-review.md new file mode 100644 index 0000000..fbb241b --- /dev/null +++ b/docs/edge-cases-review.md @@ -0,0 +1,109 @@ +# Edge Cases Review + +Date: 2026-04-27 + +Scope reviewed: + +- Version gate and bundle registry +- Orchestrator finalization and retention paths +- Inbound receiver / ingester dispatch +- Live-DB test coverage for the new v18 path + +Verification performed: + +- `uv run pytest tests/test_version_gate.py tests/test_capabilities.py tests/test_schemas.py tests/test_manifest.py -q` +- Result: `46 passed` + +Not performed: + +- No live DB test run against `PARQCAST_TEST_DB` +- No Odoo addon install/upgrade test + +## Findings + +### 1. Receiver still hardcodes v19 ingesters, so v18 inbound writeback can dispatch through the wrong actor + +Severity: high + +The receiver loads ingesters from `ALL_INGESTERS`, and that map is still pinned to v19 classes only. The code comment explicitly says it is kept v19-keyed, and `Receiver.run()` does not resolve the active Odoo major before choosing an ingester class. [packages/parqcast-ingesters/src/parqcast/ingesters/__init__.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-ingesters/src/parqcast/ingesters/__init__.py:15) [packages/parqcast-ingesters/src/parqcast/receiver/__init__.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-ingesters/src/parqcast/receiver/__init__.py:55) + +This is no longer safe after adding v18. The concrete proof point is purchase-order writeback: `PurchaseActorV18` writes `product_uom`, while `PurchaseActorV19` writes `product_uom_id`. If the receiver is pointed at an Odoo 18 database, the current dispatch path will still instantiate the v19 actor for `"PO"` decisions. [packages/parqcast-ingesters/src/parqcast/ingesters/v18/purchase_actor.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-ingesters/src/parqcast/ingesters/v18/purchase_actor.py:53) [packages/parqcast-ingesters/src/parqcast/ingesters/v19/purchase_actor.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-ingesters/src/parqcast/ingesters/v19/purchase_actor.py:53) + +Recommended direction: + +- Make the receiver version-aware by resolving the current major from `env.cr` and building its dispatch table from `REGISTRY[version].ingesters`. +- If cross-version manifests are expected to be replayed elsewhere, include the source Odoo major in the receiver contract and validate it before dispatch. + +### 2. Automatic cleanup bypasses the configured retention policy and can orphan attachment payloads + +Severity: high + +The addon UI exposes `parqcast_cleanup_days`, and the model-level cleanup action deletes runs older than that cutoff. [packages/parqcast/models/parqcast_settings.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/models/parqcast_settings.py:66) [packages/parqcast/models/parqcast_run.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/models/parqcast_run.py:76) + +But the actual automatic path in the orchestrator does something different: every successful run calls the core SQL helper `ExportRun.cleanup_old()` with its default `keep_last=3`, so retention is really "keep the last 3 done runs", not "delete after N days". [packages/parqcast-collectors/src/parqcast/orchestrator/__init__.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-collectors/src/parqcast/orchestrator/__init__.py:376) [packages/parqcast-core/src/parqcast/core/tracking.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-core/src/parqcast/core/tracking.py:126) + +That would already be a policy mismatch, but it gets worse with the default attachment transport. Export files are stored as `ir.attachment` rows tied to `parqcast.run` via `res_model` / `res_id`. The SQL cleanup only deletes from `parqcast_export_run`; it does not touch attachments. That means successful exports can leave behind orphaned attachment rows and binary payloads once their run rows are deleted. [packages/parqcast/models/transport_attachment.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/models/transport_attachment.py:36) + +Recommended direction: + +- Pick one retention model and enforce it in one place. +- If attachment transport stays the default, cleanup must go through an Odoo-aware path that also deletes related attachments. + +### 3. Manifest metadata falls back to `odoo_version="17.0"` instead of the actual exporting database version + +Severity: medium + +`build_manifest()` defaults the top-level `odoo_version` to `"17.0"`. The orchestrator's `_finalize()` call does not pass `caps.odoo_version`, even though it re-probes capabilities immediately before building the manifest. The exported manifest therefore carries the wrong top-level Odoo version metadata for both v18 and v19 runs. [packages/parqcast-core/src/parqcast/core/manifest.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-core/src/parqcast/core/manifest.py:14) [packages/parqcast-collectors/src/parqcast/orchestrator/__init__.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-collectors/src/parqcast/orchestrator/__init__.py:354) + +There is a partial workaround because `manifest["capabilities"]` is filled afterwards, but any downstream reader using the manifest's top-level `odoo_version` field will see incorrect data. + +Recommended direction: + +- Pass the actual `caps.odoo_version` into `build_manifest()`. +- Remove the misleading default, or make it explicit that callers must provide it. + +### 4. Finalized manifests lose the actual chunk failure reason + +Severity: medium + +Chunk errors are stored with a real `error_message` when collection or upload fails. [packages/parqcast-core/src/parqcast/core/tracking.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-core/src/parqcast/core/tracking.py:283) + +During finalization, however, the manifest error list is built as `f"{c.collector}: {c.state}"`. Since `c.state` is always just `"error"` for failed chunks, operators lose the actual exception text in the exported manifest. [packages/parqcast-collectors/src/parqcast/orchestrator/__init__.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-collectors/src/parqcast/orchestrator/__init__.py:352) + +Impact: + +- Post-mortem debugging from `manifest.json` becomes much harder. +- Remote systems consuming manifests cannot distinguish schema failures, permission failures, transport failures, or data-quality failures. + +Recommended direction: + +- Include `error_message` in the finalized manifest. +- Consider separating collection errors from upload errors if downstream automation needs to react differently. + +### 5. The live-DB test file is only partially version-parameterized + +Severity: medium + +Most of `tests/test_live_db.py` is bundle-aware: `_probe_and_build()` resolves `REGISTRY[TEST_ODOO_VERSION]` and calls that bundle's `probe_capabilities`. [tests/test_live_db.py](/Users/thatoleg/datastructsro/parqcast/tests/test_live_db.py:36) + +But `test_probe_capabilities()` still imports and calls `probe_v19()` directly. So a run with `PARQCAST_TEST_ODOO_VERSION=18` does not fully exercise the v18-specific probe entrypoint even though the surrounding test module suggests it does. [tests/test_live_db.py](/Users/thatoleg/datastructsro/parqcast/tests/test_live_db.py:27) [tests/test_live_db.py](/Users/thatoleg/datastructsro/parqcast/tests/test_live_db.py:115) + +This is a test gap rather than a runtime bug, but it matters because the branch's main risk surface is exactly the version split. + +Recommended direction: + +- Replace the direct `probe_v19()` call with `bundle.probe_capabilities(...)` or parametrize the test over both wrappers. + +## Watch List + +### Bundle bootstrap makes the install-time gate weaker than it looks + +`assert_supported()` accepts a major if it exists in `REGISTRY`, and `REGISTRY` is pre-bootstrapped with empty v18/v19 bundles before the version-specific subpackages are imported. The addon `_register_hook()` therefore proves only that the major is listed, not that the corresponding bundle was imported successfully or is complete. [packages/parqcast-core/src/parqcast/core/registry.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-core/src/parqcast/core/registry.py:49) [packages/parqcast-core/src/parqcast/core/version_gate.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-core/src/parqcast/core/version_gate.py:41) [packages/parqcast/models/parqcast_version_gate.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/models/parqcast_version_gate.py:19) + +That is acceptable during staged migrations, but it is a weak fit for a production install/upgrade guard. A broken import in a version bundle would still let installation proceed and only fail later when the cron path hits `_resolve_bundle()`. + +### Keyset pagination assumes collectors do not already define their own `ORDER BY` + +`BaseCollector.collect()` appends `ORDER BY {pk_column}` whenever keyset bounds are applied. That is fine for the current collector set, but it will break as soon as a future collector ships SQL with its own trailing `ORDER BY`. [packages/parqcast-collectors/src/parqcast/collectors/base.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-collectors/src/parqcast/collectors/base.py:101) + +This is not a current regression, but it is a good invariance to encode in tests if more complex collectors get added. diff --git a/docs/settings-improvements.md b/docs/settings-improvements.md new file mode 100644 index 0000000..f4ccf64 --- /dev/null +++ b/docs/settings-improvements.md @@ -0,0 +1,153 @@ +# Settings Improvements + +Date: 2026-04-27 + +Scope reviewed: + +- Settings model and config parameters +- Settings form UI +- Cron defaults +- Retention and attachment behavior + +## Recommended Changes + +### 1. Make retention settings match runtime behavior + +Current issue: + +- The settings UI exposes `parqcast_cleanup_days` as a days-based retention policy. +- The automatic runtime path instead deletes all but the last 3 successful runs. + +Relevant files: + +- [packages/parqcast/models/parqcast_settings.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/models/parqcast_settings.py:66) +- [packages/parqcast/models/parqcast_run.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/models/parqcast_run.py:76) +- [packages/parqcast-core/src/parqcast/core/tracking.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-core/src/parqcast/core/tracking.py:126) +- [packages/parqcast-collectors/src/parqcast/orchestrator/__init__.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-collectors/src/parqcast/orchestrator/__init__.py:376) + +Suggestion: + +- Pick one retention model and enforce it consistently. +- If the intended policy is days-based, wire `parqcast_cleanup_days` into the orchestrator cleanup path. +- If the intended policy is count-based, change the settings label and help text accordingly. + +### 2. Make attachment cleanup explicit + +Current issue: + +- Attachment transport stores exports as `ir.attachment` rows linked to `parqcast.run`. +- SQL cleanup of old runs does not automatically clean up those attachments. + +Relevant file: + +- [packages/parqcast/models/transport_attachment.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/models/transport_attachment.py:36) + +Suggestion: + +- When deleting old runs, also delete their attachments through an Odoo-aware cleanup path. +- Add an optional “purge uploaded attachments after N days” policy if attachment transport remains the default. +- Show a short storage warning in the settings UI when `attachment` transport is selected. + +### 3. Fix the schedule wording + +Current issue: + +- The UI says `Once daily (midnight UTC)`. +- The implementation only switches the cron interval to `1 day`; it does not actually normalize `nextcall` to midnight UTC. + +Relevant files: + +- [packages/parqcast/models/parqcast_settings.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/models/parqcast_settings.py:84) +- [packages/parqcast/data/ir_cron_data.xml](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/data/ir_cron_data.xml:4) + +Suggestion: + +- Either actually schedule the cron at midnight UTC, or rename the option to something accurate like `Every day`. +- If operators care about exact timing, expose a run time and timezone instead of hardcoding wording. + +### 4. Add a status block to settings + +Current issue: + +- The settings page is mostly input-only. +- Operators cannot quickly see whether Parqcast is healthy without leaving the page. + +Relevant file: + +- [packages/parqcast/views/parqcast_settings_views.xml](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/views/parqcast_settings_views.xml:8) + +Suggestion: + +- Add a read-only status section showing detected Odoo major, version-gate status, selected company, transport type, next cron run, last run state, and last run error when present. + +This would make misconfiguration much easier to diagnose. + +### 5. Add transport validation actions + +Current issue: + +- The form hides and shows transport-specific fields, but it does not validate the configured destination proactively. + +Relevant files: + +- [packages/parqcast/views/parqcast_settings_views.xml](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/views/parqcast_settings_views.xml:51) +- [packages/parqcast/models/parqcast_cron.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/models/parqcast_cron.py:16) + +Suggestion: + +- Add `Test connection` / `Test write` actions for local filesystem, HTTP server, S3, and attachment transport. + +- Validate required fields server-side before saving or before running the first export. +- Return actionable errors rather than waiting for cron to fail. + +### 6. Improve manual run feedback + +Current issue: + +- `Run Export Now` always returns a generic success notification. +- It does not reflect whether the tick planned, collected, uploaded, paused, or failed. + +Relevant file: + +- [packages/parqcast/models/parqcast_settings.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/models/parqcast_settings.py:115) + +Suggestion: + +- Include the returned state in the notification. +- Include the run UUID when available. +- Surface immediate errors in the notification if the tick fails. + +### 7. Add guardrails around time budget + +Current issue: + +- `parqcast_time_budget` is free-form. +- Very low values can make the system appear broken because planning or upload may never make visible progress. + +Relevant files: + +- [packages/parqcast/models/parqcast_settings.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/models/parqcast_settings.py:72) +- [packages/parqcast/views/parqcast_settings_views.xml](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/views/parqcast_settings_views.xml:32) + +Suggestion: + +- Add sane minimum and recommended values in help text or validation. +- Consider presets such as `safe`, `balanced`, and `aggressive` if operators are not expected to tune raw seconds. + +## Suggested Priority + +Implement first: + +- retention consistency +- attachment cleanup +- schedule wording fix + +Implement second: + +- status block +- transport validation +- better manual-run feedback + +Implement later: + +- time-budget guardrails diff --git a/packages/parqcast-collectors/src/parqcast/orchestrator/__init__.py b/packages/parqcast-collectors/src/parqcast/orchestrator/__init__.py index 25edc03..e2bb3d5 100644 --- a/packages/parqcast-collectors/src/parqcast/orchestrator/__init__.py +++ b/packages/parqcast-collectors/src/parqcast/orchestrator/__init__.py @@ -366,6 +366,7 @@ def _finalize(self, cr: ReadCursor, run: ExportRun, prefix: str, t0: float) -> J total_duration=round(time.monotonic() - t0, 3), errors=errors, warnings=[], + started_at=run.started_at, ) manifest["capabilities"] = caps.summary() manifest["run_uuid"] = run.run_uuid diff --git a/packages/parqcast-core/src/parqcast/core/manifest.py b/packages/parqcast-core/src/parqcast/core/manifest.py index 91f4246..22cab03 100644 --- a/packages/parqcast-core/src/parqcast/core/manifest.py +++ b/packages/parqcast-core/src/parqcast/core/manifest.py @@ -20,11 +20,15 @@ def build_manifest( errors: list[str] | None = None, warnings: list[str] | None = None, total_duration: float = 0.0, + started_at: datetime | None = None, ) -> JsonDict: + finished_at = datetime.now(UTC) return { "version": parqcast_version, - "schema_version": datetime.now(UTC).strftime("%Y-%m-%d"), - "timestamp": datetime.now(UTC).isoformat(), + "schema_version": finished_at.strftime("%Y-%m-%d"), + "timestamp": finished_at.isoformat(), + "started_at": started_at.isoformat() if started_at else None, + "finished_at": finished_at.isoformat(), "company": company, "company_id": company_id, "odoo_version": odoo_version, diff --git a/packages/parqcast-core/src/parqcast/core/tracking.py b/packages/parqcast-core/src/parqcast/core/tracking.py index c7a775b..8aaaa86 100644 --- a/packages/parqcast-core/src/parqcast/core/tracking.py +++ b/packages/parqcast-core/src/parqcast/core/tracking.py @@ -24,11 +24,19 @@ class ExportRun: STATES = ("pending", "collecting", "uploading", "done", "error") - def __init__(self, id: int, run_uuid: str, state: str, company_id: int | None = None): + def __init__( + self, + id: int, + run_uuid: str, + state: str, + company_id: int | None = None, + started_at: datetime | None = None, + ): self.id = id self.run_uuid = run_uuid self.state = state self.company_id = company_id + self.started_at = started_at @classmethod def ensure_table(cls, cr: ReadCursor): @@ -54,28 +62,29 @@ def ensure_table(cls, cr: ReadCursor): @classmethod def create(cls, cr: ReadCursor, company_id: int | None = None, company_name: str = "") -> ExportRun: run_uuid = str(_uuid.uuid4()) + started_at = datetime.now(UTC) cr.execute( """ INSERT INTO parqcast_export_run (run_uuid, state, company_id, company_name, started_at) VALUES (%s, 'pending', %s, %s, %s) RETURNING id """, - (run_uuid, company_id, company_name, datetime.now(UTC)), + (run_uuid, company_id, company_name, started_at), ) run_id = fetch_one(cr)[0] - return cls(id=run_id, run_uuid=run_uuid, state="pending", company_id=company_id) + return cls(id=run_id, run_uuid=run_uuid, state="pending", company_id=company_id, started_at=started_at) @classmethod def find_active(cls, cr: ReadCursor) -> ExportRun | None: cr.execute(""" - SELECT id, run_uuid, state, company_id + SELECT id, run_uuid, state, company_id, started_at FROM parqcast_export_run WHERE state NOT IN ('done', 'error') ORDER BY id DESC LIMIT 1 """) row = fetch_one_or_none(cr) if row: - return cls(id=row[0], run_uuid=row[1], state=row[2], company_id=row[3]) + return cls(id=row[0], run_uuid=row[1], state=row[2], company_id=row[3], started_at=row[4]) return None def set_state(self, cr: ReadCursor, state: str): diff --git a/packages/parqcast-core/src/parqcast/schemas/inbound.py b/packages/parqcast-core/src/parqcast/schemas/inbound.py index a57012f..20cd3b0 100644 --- a/packages/parqcast-core/src/parqcast/schemas/inbound.py +++ b/packages/parqcast-core/src/parqcast/schemas/inbound.py @@ -14,6 +14,8 @@ DECISIONS_SCHEMA = pa.schema( [ ("decision_id", OdooCode), + ("run_uuid", OdooText), + ("decision_timestamp", OdooDatetime), ("decision_type", OdooState), ("status", OdooState), ("item_name", OdooName),