diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..ab1f416 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,10 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Ignored default folder with query files +/queries/ +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..d4f91c7 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,7 @@ + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..e4d4fa2 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/parqcast.iml b/.idea/parqcast.iml new file mode 100644 index 0000000..d6ebd48 --- /dev/null +++ b/.idea/parqcast.iml @@ -0,0 +1,9 @@ + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..35eb1dd --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/AGENTS.MD b/AGENTS.MD index 92c91b5..1fc043b 100644 --- a/AGENTS.MD +++ b/AGENTS.MD @@ -13,7 +13,7 @@ Parqcast is strictly layered. Agents must NEVER introduce imports that violate t - **`packages/parqcast-collectors/`**: Executes fast raw SQL queries using `psycopg2` to build arrow arrays. **Constraint:** Must NEVER import `odoo` ORM models. - **`packages/parqcast-ingesters/`**: Pulls data via Odoo ORM to push planning decisions back. This is the ONLY python package (outside the addon itself) permitted to use dynamic Odoo typing. - **`packages/parqcast-transport-http/`**: Handles HTTP uploads of generated chunks. -- **`packages/parqcast/`**: The Odoo addon. It bridges the environment with the workspace packages. **Constraint:** Must pass Odoo Community Association (OCA) linting via `bash scripts/oca_lint.sh`. +- **`packages/parqcast/`**: The Odoo addon. It bridges the environment with the workspace packages. **Constraint:** Must pass Odoo Community Association (OCA) linting via `bash scripts/oca_lint.sh`. Must **NEVER** contain inline raw SQL queries (`cr.execute`); all tracking SQL is delegated to `packages/parqcast-core`. Transports must be decoupled and registered via `parqcast/utils/transport_registry.py`. ## 3. Odoo Versioning (PEP 695 Generics) Parqcast certifies per Odoo major. @@ -24,3 +24,7 @@ Parqcast certifies per Odoo major. ## 4. Testing - Use `uv run pytest tests/ -x -q` to verify logic. - Do not run `tests/test_live_db.py` without the `PARQCAST_TEST_DB` environment variable set. + +## 5. Data Architecture & Schemas +When answering questions about what data is available, writing SQL extractors, or modifying PyArrow schemas, you MUST first read `docs/DATA_DICTIONARY.md`. +* All outbound schemas are defined and strictly typed in `packages/parqcast-core/src/parqcast/schemas/outbound.py`. diff --git a/GEMINI.md b/GEMINI.md new file mode 100644 index 0000000..5f92def --- /dev/null +++ b/GEMINI.md @@ -0,0 +1,19 @@ +# Gemini Code Assist Context (Parqcast) + +This file provides workspace context and system instructions specifically tailored for Google Gemini Code Assist and Gemini-powered agents (like Antigravity) working in the `parqcast` repository. + +## 1. Universal Project Rules +Gemini must strictly adhere to the universal architectural boundaries defined in **`AGENTS.MD`**. +* **Do not duplicate those rules here.** Always cross-reference `AGENTS.MD` for the strict package isolation rules (e.g., why `parqcast-core` cannot import `odoo`). + +## 2. Gemini Context Retrieval Instructions +When asked to write code, debug, or answer architectural questions, Gemini should pull the following files into its context window: +* **Schema Inquiries:** If the user asks about available data, columns, or ML decisions, immediately reference `docs/DATA_DICTIONARY.md` and `packages/parqcast-core/src/parqcast/schemas/outbound.py`. +* **Action/Ingester Logic:** If writing code that pushes decisions back to Odoo, reference `packages/parqcast-core/src/parqcast/schemas/inbound.py`. + +## 3. Gemini Coding Style & Toolchain +* **Package Manager:** We strictly use `uv`. When suggesting terminal commands to the user, always format them as `uv run ` (e.g., `uv run ruff check .`, `uv run pytest`). Do not suggest `pip install`. +* **Strict Pyright:** Gemini must write Python code that satisfies `pyright` strict mode. + * Avoid `Any` and implicit `Unknown` types. + * Utilize PEP 695 generics (`type V18`, `type V19`) exactly as shown in the project stubs to enforce Odoo major version safety at compile time. +* **Odoo ORM Restriction:** When generating code, remember that only `parqcast-ingesters` is permitted to invoke the Odoo `env` object dynamically. `parqcast-collectors` must always use raw PostgreSQL queries via `psycopg2`. diff --git a/README.md b/README.md index 0102ff1..4622beb 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ This repo is a `uv` workspace with four Python packages plus the Odoo addon: |------|---------| | `packages/parqcast-core/` | Schemas, transport, tracking, capabilities, manifest, protocols (pyarrow only) | | `packages/parqcast-collectors/` | SQL-based data extractors + time-budgeted orchestrator | -| `packages/parqcast-ingesters/` | ORM-based writers (push planning decisions back into Odoo) | +| `packages/parqcast-ingesters/` | ORM-based writers (push prescriptive ML decisions like POs and dynamic lead times back into Odoo) | | `packages/parqcast-transport-http/` | HTTP upload transport | | `packages/parqcast/` | The Odoo 19 addon (settings UI, cron job, env adapter) — see [its README](packages/parqcast/README.rst) | diff --git a/docs/DATA_DICTIONARY.md b/docs/DATA_DICTIONARY.md new file mode 100644 index 0000000..7d55e7d --- /dev/null +++ b/docs/DATA_DICTIONARY.md @@ -0,0 +1,85 @@ +# Parqcast Data Dictionary + +## Overview +This document serves as the single source of truth for the data architecture within the `parqcast` repository. Parqcast extracts operational data from Odoo, serializes it into Apache Parquet format via PyArrow, and transmits it for external forecasting, predictive replenishment, and supply chain planning. + +**Critical Note for AI Agents & Developers:** +Data is **NOT** extracted via the standard Odoo ORM (`env['...'].search_read()`). Instead, it is extracted using extremely fast, raw PostgreSQL queries defined in `packages/parqcast-collectors/`. The extracted data strictly conforms to the PyArrow schemas defined in `packages/parqcast-core/src/parqcast/schemas/outbound.py`. + +When answering questions about available data or modifying the pipeline, you MUST refer to `outbound.py` as the ultimate authority on what data columns are currently supported. + +--- + +## Available Data Entities + +Below is a categorized list of the primary data entities currently extracted by Parqcast. + +### 1. Manufacturing & Production +- **`MRP_PRODUCTION`** (`mrp_production`): Core manufacturing orders. Extracts state, priorities, and crucial scheduling dates to feed production planning algorithms. + - **Schema Fields:** `_odoo_mo_id`, `name`, `_odoo_product_id`, `product_name`, `_odoo_bom_id`, `product_qty`, `qty_producing`, `_odoo_uom_id`, `state`, `priority`, `reservation_state`, `date_start`, `date_finished`, `date_deadline`, `_odoo_location_src_id`, `_odoo_location_dest_id`, `_odoo_picking_type_id`, `origin`, `_odoo_company_id`. +- **`MRP_WORKORDER`** (`mrp_workorder`): Specific routing operations tied to an MO. Extracts durations and costs to model workcenter capacity. + - **Schema Fields:** `_odoo_wo_id`, `_odoo_mo_id`, `display_name`, `_odoo_workcenter_id`, `workcenter_name`, `_odoo_operation_id`, `state`, `date_start`, `date_finished`, `duration_expected`, `duration_unit`, `is_user_working`, `employee_costs_hour`. +- **`BOM`** (`mrp_bom`): Bills of Materials. Essential for recursive material planning. + - **Schema Fields:** `_odoo_bom_id`, `_odoo_product_tmpl_id`, `_odoo_product_id`, `product_name`, `bom_type`, `product_qty`, `_odoo_uom_id`, `uom_name`, `product_qty_multiple`, `produce_delay`, `days_to_prepare_mo`, `sequence`, `code`, `_odoo_company_id`. +- **`BOM_LINES`** (`mrp_bom_line`): Component-level BOM requirements. + - **Schema Fields:** `_odoo_bom_line_id`, `_odoo_bom_id`, `_odoo_product_id`, `product_name`, `product_qty`, `_odoo_uom_id`, `uom_name`, `_odoo_operation_id`, `operation_name`, `bom_product_template_attribute_value_ids`. +- **`BOM_OPERATIONS`** (`mrp_routing_workcenter`): Standard cycle times. + - **Schema Fields:** `_odoo_operation_id`, `_odoo_bom_id`, `name`, `sequence`, `_odoo_workcenter_id`, `workcenter_name`, `time_cycle`, `workcenter_quantity`, `search_mode`, `_odoo_skill_id`, `skill_name`, `post_operation_time`. +- **`WORKCENTER`** (`mrp_workcenter`): Captures default capacity and costs to constrain production schedules. + - **Schema Fields:** `_odoo_workcenter_id`, `name`, `_odoo_resource_id`, `default_capacity`, `time_efficiency`, `is_tool`, `is_constrained`, `_odoo_owner_id`, `owner_name`, `_odoo_calendar_id`, `calendar_name`, `post_operation_time`, `employee_costs_hour`, `_odoo_company_id`. + +### 2. Inventory & Stock +- **`STOCK_QUANT`** (`stock_quant`): Live, real-time snapshot of inventory. Distinguishes between physical and reserved quantity. + - **Schema Fields:** `_odoo_quant_id`, `_odoo_product_id`, `product_name`, `product_code`, `_odoo_location_id`, `location_name`, `location_usage`, `_odoo_warehouse_id`, `warehouse_code`, `quantity`, `reserved_quantity`, `_odoo_lot_id`, `lot_name`, `expiry_date`, `_odoo_package_id`, `package_name`, `_odoo_owner_id`, `owner_name`, `in_date`, `_odoo_company_id`. +- **`STOCK_MOVE`** (`stock_move`): Planned or pending inventory movements. + - **Schema Fields:** `_odoo_move_id`, `_odoo_product_id`, `product_name`, `product_uom_qty`, `quantity`, `_odoo_uom_id`, `state`, `priority`, `_odoo_picking_id`, `picking_name`, `_odoo_location_id`, `_odoo_location_dest_id`, `_odoo_warehouse_id`, `_odoo_production_id`, `_odoo_sale_line_id`, `_odoo_purchase_line_id`, `origin`, `date`, `date_deadline`, `delay_alert_date`, `procure_method`, `move_orig_ids`, `move_dest_ids`, `is_subcontract`. +- **`STOCK_MOVE_LINE`** (`stock_move_line`): Granular execution of moves. Tracks the exact lot and expiration. + - **Schema Fields:** `_odoo_move_line_id`, `_odoo_move_id`, `_odoo_picking_id`, `_odoo_product_id`, `_odoo_product_uom_id`, `quantity`, `_odoo_lot_id`, `lot_name`, `_odoo_package_id`, `_odoo_result_package_id`, `_odoo_owner_id`, `_odoo_location_id`, `_odoo_location_dest_id`, `state`, `date`, `expiration_date`, `_odoo_company_id`. +- **`STOCK_LOCATION`** (`stock_location`): The spatial hierarchy, noting location usage and removal strategy. + - **Schema Fields:** `_odoo_location_id`, `name`, `complete_name`, `usage`, `_odoo_parent_id`, `_odoo_warehouse_id`, `warehouse_code`, `removal_strategy`, `_odoo_storage_category_id`, `storage_category_name`, `active`, `_odoo_company_id`. +- **`STOCK_PICKING`** (`stock_picking`): The overarching transfer document (receipts/deliveries). + - **Schema Fields:** `_odoo_picking_id`, `name`, `state`, `picking_type_code`, `_odoo_location_id`, `_odoo_location_dest_id`, `scheduled_date`, `date_deadline`, `date_done`, `origin`, `_odoo_partner_id`, `_odoo_backorder_id`, `_odoo_sale_id`, `priority`, `move_type`, `_odoo_company_id`. +- **`ORDERPOINT`** (`stock_warehouse_orderpoint`): Configured reordering rules. + - **Schema Fields:** `_odoo_orderpoint_id`, `_odoo_product_id`, `product_name`, `_odoo_warehouse_id`, `warehouse_code`, `_odoo_location_id`, `location_name`, `product_min_qty`, `product_max_qty`, `trigger`, `active`, `snoozed_until`, `_odoo_route_id`, `route_name`, `deadline_date`, `_odoo_replenishment_uom_id`, `qty_to_order_computed`, `qty_to_order_manual`, `_odoo_supplier_id`, `_odoo_company_id`. + +### 3. Sales & Purchasing +- **`SALE_ORDER`** (`sale_order`): Customer orders. + - **Schema Fields:** `_odoo_so_id`, `name`, `state`, `_odoo_partner_id`, `date_order`, `commitment_date`, `_odoo_warehouse_id`, `warehouse_code`, `picking_policy`, `amount_untaxed`, `amount_total`, `_odoo_currency_id`, `invoice_status`, `delivery_status`, `_odoo_company_id`. +- **`SALE_ORDER_LINE`** (`sale_order_line`): Specific order lines tracking requested vs delivered quantities. + - **Schema Fields:** `_odoo_sol_id`, `_odoo_so_id`, `so_name`, `_odoo_product_id`, `product_name`, `product_uom_qty`, `qty_delivered`, `qty_invoiced`, `_odoo_uom_id`, `uom_name`, `price_unit`, `discount`, `price_subtotal`, `customer_lead`, `_odoo_partner_id`, `partner_name`, `is_company`, `commitment_date`, `date_order`, `so_state`, `sol_state`, `_odoo_warehouse_id`, `warehouse_code`, `picking_policy`. +- **`PURCHASE_ORDER`** (`purchase_order`): Vendor orders showing status and planned dates. + - **Schema Fields:** `_odoo_po_id`, `name`, `state`, `_odoo_partner_id`, `date_order`, `date_approve`, `date_planned`, `receipt_status`, `invoice_status`, `amount_untaxed`, `amount_total`, `_odoo_currency_id`, `_odoo_picking_type_id`, `_odoo_warehouse_id`, `origin`, `priority`, `_odoo_company_id`. +- **`PURCHASE_ORDER_LINE`** (`purchase_order_line`): Purchased quantities and unit prices. + - **Schema Fields:** `_odoo_pol_id`, `_odoo_po_id`, `po_name`, `po_state`, `receipt_status`, `_odoo_product_id`, `product_name`, `product_qty`, `qty_received`, `_odoo_uom_id`, `uom_name`, `price_unit`, `_odoo_currency_id`, `date_order`, `date_planned`, `_odoo_partner_id`, `partner_name`, `_odoo_warehouse_id`, `origin`, `_odoo_sale_line_id`, `_odoo_orderpoint_id`, `_odoo_company_id`. +- **`POS_ORDER` & `POS_ORDER_LINE`** (`pos_order`): Point of Sale transactions for retail forecasting. + - **POS_ORDER Schema Fields:** `_odoo_pos_order_id`, `name`, `state`, `date_order`, `_odoo_partner_id`, `_odoo_session_id`, `_odoo_config_id`, `amount_total`, `amount_tax`, `amount_paid`, `is_refund`, `_odoo_company_id`. + - **POS_ORDER_LINE Schema Fields:** `_odoo_pos_line_id`, `_odoo_pos_order_id`, `_odoo_product_id`, `full_product_name`, `qty`, `price_unit`, `price_subtotal`, `price_subtotal_incl`, `discount`, `_odoo_company_id`. + +### 4. Master Data +- **`PRODUCT`** (`product_product` / `product_template`): Core item master. Extracts critical rules to determine how items are procured. + - **Schema Fields:** `_odoo_product_id`, `_odoo_template_id`, `name`, `name_lang2`, `name_lang3`, `default_code`, `barcode`, `display_name`, `_odoo_uom_id`, `uom_name`, `uom_factor`, `list_price`, `standard_price`, `weight`, `volume`, `_odoo_categ_id`, `category_name`, `purchase_ok`, `sale_ok`, `is_storable`, `active`, `product_type`, `description_sale`, `invoice_policy`, `purchase_method`, `use_expiration_date`, `service_tracking`, `sale_delay`, `route_ids`, `route_names`, `expiration_time`, `tracking`, `_odoo_company_id`. +- **`PRODUCT_SUPPLIERINFO`** (`product_supplierinfo`): Vendor pricing and constraints. Critical for replenishment planners. + - **Schema Fields:** `_odoo_supplierinfo_id`, `_odoo_template_id`, `_odoo_product_id`, `_odoo_partner_id`, `partner_name`, `delay`, `min_qty`, `price`, `_odoo_currency_id`, `currency_name`, `sequence`, `date_start`, `date_end`, `batching_window`, `is_subcontractor`. +- **`PARTNER`** (`res_partner`): Suppliers and customers. + - **Schema Fields:** `_odoo_partner_id`, `name`, `ref`, `is_company`, `_odoo_parent_id`, `parent_name`, `_odoo_country_id`, `customer_rank`, `supplier_rank`, `active`, `_odoo_company_id`. +- **`COMPANY` & `CURRENCY`** (`res_company`, `res_currency`): Context variables like `security_lead` and exchange rates. + - **COMPANY Schema Fields:** `_odoo_company_id`, `name`, `_odoo_currency_id`, `_odoo_parent_id`, `security_lead`, `active`, `_odoo_country_id`. + - **CURRENCY Schema Fields:** `_odoo_currency_id`, `name`, `symbol`, `decimal_places`, `rounding`, `active`, `_odoo_rate_id`, `rate`, `rate_date`, `_odoo_rate_company_id`. + +### 5. Inbound Actions & Decisions (The Ingesters) +While the entities above describe the data being extracted *out* of Odoo via `parqcast-collectors` (using raw SQL), the external planner also pushes actionable decisions *back into* Odoo. This data is handled by `parqcast-ingesters`, which is the ONLY package explicitly permitted to use the standard Odoo ORM (e.g., `env['purchase.order'].create(...)`) to safely trigger Odoo's internal workflows. + +- **`DECISIONS`** (`inbound.py`): The prescriptive actions prescribed by the external AI/planner (e.g., creating a Purchase Order, updating an Orderpoint, scheduling Manufacturing). + - **Schema Fields:** `decision_id`, `decision_type`, `status`, `item_name`, `_odoo_product_id`, `_odoo_uom_id`, `location_name`, `_odoo_location_id`, `quantity`, `start_date`, `end_date`, `supplier_name`, `_odoo_supplier_id`, `_odoo_bom_id`, `origin_location`, `destination_location`, `parent_reference`, `workcenter_name`, `_odoo_workcenter_id`, `batch`, `remark`, `min_quantity`, `max_quantity`, `delay`. + +--- + +## Technical Nuances + +1. **Internationalization (i18n):** Queries are dynamically parameterized to extract translatable JSONB fields in the company's language. The `parqcast-collectors` system injects the primary language via `self.caps.lang(1)` into the raw SQL (e.g., `name->>'{lang1}'`). `en_US` is only used as a final fallback if the environment lacks language capabilities entirely. This guarantees that LLMs and external systems receive the correctly localized entity names natively. +2. **Timestamps:** All dates and datetimes are exported as UTC to ensure cross-system consistency. +3. **Odoo IDs:** Primary and foreign keys are explicitly typed and prefixed with `_odoo_` in the Parquet schema (e.g., `_odoo_product_id`) to differentiate them from downstream system IDs. +4. **Boolean & Numeric Types:** We strictly map Odoo's internal types to optimized PyArrow equivalents (e.g., `OdooMonetary`, `OdooQuantity`, `OdooId`) to compress file sizes and maximize memory efficiency. + +--- +*For the exact, up-to-date column definitions, always inspect `outbound.py`.* diff --git a/docs/edge-cases-review.md b/docs/edge-cases-review.md new file mode 100644 index 0000000..fbb241b --- /dev/null +++ b/docs/edge-cases-review.md @@ -0,0 +1,109 @@ +# Edge Cases Review + +Date: 2026-04-27 + +Scope reviewed: + +- Version gate and bundle registry +- Orchestrator finalization and retention paths +- Inbound receiver / ingester dispatch +- Live-DB test coverage for the new v18 path + +Verification performed: + +- `uv run pytest tests/test_version_gate.py tests/test_capabilities.py tests/test_schemas.py tests/test_manifest.py -q` +- Result: `46 passed` + +Not performed: + +- No live DB test run against `PARQCAST_TEST_DB` +- No Odoo addon install/upgrade test + +## Findings + +### 1. Receiver still hardcodes v19 ingesters, so v18 inbound writeback can dispatch through the wrong actor + +Severity: high + +The receiver loads ingesters from `ALL_INGESTERS`, and that map is still pinned to v19 classes only. The code comment explicitly says it is kept v19-keyed, and `Receiver.run()` does not resolve the active Odoo major before choosing an ingester class. [packages/parqcast-ingesters/src/parqcast/ingesters/__init__.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-ingesters/src/parqcast/ingesters/__init__.py:15) [packages/parqcast-ingesters/src/parqcast/receiver/__init__.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-ingesters/src/parqcast/receiver/__init__.py:55) + +This is no longer safe after adding v18. The concrete proof point is purchase-order writeback: `PurchaseActorV18` writes `product_uom`, while `PurchaseActorV19` writes `product_uom_id`. If the receiver is pointed at an Odoo 18 database, the current dispatch path will still instantiate the v19 actor for `"PO"` decisions. [packages/parqcast-ingesters/src/parqcast/ingesters/v18/purchase_actor.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-ingesters/src/parqcast/ingesters/v18/purchase_actor.py:53) [packages/parqcast-ingesters/src/parqcast/ingesters/v19/purchase_actor.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-ingesters/src/parqcast/ingesters/v19/purchase_actor.py:53) + +Recommended direction: + +- Make the receiver version-aware by resolving the current major from `env.cr` and building its dispatch table from `REGISTRY[version].ingesters`. +- If cross-version manifests are expected to be replayed elsewhere, include the source Odoo major in the receiver contract and validate it before dispatch. + +### 2. Automatic cleanup bypasses the configured retention policy and can orphan attachment payloads + +Severity: high + +The addon UI exposes `parqcast_cleanup_days`, and the model-level cleanup action deletes runs older than that cutoff. [packages/parqcast/models/parqcast_settings.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/models/parqcast_settings.py:66) [packages/parqcast/models/parqcast_run.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/models/parqcast_run.py:76) + +But the actual automatic path in the orchestrator does something different: every successful run calls the core SQL helper `ExportRun.cleanup_old()` with its default `keep_last=3`, so retention is really "keep the last 3 done runs", not "delete after N days". [packages/parqcast-collectors/src/parqcast/orchestrator/__init__.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-collectors/src/parqcast/orchestrator/__init__.py:376) [packages/parqcast-core/src/parqcast/core/tracking.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-core/src/parqcast/core/tracking.py:126) + +That would already be a policy mismatch, but it gets worse with the default attachment transport. Export files are stored as `ir.attachment` rows tied to `parqcast.run` via `res_model` / `res_id`. The SQL cleanup only deletes from `parqcast_export_run`; it does not touch attachments. That means successful exports can leave behind orphaned attachment rows and binary payloads once their run rows are deleted. [packages/parqcast/models/transport_attachment.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/models/transport_attachment.py:36) + +Recommended direction: + +- Pick one retention model and enforce it in one place. +- If attachment transport stays the default, cleanup must go through an Odoo-aware path that also deletes related attachments. + +### 3. Manifest metadata falls back to `odoo_version="17.0"` instead of the actual exporting database version + +Severity: medium + +`build_manifest()` defaults the top-level `odoo_version` to `"17.0"`. The orchestrator's `_finalize()` call does not pass `caps.odoo_version`, even though it re-probes capabilities immediately before building the manifest. The exported manifest therefore carries the wrong top-level Odoo version metadata for both v18 and v19 runs. [packages/parqcast-core/src/parqcast/core/manifest.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-core/src/parqcast/core/manifest.py:14) [packages/parqcast-collectors/src/parqcast/orchestrator/__init__.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-collectors/src/parqcast/orchestrator/__init__.py:354) + +There is a partial workaround because `manifest["capabilities"]` is filled afterwards, but any downstream reader using the manifest's top-level `odoo_version` field will see incorrect data. + +Recommended direction: + +- Pass the actual `caps.odoo_version` into `build_manifest()`. +- Remove the misleading default, or make it explicit that callers must provide it. + +### 4. Finalized manifests lose the actual chunk failure reason + +Severity: medium + +Chunk errors are stored with a real `error_message` when collection or upload fails. [packages/parqcast-core/src/parqcast/core/tracking.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-core/src/parqcast/core/tracking.py:283) + +During finalization, however, the manifest error list is built as `f"{c.collector}: {c.state}"`. Since `c.state` is always just `"error"` for failed chunks, operators lose the actual exception text in the exported manifest. [packages/parqcast-collectors/src/parqcast/orchestrator/__init__.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-collectors/src/parqcast/orchestrator/__init__.py:352) + +Impact: + +- Post-mortem debugging from `manifest.json` becomes much harder. +- Remote systems consuming manifests cannot distinguish schema failures, permission failures, transport failures, or data-quality failures. + +Recommended direction: + +- Include `error_message` in the finalized manifest. +- Consider separating collection errors from upload errors if downstream automation needs to react differently. + +### 5. The live-DB test file is only partially version-parameterized + +Severity: medium + +Most of `tests/test_live_db.py` is bundle-aware: `_probe_and_build()` resolves `REGISTRY[TEST_ODOO_VERSION]` and calls that bundle's `probe_capabilities`. [tests/test_live_db.py](/Users/thatoleg/datastructsro/parqcast/tests/test_live_db.py:36) + +But `test_probe_capabilities()` still imports and calls `probe_v19()` directly. So a run with `PARQCAST_TEST_ODOO_VERSION=18` does not fully exercise the v18-specific probe entrypoint even though the surrounding test module suggests it does. [tests/test_live_db.py](/Users/thatoleg/datastructsro/parqcast/tests/test_live_db.py:27) [tests/test_live_db.py](/Users/thatoleg/datastructsro/parqcast/tests/test_live_db.py:115) + +This is a test gap rather than a runtime bug, but it matters because the branch's main risk surface is exactly the version split. + +Recommended direction: + +- Replace the direct `probe_v19()` call with `bundle.probe_capabilities(...)` or parametrize the test over both wrappers. + +## Watch List + +### Bundle bootstrap makes the install-time gate weaker than it looks + +`assert_supported()` accepts a major if it exists in `REGISTRY`, and `REGISTRY` is pre-bootstrapped with empty v18/v19 bundles before the version-specific subpackages are imported. The addon `_register_hook()` therefore proves only that the major is listed, not that the corresponding bundle was imported successfully or is complete. [packages/parqcast-core/src/parqcast/core/registry.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-core/src/parqcast/core/registry.py:49) [packages/parqcast-core/src/parqcast/core/version_gate.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-core/src/parqcast/core/version_gate.py:41) [packages/parqcast/models/parqcast_version_gate.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/models/parqcast_version_gate.py:19) + +That is acceptable during staged migrations, but it is a weak fit for a production install/upgrade guard. A broken import in a version bundle would still let installation proceed and only fail later when the cron path hits `_resolve_bundle()`. + +### Keyset pagination assumes collectors do not already define their own `ORDER BY` + +`BaseCollector.collect()` appends `ORDER BY {pk_column}` whenever keyset bounds are applied. That is fine for the current collector set, but it will break as soon as a future collector ships SQL with its own trailing `ORDER BY`. [packages/parqcast-collectors/src/parqcast/collectors/base.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-collectors/src/parqcast/collectors/base.py:101) + +This is not a current regression, but it is a good invariance to encode in tests if more complex collectors get added. diff --git a/docs/settings-improvements.md b/docs/settings-improvements.md new file mode 100644 index 0000000..f4ccf64 --- /dev/null +++ b/docs/settings-improvements.md @@ -0,0 +1,153 @@ +# Settings Improvements + +Date: 2026-04-27 + +Scope reviewed: + +- Settings model and config parameters +- Settings form UI +- Cron defaults +- Retention and attachment behavior + +## Recommended Changes + +### 1. Make retention settings match runtime behavior + +Current issue: + +- The settings UI exposes `parqcast_cleanup_days` as a days-based retention policy. +- The automatic runtime path instead deletes all but the last 3 successful runs. + +Relevant files: + +- [packages/parqcast/models/parqcast_settings.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/models/parqcast_settings.py:66) +- [packages/parqcast/models/parqcast_run.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/models/parqcast_run.py:76) +- [packages/parqcast-core/src/parqcast/core/tracking.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-core/src/parqcast/core/tracking.py:126) +- [packages/parqcast-collectors/src/parqcast/orchestrator/__init__.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast-collectors/src/parqcast/orchestrator/__init__.py:376) + +Suggestion: + +- Pick one retention model and enforce it consistently. +- If the intended policy is days-based, wire `parqcast_cleanup_days` into the orchestrator cleanup path. +- If the intended policy is count-based, change the settings label and help text accordingly. + +### 2. Make attachment cleanup explicit + +Current issue: + +- Attachment transport stores exports as `ir.attachment` rows linked to `parqcast.run`. +- SQL cleanup of old runs does not automatically clean up those attachments. + +Relevant file: + +- [packages/parqcast/models/transport_attachment.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/models/transport_attachment.py:36) + +Suggestion: + +- When deleting old runs, also delete their attachments through an Odoo-aware cleanup path. +- Add an optional “purge uploaded attachments after N days” policy if attachment transport remains the default. +- Show a short storage warning in the settings UI when `attachment` transport is selected. + +### 3. Fix the schedule wording + +Current issue: + +- The UI says `Once daily (midnight UTC)`. +- The implementation only switches the cron interval to `1 day`; it does not actually normalize `nextcall` to midnight UTC. + +Relevant files: + +- [packages/parqcast/models/parqcast_settings.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/models/parqcast_settings.py:84) +- [packages/parqcast/data/ir_cron_data.xml](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/data/ir_cron_data.xml:4) + +Suggestion: + +- Either actually schedule the cron at midnight UTC, or rename the option to something accurate like `Every day`. +- If operators care about exact timing, expose a run time and timezone instead of hardcoding wording. + +### 4. Add a status block to settings + +Current issue: + +- The settings page is mostly input-only. +- Operators cannot quickly see whether Parqcast is healthy without leaving the page. + +Relevant file: + +- [packages/parqcast/views/parqcast_settings_views.xml](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/views/parqcast_settings_views.xml:8) + +Suggestion: + +- Add a read-only status section showing detected Odoo major, version-gate status, selected company, transport type, next cron run, last run state, and last run error when present. + +This would make misconfiguration much easier to diagnose. + +### 5. Add transport validation actions + +Current issue: + +- The form hides and shows transport-specific fields, but it does not validate the configured destination proactively. + +Relevant files: + +- [packages/parqcast/views/parqcast_settings_views.xml](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/views/parqcast_settings_views.xml:51) +- [packages/parqcast/models/parqcast_cron.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/models/parqcast_cron.py:16) + +Suggestion: + +- Add `Test connection` / `Test write` actions for local filesystem, HTTP server, S3, and attachment transport. + +- Validate required fields server-side before saving or before running the first export. +- Return actionable errors rather than waiting for cron to fail. + +### 6. Improve manual run feedback + +Current issue: + +- `Run Export Now` always returns a generic success notification. +- It does not reflect whether the tick planned, collected, uploaded, paused, or failed. + +Relevant file: + +- [packages/parqcast/models/parqcast_settings.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/models/parqcast_settings.py:115) + +Suggestion: + +- Include the returned state in the notification. +- Include the run UUID when available. +- Surface immediate errors in the notification if the tick fails. + +### 7. Add guardrails around time budget + +Current issue: + +- `parqcast_time_budget` is free-form. +- Very low values can make the system appear broken because planning or upload may never make visible progress. + +Relevant files: + +- [packages/parqcast/models/parqcast_settings.py](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/models/parqcast_settings.py:72) +- [packages/parqcast/views/parqcast_settings_views.xml](/Users/thatoleg/datastructsro/parqcast/packages/parqcast/views/parqcast_settings_views.xml:32) + +Suggestion: + +- Add sane minimum and recommended values in help text or validation. +- Consider presets such as `safe`, `balanced`, and `aggressive` if operators are not expected to tune raw seconds. + +## Suggested Priority + +Implement first: + +- retention consistency +- attachment cleanup +- schedule wording fix + +Implement second: + +- status block +- transport validation +- better manual-run feedback + +Implement later: + +- time-budget guardrails diff --git a/packages/parqcast-collectors/src/parqcast/collectors/company.py b/packages/parqcast-collectors/src/parqcast/collectors/company.py index 0140096..218d74b 100644 --- a/packages/parqcast-collectors/src/parqcast/collectors/company.py +++ b/packages/parqcast-collectors/src/parqcast/collectors/company.py @@ -14,7 +14,8 @@ def get_sql(self): rc.id, rc.name, rc.currency_id, rc.parent_id, COALESCE(rc.security_lead, 0), - rc.active + rc.active, + rc.country_id FROM res_company rc """, None, diff --git a/packages/parqcast-collectors/src/parqcast/orchestrator/__init__.py b/packages/parqcast-collectors/src/parqcast/orchestrator/__init__.py index 25edc03..e2bb3d5 100644 --- a/packages/parqcast-collectors/src/parqcast/orchestrator/__init__.py +++ b/packages/parqcast-collectors/src/parqcast/orchestrator/__init__.py @@ -366,6 +366,7 @@ def _finalize(self, cr: ReadCursor, run: ExportRun, prefix: str, t0: float) -> J total_duration=round(time.monotonic() - t0, 3), errors=errors, warnings=[], + started_at=run.started_at, ) manifest["capabilities"] = caps.summary() manifest["run_uuid"] = run.run_uuid diff --git a/packages/parqcast-core/src/parqcast/core/manifest.py b/packages/parqcast-core/src/parqcast/core/manifest.py index 91f4246..22cab03 100644 --- a/packages/parqcast-core/src/parqcast/core/manifest.py +++ b/packages/parqcast-core/src/parqcast/core/manifest.py @@ -20,11 +20,15 @@ def build_manifest( errors: list[str] | None = None, warnings: list[str] | None = None, total_duration: float = 0.0, + started_at: datetime | None = None, ) -> JsonDict: + finished_at = datetime.now(UTC) return { "version": parqcast_version, - "schema_version": datetime.now(UTC).strftime("%Y-%m-%d"), - "timestamp": datetime.now(UTC).isoformat(), + "schema_version": finished_at.strftime("%Y-%m-%d"), + "timestamp": finished_at.isoformat(), + "started_at": started_at.isoformat() if started_at else None, + "finished_at": finished_at.isoformat(), "company": company, "company_id": company_id, "odoo_version": odoo_version, diff --git a/packages/parqcast-core/src/parqcast/core/tracking.py b/packages/parqcast-core/src/parqcast/core/tracking.py index c7a775b..8aaaa86 100644 --- a/packages/parqcast-core/src/parqcast/core/tracking.py +++ b/packages/parqcast-core/src/parqcast/core/tracking.py @@ -24,11 +24,19 @@ class ExportRun: STATES = ("pending", "collecting", "uploading", "done", "error") - def __init__(self, id: int, run_uuid: str, state: str, company_id: int | None = None): + def __init__( + self, + id: int, + run_uuid: str, + state: str, + company_id: int | None = None, + started_at: datetime | None = None, + ): self.id = id self.run_uuid = run_uuid self.state = state self.company_id = company_id + self.started_at = started_at @classmethod def ensure_table(cls, cr: ReadCursor): @@ -54,28 +62,29 @@ def ensure_table(cls, cr: ReadCursor): @classmethod def create(cls, cr: ReadCursor, company_id: int | None = None, company_name: str = "") -> ExportRun: run_uuid = str(_uuid.uuid4()) + started_at = datetime.now(UTC) cr.execute( """ INSERT INTO parqcast_export_run (run_uuid, state, company_id, company_name, started_at) VALUES (%s, 'pending', %s, %s, %s) RETURNING id """, - (run_uuid, company_id, company_name, datetime.now(UTC)), + (run_uuid, company_id, company_name, started_at), ) run_id = fetch_one(cr)[0] - return cls(id=run_id, run_uuid=run_uuid, state="pending", company_id=company_id) + return cls(id=run_id, run_uuid=run_uuid, state="pending", company_id=company_id, started_at=started_at) @classmethod def find_active(cls, cr: ReadCursor) -> ExportRun | None: cr.execute(""" - SELECT id, run_uuid, state, company_id + SELECT id, run_uuid, state, company_id, started_at FROM parqcast_export_run WHERE state NOT IN ('done', 'error') ORDER BY id DESC LIMIT 1 """) row = fetch_one_or_none(cr) if row: - return cls(id=row[0], run_uuid=row[1], state=row[2], company_id=row[3]) + return cls(id=row[0], run_uuid=row[1], state=row[2], company_id=row[3], started_at=row[4]) return None def set_state(self, cr: ReadCursor, state: str): diff --git a/packages/parqcast-core/src/parqcast/schemas/inbound.py b/packages/parqcast-core/src/parqcast/schemas/inbound.py index 4f20a0b..20cd3b0 100644 --- a/packages/parqcast-core/src/parqcast/schemas/inbound.py +++ b/packages/parqcast-core/src/parqcast/schemas/inbound.py @@ -4,6 +4,7 @@ OdooCode, OdooDatetime, OdooId, + OdooInteger, OdooName, OdooQuantity, OdooState, @@ -13,6 +14,8 @@ DECISIONS_SCHEMA = pa.schema( [ ("decision_id", OdooCode), + ("run_uuid", OdooText), + ("decision_timestamp", OdooDatetime), ("decision_type", OdooState), ("status", OdooState), ("item_name", OdooName), @@ -35,5 +38,6 @@ ("remark", OdooText), ("min_quantity", OdooQuantity), ("max_quantity", OdooQuantity), + ("delay", OdooInteger), ] ) diff --git a/packages/parqcast-core/src/parqcast/schemas/odoo_types.py b/packages/parqcast-core/src/parqcast/schemas/odoo_types.py index 1278a2e..c9c1c9f 100644 --- a/packages/parqcast-core/src/parqcast/schemas/odoo_types.py +++ b/packages/parqcast-core/src/parqcast/schemas/odoo_types.py @@ -19,6 +19,10 @@ # --- Odoo Integer fields --- +OdooInteger = pa.int64() +"""Generic integer field (e.g. delay, sequence lengths). +Maps to: fields.Integer""" + OdooId = pa.int64() """Many2one FK, Integer PK, or any Odoo integer field. Maps to: fields.Many2one, fields.Integer, fields.Id""" diff --git a/packages/parqcast-core/src/parqcast/schemas/outbound.py b/packages/parqcast-core/src/parqcast/schemas/outbound.py index 40dd18f..cb1f75c 100644 --- a/packages/parqcast-core/src/parqcast/schemas/outbound.py +++ b/packages/parqcast-core/src/parqcast/schemas/outbound.py @@ -436,6 +436,7 @@ ("_odoo_parent_id", OdooId), ("security_lead", OdooFloat), ("active", OdooBoolean), + ("_odoo_country_id", OdooId), ] ) diff --git a/packages/parqcast-ingesters/src/parqcast/ingesters/v18/__init__.py b/packages/parqcast-ingesters/src/parqcast/ingesters/v18/__init__.py index d50ae83..e867e44 100644 --- a/packages/parqcast-ingesters/src/parqcast/ingesters/v18/__init__.py +++ b/packages/parqcast-ingesters/src/parqcast/ingesters/v18/__init__.py @@ -13,6 +13,7 @@ from .production_actor import ProductionActorV18 from .purchase_actor import PurchaseActorV18 from .reschedule_actor import RescheduleActorV18 +from .supplierinfo_actor import SupplierinfoActorV18 append_to_bundle( "18", @@ -22,6 +23,7 @@ DistributionActorV18, RescheduleActorV18, OrderpointActorV18, + SupplierinfoActorV18, ), ) @@ -31,4 +33,5 @@ "ProductionActorV18", "PurchaseActorV18", "RescheduleActorV18", + "SupplierinfoActorV18", ] diff --git a/packages/parqcast-ingesters/src/parqcast/ingesters/v18/supplierinfo_actor.py b/packages/parqcast-ingesters/src/parqcast/ingesters/v18/supplierinfo_actor.py new file mode 100644 index 0000000..099522d --- /dev/null +++ b/packages/parqcast-ingesters/src/parqcast/ingesters/v18/supplierinfo_actor.py @@ -0,0 +1,52 @@ +import pyarrow as pa + +from parqcast.core.protocols import OdooEnvironment +from parqcast.core.version import V18 + +from ..base import BaseIngester, IngestResult + + +class SupplierinfoActorV18(BaseIngester[V18]): + """Update vendor lead times (delay) based on ML empirical calculations.""" + + decision_type = "LEAD_TIME" + + def apply(self, decisions: pa.Table, env: OdooEnvironment) -> IngestResult: + df = decisions.to_pydict() + updated = 0 + errors = 0 + + for i in range(decisions.num_rows): + supplier_id = df["_odoo_supplier_id"][i] + product_id = df["_odoo_product_id"][i] + delay = df["delay"][i] + + if not supplier_id or not product_id or delay is None: + errors += 1 + continue + + product = env["product.product"].browse(product_id) + if not product.exists(): + errors += 1 + continue + + # Find the exact supplier pricelist link + supplier_info = env["product.supplierinfo"].search( + [ + ("partner_id", "=", supplier_id), + ("product_tmpl_id", "=", product.product_tmpl_id.id), + ], + limit=1, + ) + + if supplier_info: + supplier_info.write({"delay": delay}) + updated += 1 + else: + errors += 1 # We don't create new supplierinfo here, only update existing + + return IngestResult(updated=updated, errors=errors) + + def cleanup_previous(self, env: OdooEnvironment, company_id: int) -> int: + # We don't cleanup past empirical lead times, they are a permanent state update + return 0 diff --git a/packages/parqcast-ingesters/src/parqcast/ingesters/v19/__init__.py b/packages/parqcast-ingesters/src/parqcast/ingesters/v19/__init__.py index 5333890..0520b5d 100644 --- a/packages/parqcast-ingesters/src/parqcast/ingesters/v19/__init__.py +++ b/packages/parqcast-ingesters/src/parqcast/ingesters/v19/__init__.py @@ -13,6 +13,7 @@ from .production_actor import ProductionActorV19 from .purchase_actor import PurchaseActorV19 from .reschedule_actor import RescheduleActorV19 +from .supplierinfo_actor import SupplierinfoActorV19 append_to_bundle( "19", @@ -22,6 +23,7 @@ DistributionActorV19, RescheduleActorV19, OrderpointActorV19, + SupplierinfoActorV19, ), ) @@ -31,4 +33,5 @@ "ProductionActorV19", "PurchaseActorV19", "RescheduleActorV19", + "SupplierinfoActorV19", ] diff --git a/packages/parqcast-ingesters/src/parqcast/ingesters/v19/supplierinfo_actor.py b/packages/parqcast-ingesters/src/parqcast/ingesters/v19/supplierinfo_actor.py new file mode 100644 index 0000000..d3723f3 --- /dev/null +++ b/packages/parqcast-ingesters/src/parqcast/ingesters/v19/supplierinfo_actor.py @@ -0,0 +1,52 @@ +import pyarrow as pa + +from parqcast.core.protocols import OdooEnvironment +from parqcast.core.version import V19 + +from ..base import BaseIngester, IngestResult + + +class SupplierinfoActorV19(BaseIngester[V19]): + """Update vendor lead times (delay) based on ML empirical calculations.""" + + decision_type = "LEAD_TIME" + + def apply(self, decisions: pa.Table, env: OdooEnvironment) -> IngestResult: + df = decisions.to_pydict() + updated = 0 + errors = 0 + + for i in range(decisions.num_rows): + supplier_id = df["_odoo_supplier_id"][i] + product_id = df["_odoo_product_id"][i] + delay = df["delay"][i] + + if not supplier_id or not product_id or delay is None: + errors += 1 + continue + + product = env["product.product"].browse(product_id) + if not product.exists(): + errors += 1 + continue + + # Find the exact supplier pricelist link + supplier_info = env["product.supplierinfo"].search( + [ + ("partner_id", "=", supplier_id), + ("product_tmpl_id", "=", product.product_tmpl_id.id), + ], + limit=1, + ) + + if supplier_info: + supplier_info.write({"delay": delay}) + updated += 1 + else: + errors += 1 # We don't create new supplierinfo here, only update existing + + return IngestResult(updated=updated, errors=errors) + + def cleanup_previous(self, env: OdooEnvironment, company_id: int) -> int: + # We don't cleanup past empirical lead times, they are a permanent state update + return 0