From e21228e491354c44b1ae73f45f35405923507c48 Mon Sep 17 00:00:00 2001 From: Victor Garcia Date: Tue, 28 Jul 2026 11:48:58 -0600 Subject: [PATCH 01/15] chore(security): protect public repository data --- .gitignore | 16 +++ .npmignore | 16 +++ PUBLIC_DATA_POLICY.md | 46 +++++++ README.md | 13 +- package.json | 4 + scripts/check-public-data.py | 113 ++++++++++++++++++ .../references/FIXTURE_PROVENANCE.md | 12 ++ skills/steel-nest/references/example_job.json | 45 +++---- .../steel-nest/references/job_template.json | 9 +- skills/steel-rfq/SKILL.md | 40 +++---- .../assets/company-profile.example.json | 8 +- .../references/takeoff-procedures.md | 4 +- .../steel-takeoff/scripts/calculate-weight.sh | 9 +- tests/test_public_data_policy.py | 23 ++++ 14 files changed, 289 insertions(+), 69 deletions(-) create mode 100644 .npmignore create mode 100644 PUBLIC_DATA_POLICY.md create mode 100644 scripts/check-public-data.py create mode 100644 skills/steel-nest/references/FIXTURE_PROVENANCE.md create mode 100644 tests/test_public_data_policy.py diff --git a/.gitignore b/.gitignore index 0c62d0c..0193381 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,19 @@ outputs/ .a5c/ __pycache__/ *.pyc + +# Local company, project, and generated business data +.pi-steel/ +company-profile.json +**/company-profile.json +private/ +local-data/ +customer-data/ +vendor-data/ +*.local.json +*.private.json +*.xlsx +*.xls +*.pdf +*.dxf +*.png diff --git a/.npmignore b/.npmignore new file mode 100644 index 0000000..4db4880 --- /dev/null +++ b/.npmignore @@ -0,0 +1,16 @@ +**/__pycache__/ +**/*.pyc +**/company-profile.json +.pi-steel/ +private/ +local-data/ +customer-data/ +vendor-data/ +outputs/ +*.local.json +*.private.json +*.xlsx +*.xls +*.pdf +*.dxf +*.png diff --git a/PUBLIC_DATA_POLICY.md b/PUBLIC_DATA_POLICY.md new file mode 100644 index 0000000..3f1ca6b --- /dev/null +++ b/PUBLIC_DATA_POLICY.md @@ -0,0 +1,46 @@ +# Public Data Policy + +This repository is public. Source code, documentation, tests, fixtures, generated +goldens, commit metadata, issue text, and release artifacts must be safe for public +distribution. + +## Allowed + +- Clearly labeled synthetic projects, customers, vendors, people, and identifiers. +- Published standards data whose source, edition, license, and redistribution rights + are documented. +- Invented dimensions and quantities that do not reproduce a private project. +- Placeholder configuration such as `Example Fabricator` and `Example City, ST`. +- Product behavior and generic steel-domain terminology. + +## Prohibited + +- Private company names or identifiers other than the public StructuPath product + identity. +- Customer, vendor, employee, subcontractor, or project names and contact details. +- Real job numbers, drawing references, addresses, schedules, quotes, bids, rates, + margins, payroll, costs, terms, or contract language. +- Production drawings, takeoffs, BOMs, nests, RFQs, quotes, reports, screenshots, or + generated artifacts, including "anonymized" copies derived from them. +- Company profiles, logos, signatures, credentials, tokens, local absolute paths, or + exported cloud files. +- Facts that link this product to a private operating company, facility, customer, or + internal workflow unless separately approved for publication. + +## Fixture Rules + +1. Create fixtures from scratch; do not sanitize production files for public use. +2. Prefix project-like identifiers with `SYNTHETIC-` or `EXAMPLE-`. +3. Use `Example Customer`, `Example Vendor`, `Example Fabricator`, and + `Example City, ST` unless a test requires another obviously fictional value. +4. Omit prices and commercial terms unless the test specifically requires them. When + required, label values as synthetic in the same fixture. +5. Keep private inputs and generated outputs outside the repository in ignored + directories such as `private/`, `local-data/`, `customer-data/`, or `outputs/`. +6. Run `python3 scripts/check-public-data.py` before every commit and release. +7. Put private names and identifiers, one per line, in the ignored local file + `.pi-steel/private-terms.txt`; the scanner checks them without committing the + denylist or echoing matched text. + +Git history is public too. Removing a value from the current tree does not remove it +from prior commits; history cleanup requires an explicit coordinated rewrite. diff --git a/README.md b/README.md index b4c360a..ca78456 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # pi-steel -Structural steel estimating skills for the [Pi coding agent](https://pi.dev) — built by working steel estimators, not by people guessing what a takeoff is. +Structural steel estimating skills for the [Pi coding agent](https://pi.dev). -By [StructuPath](https://structupath.ai), from the team behind a production structural-steel fabrication shop in Denver, CO. +By [StructuPath](https://structupath.ai). ## Install @@ -46,7 +46,14 @@ Turns a steel estimate/takeoff spreadsheet into a standardized vendor RFQ (.xlsx The three skills chain into a full estimating pipeline: **takeoff → nest → RFQ**. -One-time setup: copy `skills/steel-rfq/assets/company-profile.example.json` to `company-profile.json` and put in your company name, city, and payment terms. The skill will ask and offer to save it if you skip this. +One-time setup: copy `skills/steel-rfq/assets/company-profile.example.json` to the +ignored path `.pi-steel/company-profile.json` in your project and enter approved +company and commercial information there. Never add the completed profile to this +repository. + +Keep company profiles, customer files, vendor information, live pricing, and generated +artifacts outside this repository. Public examples are synthetic and must follow +[`PUBLIC_DATA_POLICY.md`](PUBLIC_DATA_POLICY.md). > "Send this takeoff out for pricing" > "Generate an RFQ from this estimate" diff --git a/package.json b/package.json index 5737576..8f16237 100644 --- a/package.json +++ b/package.json @@ -37,7 +37,11 @@ }, "files": [ "skills", + "!skills/**/__pycache__", + "!skills/**/*.pyc", + "!skills/**/company-profile.json", "README.md", + "PUBLIC_DATA_POLICY.md", "LICENSE" ] } diff --git a/scripts/check-public-data.py b/scripts/check-public-data.py new file mode 100644 index 0000000..00267eb --- /dev/null +++ b/scripts/check-public-data.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Fail when public repository files contain known private-data indicators.""" + +from __future__ import annotations + +import re +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +ALLOWED_SUFFIXES = { + ".csv", + ".json", + ".md", + ".py", + ".sh", + ".toml", + ".txt", + ".yaml", + ".yml", +} +ALLOWED_NAMES = {".gitignore", "LICENSE", "package.json"} +SKIP_PARTS = {".git", ".a5c", "__pycache__", "node_modules"} +SKIP_FILES = {Path("scripts/check-public-data.py")} +FORBIDDEN_BINARY_SUFFIXES = { + ".doc", + ".docx", + ".dxf", + ".jpeg", + ".jpg", + ".pdf", + ".png", + ".xls", + ".xlsx", +} +PATTERNS = { + "private operating-company claim": re.compile( + r"team behind (?:a|the) production structural[- ]steel", re.I + ), + "local absolute path": re.compile(r"(?:/Users/|/home/|[A-Z]:\\\\Users\\\\)"), + "realistic sales-order identifier": re.compile(r"\bSO-\d{3,}\b", re.I), + "current-market pricing claim": re.compile(r"\bcurrent (?:market )?rates?\b", re.I), + "contact email": re.compile(r"\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b", re.I), + "phone number": re.compile( + r"(? list[Path]: + result = subprocess.run( + ["git", "ls-files", "--cached", "--others", "--exclude-standard"], + cwd=ROOT, + check=True, + capture_output=True, + text=True, + ) + return [ROOT / line for line in result.stdout.splitlines() if line] + + +def main() -> int: + findings: list[str] = [] + patterns = dict(PATTERNS) + private_terms = ROOT / ".pi-steel" / "private-terms.txt" + if private_terms.is_file(): + terms = [ + line.strip() + for line in private_terms.read_text(encoding="utf-8").splitlines() + if line.strip() and not line.lstrip().startswith("#") + ] + if terms: + patterns["private local denylist term"] = re.compile( + "|".join(re.escape(term) for term in sorted(terms, key=len, reverse=True)), + re.I, + ) + + for path in tracked_files(): + relative = path.relative_to(ROOT) + if ( + relative in SKIP_FILES + or any(part in SKIP_PARTS for part in relative.parts) + or not path.is_file() + ): + continue + suffix = path.suffix.lower() + if suffix in FORBIDDEN_BINARY_SUFFIXES: + findings.append(f"{relative}: public repository must not contain {suffix} artifacts") + continue + if suffix not in ALLOWED_SUFFIXES and path.name not in ALLOWED_NAMES: + continue + text = path.read_text(encoding="utf-8", errors="replace") + for label, pattern in patterns.items(): + for match in pattern.finditer(text): + line = text.count("\n", 0, match.start()) + 1 + findings.append(f"{relative}:{line}: {label}") + + if findings: + print("Public-data check failed:", file=sys.stderr) + for finding in findings: + print(f" - {finding}", file=sys.stderr) + return 1 + + print("Public-data check passed.") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/steel-nest/references/FIXTURE_PROVENANCE.md b/skills/steel-nest/references/FIXTURE_PROVENANCE.md new file mode 100644 index 0000000..4a5b53b --- /dev/null +++ b/skills/steel-nest/references/FIXTURE_PROVENANCE.md @@ -0,0 +1,12 @@ +# Fixture Provenance + +`example_job.json` and `job_template.json` are synthetic public examples created +from scratch for pi-steel documentation and tests. They are not copied, transformed, +rounded, renamed, or anonymized from a company, customer, vendor, bid, drawing, +takeoff, inventory record, or production job. + +- Creator: StructuPath pi-steel maintainers +- Creation method: deliberately invented geometry and identifiers +- Public-data review: 2026-07-28 +- Commercial data: none +- Private source artifacts: none diff --git a/skills/steel-nest/references/example_job.json b/skills/steel-nest/references/example_job.json index 2ed9ef9..01072dc 100644 --- a/skills/steel-nest/references/example_job.json +++ b/skills/steel-nest/references/example_job.json @@ -1,6 +1,6 @@ { - "job_name": "SO-4471 Conveyor Frame Brackets", - "customer": "Rocky Mtn Aggregate", + "job_name": "SYNTHETIC-DEMO-001 Generated Training Geometry", + "customer": "Example Customer", "settings": { "kerf_in": 0.06, "part_gap_in": 0.25, @@ -10,38 +10,31 @@ }, "stock": [ { - "name": "A36 Plate 1/2\"", - "width": 96, - "height": 48, - "thickness": 0.5, - "qty": 6, - "cost_per_lb": 0.92 + "name": "Synthetic Plate", + "width": 60, + "height": 30, + "thickness": 0.375, + "qty": 3 } ], "parts": [ - { "name": "Base Plate BP1", "width": 14, "height": 10, "qty": 6, "shape": "rect", + { "name": "Training Rect A", "width": 11, "height": 7, "qty": 3, "shape": "rect", "holes": [ - { "dia": 0.875, "x": 2, "y": 2 }, - { "dia": 0.875, "x": 12, "y": 2 }, - { "dia": 0.875, "x": 2, "y": 8 }, - { "dia": 0.875, "x": 12, "y": 8 } + { "dia": 0.75, "x": 2, "y": 2 }, + { "dia": 0.75, "x": 9, "y": 5 } ] }, - { "name": "Column Cap CC1", "width": 16, "height": 16, "qty": 4, "shape": "rect", + { "name": "Training Rect B", "width": 13, "height": 9, "qty": 2, "shape": "rect", "holes": [ - { "dia": 1.0625, "x": 3, "y": 3 }, - { "dia": 1.0625, "x": 13, "y": 3 }, - { "dia": 1.0625, "x": 3, "y": 13 }, - { "dia": 1.0625, "x": 13, "y": 13 } + { "dia": 0.625, "x": 3, "y": 3 }, + { "dia": 0.625, "x": 10, "y": 6 } ] }, - { "name": "Stiffener ST1", "width": 9, "height": 6, "qty": 20, "shape": "rect" }, - { "name": "Gusset G1", "width": 12, "height": 12, "qty": 12, "shape": "irregular", "area": 72 }, - { "name": "Flat Bar FB1", "width": 30, "height": 4, "qty": 10, "shape": "rect" }, - { "name": "Cover CV1", "width": 24, "height": 18, "qty": 3, "shape": "rect", + { "name": "Training Irregular C", "width": 9, "height": 8, "qty": 4, "shape": "irregular", "area": 28 }, + { "name": "Training Rect D", "width": 17, "height": 5, "qty": 5, "shape": "rect" }, + { "name": "Training Rect E", "width": 19, "height": 11, "qty": 2, "shape": "rect", "holes": [ - { "w": 6, "h": 4, "x": 12, "y": 9 } + { "w": 5, "h": 3, "x": 9.5, "y": 5.5 } ] }, - { "name": "Clip CL1", "width": 5, "height": 3, "qty": 24, "shape": "rect", - "holes": [ { "dia": 0.5625, "x": 2.5, "y": 1.5 } ] }, - { "name": "Web Plate WP1", "width": 40, "height": 12, "qty": 4, "shape": "rect" } + { "name": "Training Rect F", "width": 6, "height": 4, "qty": 7, "shape": "rect", + "holes": [ { "dia": 0.5, "x": 3, "y": 2 } ] } ] } diff --git a/skills/steel-nest/references/job_template.json b/skills/steel-nest/references/job_template.json index a2f6648..f9d2210 100644 --- a/skills/steel-nest/references/job_template.json +++ b/skills/steel-nest/references/job_template.json @@ -1,6 +1,6 @@ { - "job_name": "SO-XXXX Short description", - "customer": "Customer name", + "job_name": "SYNTHETIC-DEMO-XXX Short description", + "customer": "Example Customer", "_comment_settings": "Cut/gap in inches. kerf = torch cut width (plasma ~0.06, oxy ~0.1, laser ~0.02). part_gap = clearance between parts on top of kerf. edge_margin = keep-out from plate edge (clamp/grip zone). density lb/in^3: A36 steel = 0.2836, aluminum = 0.098, stainless 304 = 0.289.", "settings": { @@ -11,15 +11,14 @@ "density_lb_in3": 0.2836 }, - "_comment_stock": "One entry per plate size you have on hand. qty = how many sheets available (or set \"unlimited\": true to buy as many as needed). Price EITHER by cost_per_lb OR cost_per_sheet — cost_per_lb is preferred because it also values the scrap.", + "_comment_stock": "One entry per plate size you have on hand. qty = how many sheets available (or set \"unlimited\": true to buy as many as needed). Add either cost_per_lb or cost_per_sheet only from an approved project input; public templates intentionally contain no pricing.", "stock": [ { "name": "A36 Plate 1/2\"", "width": 96, "height": 48, "thickness": 0.5, - "qty": 4, - "cost_per_lb": 0.92 + "qty": 4 } ], diff --git a/skills/steel-rfq/SKILL.md b/skills/steel-rfq/SKILL.md index c695774..c26e028 100644 --- a/skills/steel-rfq/SKILL.md +++ b/skills/steel-rfq/SKILL.md @@ -13,18 +13,18 @@ The RFQ format was designed around how steel vendors actually work — materials ## Company Profile (required setup) -The RFQ carries the requesting company's identity. Load it from -`assets/company-profile.json` in this skill's directory (copy -`company-profile.example.json` and edit). If the file is missing, ask the -user for their company name, city/state, and payment terms before -generating, and offer to save the answers as `company-profile.json` for -next time. +The RFQ carries the requesting company's identity. Load it from an explicit +`PI_STEEL_CONFIG` path, the ignored project-local +`.pi-steel/company-profile.json`, or the platform user-config directory. Never save +runtime company data inside this installed skill or the public repository. If no +profile is available, ask the user for their company name, city/state, and approved +terms before generating. Profile fields: - `company_name` — appears in the header and terms & conditions -- `city_state` — appears in the header (e.g., "Denver, CO") -- `payment_terms` — default "Net 30 from date of delivery" -- `quote_validity_days` — default 30 +- `city_state` — appears in the header (e.g., "Example City, ST") +- `payment_terms` — required approved text; no shipped default +- `quote_validity_days` — optional approved value; no shipped default - `logo` — optional filename in `assets/` to embed top-left Never invent company details. If the profile is incomplete, ask. @@ -125,19 +125,15 @@ Below the totals (skip a row), add a reference section: - One row per material showing the nesting layout and expected drop from the estimate - This helps cross-check vendor stock lengths against the cutting plan -### Standard Terms & Conditions +### Terms & Conditions Below the nesting table (skip a row), add: - Header: "TERMS & CONDITIONS" — dark blue bold text -- Include these standard terms, each on its own row, substituting the company name and payment terms from the company profile: - -1. **Delivery**: All material to be delivered FOB jobsite unless otherwise agreed. Vendor to confirm freight costs separately. -2. **Mill Certifications**: Mill test reports (MTRs) required for all structural steel per AISC/AWS standards. Certs must accompany delivery. -3. **Payment Terms**: [payment_terms from profile] unless otherwise negotiated in writing. -4. **Material Standards**: All wide-flange shapes to meet ASTM A992. All plate and bar to meet grade specified on this RFQ (A572 Gr.50 or A36). -5. **Substitutions**: No substitutions without prior written approval from [Company Name]. If quoting alternate sizes, clearly note in the "Alternate Size" column. -6. **Quote Validity**: Quoted prices to remain firm for [quote_validity_days] days from date of quote unless otherwise stated. -7. **Inspection**: [Company Name] reserves the right to inspect material upon delivery and reject material not meeting specifications. -8. **Cancellation**: Orders may be cancelled without penalty if material has not shipped. Restocking fees, if any, to be stated in quote. +- Load terms only from the user's approved company profile or an explicitly supplied + project template. +- Do not ship, infer, or invent default commercial terms. Missing approved terms keep + the workbook in draft/review-required status. +- Public examples must use obvious placeholders and must not contain a real company's + payment, delivery, cancellation, inspection, substitution, or purchasing policy. ### Branding / Logo If the company profile names a logo file and it exists in the skill's `assets/` directory, insert it in cell A1 area (top-left) and adjust the header text to not overlap. Otherwise use the text header as described above. @@ -152,7 +148,7 @@ If the company profile names a logo file and it exists in the skill's `assets/` Output file: `[ProjectName]_RFQ_Material_List.xlsx` - Extract project name from the estimate file (look in "Project Info" sheet or the first rows of the takeoff) - Replace spaces with underscores -- Example: `Cherokee_Boys_RFQ_Material_List.xlsx` +- Example: `Synthetic_Demo_RFQ_Material_List.xlsx` ## Step-by-Step Workflow @@ -165,7 +161,7 @@ Output file: `[ProjectName]_RFQ_Material_List.xlsx` 7. Build the RFQ spreadsheet using openpyxl following the format above 8. Add formulas for totals and verify the SUM ranges cover exactly the data rows 9. Add nesting/drop reference from the estimate notes — or, if the `steel-nest` skill has been run for this job, read its `rfq_nesting.json` output straight into the table -10. Add standard terms & conditions with profile values substituted +10. Add only user-approved terms from the selected profile or project template 11. Insert logo if configured 12. Save the file, then run `python3 scripts/recalc.py ` so formula values are computed (openpyxl writes formulas but never calculates them) 13. Verify no formula errors and present the file to the user diff --git a/skills/steel-rfq/assets/company-profile.example.json b/skills/steel-rfq/assets/company-profile.example.json index a43cefd..55403dd 100644 --- a/skills/steel-rfq/assets/company-profile.example.json +++ b/skills/steel-rfq/assets/company-profile.example.json @@ -1,7 +1,7 @@ { - "company_name": "Acme Steel Fabrication", - "city_state": "Denver, CO", - "payment_terms": "Net 30 from date of delivery", - "quote_validity_days": 30, + "company_name": "Example Fabricator", + "city_state": "Example City, ST", + "payment_terms": "ENTER APPROVED PAYMENT TERMS", + "quote_validity_days": null, "logo": null } diff --git a/skills/steel-takeoff/references/takeoff-procedures.md b/skills/steel-takeoff/references/takeoff-procedures.md index 9daa578..26f2c83 100644 --- a/skills/steel-takeoff/references/takeoff-procedures.md +++ b/skills/steel-takeoff/references/takeoff-procedures.md @@ -2,7 +2,9 @@ ## Overview -A steel takeoff quantifies every piece of structural steel on a project to determine total tonnage for bidding. Accuracy directly impacts profit margin — a 5% error on a 200-ton job is 10 tons ($32,000+ at current rates). +A steel takeoff quantifies every piece of structural steel on a project to determine +total tonnage for bidding. Accuracy directly affects quantities, schedule, and cost; +this public guide intentionally contains no market rates or company pricing. ## Takeoff Order (recommended) diff --git a/skills/steel-takeoff/scripts/calculate-weight.sh b/skills/steel-takeoff/scripts/calculate-weight.sh index 5c4f666..913ed6c 100755 --- a/skills/steel-takeoff/scripts/calculate-weight.sh +++ b/skills/steel-takeoff/scripts/calculate-weight.sh @@ -10,7 +10,7 @@ # connection_pct - Connection allowance percentage (default: 12) # misc_pct - Misc steel allowance percentage (default: 5) # -# Output: Line-by-line breakdown + totals with cost estimate +# Output: Line-by-line breakdown + weight totals. Pricing is supplied separately. # ═══════════════════════════════════════════════════════════════════ set -euo pipefail @@ -137,12 +137,5 @@ print(f" {'─'*60}") print(f" {'GRAND TOTAL:':<30} {grand_total:>12,.0f} lb ({grand_total/2000:>8,.1f} tons)") print(f" {'─'*60}") -# Cost estimates at various $/ton rates -print(f"\n COST ESTIMATES:") -tons = grand_total / 2000 -for rate in [2800, 3200, 3600, 4000]: - cost = tons * rate - print(f" @ ${rate:,}/ton: ${cost:>14,.0f}") - print(f"\n{'='*78}\n") PYTHON diff --git a/tests/test_public_data_policy.py b/tests/test_public_data_policy.py new file mode 100644 index 0000000..2ac4ef4 --- /dev/null +++ b/tests/test_public_data_policy.py @@ -0,0 +1,23 @@ +import subprocess +import sys +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +class PublicDataPolicyTests(unittest.TestCase): + def test_repository_passes_public_data_check(self): + result = subprocess.run( + [sys.executable, ROOT / "scripts" / "check-public-data.py"], + cwd=ROOT, + capture_output=True, + text=True, + ) + + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + + +if __name__ == "__main__": + unittest.main() From 1a609b0ec39b16529e879d2567bf45ac29f3911b Mon Sep 17 00:00:00 2001 From: Victor Garcia Date: Tue, 28 Jul 2026 11:51:28 -0600 Subject: [PATCH 02/15] fix(nest): suppress unsafe burn outputs --- README.md | 8 +- ...-feat-trustworthy-estimate-package-plan.md | 404 ++++++++++++++++++ skills/steel-nest/SKILL.md | 16 +- skills/steel-nest/scripts/nest.py | 84 +++- tests/test_nest_burn_guard.py | 151 +++++++ 5 files changed, 647 insertions(+), 16 deletions(-) create mode 100644 docs/plans/2026-07-28-001-feat-trustworthy-estimate-package-plan.md create mode 100644 tests/test_nest_burn_guard.py diff --git a/README.md b/README.md index ca78456..8c3daf7 100644 --- a/README.md +++ b/README.md @@ -28,11 +28,13 @@ Ask your agent things like: > "What's the lightest W-shape with depth ≥ 18" and Ix ≥ 1000?" > "Total tonnage on this BOM with 12% connections" -### `steel-nest` — plate nesting & burn-table DXF +### `steel-nest` — plate nesting & guarded DXF output -The plate-layout step CAM software does, minus the CAM seat: MaxRects bin-packing of parts onto stock plates with kerf/gap/edge-margin spacing, holes and rectangular cutouts, yield/scrap/reusable-drop numbers, and material cost. Outputs a labeled layout (PDF + PNG per plate), a cut list, and **one DXF per sheet for the burn table** (part outlines on `PROFILE`, holes on `HOLES`, origin at sheet corner — ready for ProNest/FastCAM/SigmaNEST import). +The plate-layout step CAM software does, minus the CAM seat: MaxRects bin-packing of parts onto stock plates with kerf/gap/edge-margin spacing, holes and rectangular cutouts, yield/scrap/reusable-drop numbers, and material cost. Outputs include a labeled layout (PDF + PNG per plate), a cut list, and an explicitly named all-sheets reference DXF. -Honest about its limits: rectangular parts nest exactly; irregular parts nest by bounding box (flagged, never hidden); it deliberately does **not** emit G-code — kerf comp, lead-ins, and pierce points belong to your table's real post-processor. +Per-sheet `burn_plate_N.dxf` files are emitted only for a complete rectangular nest whose supported holes remain inside their parts. Any irregular part, unplaced part, or out-of-bounds hole suppresses burn DXFs for the whole job and leaves the estimating/reference artifacts available with an explicit warning. + +Honest about its limits: rectangular parts nest exactly; irregular parts nest by bounding box (flagged, never hidden); reference DXFs are not cutting instructions; and the package deliberately does **not** emit G-code. Kerf compensation, lead-ins, pierce points, and machine-specific verification belong to the table's real CAM and post-processor. > "How many sheets does this job need?" > "Nest these parts on 96×48 plate and give me the yield" diff --git a/docs/plans/2026-07-28-001-feat-trustworthy-estimate-package-plan.md b/docs/plans/2026-07-28-001-feat-trustworthy-estimate-package-plan.md new file mode 100644 index 0000000..88ec176 --- /dev/null +++ b/docs/plans/2026-07-28-001-feat-trustworthy-estimate-package-plan.md @@ -0,0 +1,404 @@ +--- +title: "feat: Build a Trustworthy Estimate Package" +type: feat +date: 2026-07-28 +deepened: 2026-07-28 +--- + +# feat: Build a Trustworthy Estimate Package + +## Summary + +Make v0.3 a reliability milestone that turns the advertised takeoff → nest → RFQ workflow into a deterministic, review-gated estimate package. Correct the unsafe burn-output boundary first, then add shared contracts, strict validation, deterministic RFQ generation, orchestration, and release-quality verification. + +--- + +## Problem Frame + +The package has a credible steel-domain foundation and a substantial plate-nesting engine, but the three skills do not yet form an executable pipeline. `steel-takeoff` produces a loosely defined CSV, `steel-nest` consumes an unrelated JSON job, and `steel-rfq` describes a workbook that the agent must recreate on every run. Only the nesting-to-RFQ JSON handoff is implemented. + +The most urgent defect is operational: irregular parts are packed by bounding box and that same rectangle is emitted on the burn DXF `PROFILE` layer. A gusset or other irregular part can therefore look cut-ready even though the file does not contain its true outline. The package also lacks input schemas, independent placement verification, automated tests, CI, declared Python requirements and supported version ranges, and a safe location for user company configuration. + +This milestone should make the current promise trustworthy before adding more standalone skills. It does not automate vendor communication or authorize commercial decisions. + +--- + +## Requirements + +### Safety and validation + +- R1. Burn files must contain only verified cutting entities and must never present bounding boxes, sheet outlines, labels, or unresolved irregular geometry as verified CAM-import profiles. +- R2. Every executable stage must validate its input before generating downstream artifacts and return field-specific errors for malformed data. +- R3. Nesting must independently verify material compatibility, positive finite dimensions, hole containment, plate bounds, required clearances, and non-overlap. +- R4. Unplaced parts and blocking validation errors must prevent a package from reaching RFQ-ready status. +- R5. Packing utilization and net material yield must be named and calculated separately so holes and irregular-part areas cannot make one generic yield value misleading. + +### Traceability and commercial controls + +- R6. The canonical estimate must preserve stable item IDs, source revision, drawing sheet/detail, quantity basis, assumptions, exclusions, and review status when those facts are available. +- R7. Pricing inputs and allowances must record their basis and effective context; no output may present hard-coded sensitivity rates as current market pricing. +- R8. RFQ workbooks must be deterministic drafts generated from validated data and must require a complete company profile without storing runtime company data inside the installed package. +- R9. The system must generate artifacts only; sending RFQs, selecting vendors, accepting substitutions, and making awards remain explicit human actions outside this milestone. +- R10. Estimate items must distinguish fabricated parts, purchased stock, hardware, allowances, exclusions, and by-others scope so calculated allowances cannot silently become vendor quantities. +- R11. On-hand stock and purchasable stock must remain distinct so consumed inventory is traceable but omitted from vendor purchase quantities. + +### Pipeline and compatibility + +- R12. One orchestration skill must normalize, validate, calculate, nest, and generate the draft RFQ while preserving identifiers between stages. +- R13. Every artifact-producing command must emit `run-manifest.json` containing schema versions, tool versions, input and configuration hashes, explicit dates, warnings, approximations, stage outcome, and an allow-list of produced artifacts. +- R14. Existing direct-use commands and documented CSV/JSON inputs must either remain compatible through adapters or receive an explicit migration path. + +### Quality and distribution + +- R15. Money-, weight-, geometry-, and workbook-affecting behavior must have repeatable unit, invariant, integration, and golden-contract coverage. +- R16. A clean supported environment must be able to diagnose missing dependencies and verify the contents of the published npm tarball. +- R17. Documentation must distinguish estimating assistance from structural design and burn DXF import geometry from machine-ready NC/G-code. +- R18. Any named CAM compatibility or geometry-readiness claim must be backed by a recorded import acceptance check for that product and version. + +--- + +## Key Technical Decisions + +- **Canonical contract is versioned JSON with adapters:** use one normalized estimate-package model for internal handoffs while retaining CSV/XLSX import and export at user-facing boundaries. This avoids repeated model reinterpretation without forcing estimators to author JSON. +- **Validation is layered:** JSON Schema handles shape and required fields; domain validation handles steel-specific and geometric invariants; stage gates decide whether processing may continue. A schema-valid estimate can still be commercially or geometrically blocked. +- **Agents interpret, deterministic code compiles:** the agent may extract drawing facts, map unusual spreadsheet columns, and surface ambiguity. Code owns calculations, nesting, workbook rendering, manifests, and stop conditions. +- **Burn readiness is explicit and fail-closed:** per-sheet burn DXFs contain only verified cutting entities. Any approximate profile, invalid hole, or partial nest suppresses fabrication-style DXF generation; reference layouts remain available as clearly named non-fabrication artifacts. +- **Commercial outputs remain drafts:** a generated workbook can be `rfq_ready_for_review`, never “sent” or “approved.” Missing identity, unresolved blockers, or unplaced material prevents that status. +- **Determinism is semantic and input-bound:** canonical JSON uses stable key and collection ordering plus field-defined numeric precision before hashing. Identical normalized input, configuration, and explicit dates must produce identical calculations, formulas, semantic workbook projection, and manifest hashes. Volatile run metadata is recorded separately and excluded from semantic hashes; byte-identical XLSX output is not required. +- **Artifacts publish as isolated runs:** every artifact-producing command stages into a new run directory, writes the manifest allow-list and outcome, then atomically publishes that directory. A small pointer record may identify the latest run, but blocked reruns never mutate or share a directory with prior successful artifacts. The run manifest is the integrity root: it byte-hashes every produced artifact except itself and the latest-run pointer, carries a semantic hash over its canonical projection with that field omitted, and the latest-run pointer records the final manifest byte hash. +- **Shared implementation lives inside the shipped skills tree:** reusable contracts, parsers, and validators belong under `skills/_shared/` so npm packaging includes them without creating a separate service or deployment target. +- **Direct scripts use one relocatable bootstrap:** every Python entry point derives the installed `skills/` root from its own file location and imports `skills/_shared/pi_steel` through the same bootstrap. The supported baseline is Python 3.11–3.13, JSON Schema Draft 2020-12 via `jsonschema`, and `pytest` for verification. +- **Characterization precedes engine changes:** preserve current valid rectangular nesting behavior with tests before changing spacing, metrics, grouping, or output contracts. + +--- + +## High-Level Technical Design + +### Estimate package flow + +```mermaid +flowchart TB + A[Source drawings, BOM CSV, or estimate XLSX] --> B[Normalize to versioned estimate package] + B --> C{Contract and domain validation} + C -->|errors| D[Blocked QA package] + C -->|valid or acknowledged warnings| E[Weight and allowance calculation] + E --> F[Split plate groups by material, grade, thickness] + F --> G{Nest and verify placements} + G -->|unplaced or invalid geometry| D + G -->|complete with approximate geometry| L[Review-required reference nest] + G -->|verified cut geometry| H[Render safe nest artifacts] + L --> I + H --> I{RFQ prerequisites complete} + I -->|no| D + I -->|yes| J[Compile draft RFQ workbook] + J --> K[Write manifest and QA report] +``` + +### Package readiness states + +```mermaid +stateDiagram-v2 + [*] --> Draft + Draft --> Blocked: contract or domain error + Draft --> ReviewRequired: warnings or assumptions + Draft --> Validated: no unresolved findings + ReviewRequired --> Blocked: finding rejected + ReviewRequired --> Validated: findings resolved or explicitly acknowledged + ReviewRequired --> RFQDraftReviewRequired: reference nest and purchase data complete + Validated --> NestedPartial: required material remains unplaced + Validated --> NestVerified: all required material placed safely + NestedPartial --> Blocked: preserve diagnostics + NestVerified --> RFQReadyForReview: company and purchase data complete + Blocked --> Draft: source data corrected + RFQDraftReviewRequired --> [*] + RFQReadyForReview --> [*] +``` + +No package state represents vendor delivery, award, purchase authorization, or machine execution. + +--- + +## Stage Outcome Contract + +The schemas keep three vocabularies separate: + +- `run_outcome` controls process exit and publication: `ready`, `review_required`, `blocked`, `dependency_missing`, or `usage_or_internal_error`. +- `package_status` records lifecycle readiness: `draft`, `review_required`, `validated`, `nested_partial`, `nest_verified`, `rfq_draft_review_required`, or `rfq_ready_for_review`. +- `artifact_readiness` describes one artifact: `geometry_verified`, `reference_only`, `draft`, or `diagnostic`. + +A `review_required` run may contain `reference_only` nest artifacts and an RFQ `draft`; it cannot contain a `geometry_verified` burn artifact or claim `rfq_ready_for_review`. + +| Outcome | Exit | Required artifacts | Forbidden artifacts | +|---|---:|---|---| +| `ready` | 0 | Run manifest, `qa-report.json`, stage outputs | None beyond artifacts outside the requested stage | +| `review_required` | 2 | Run manifest, QA report, reviewable estimate/reference outputs, and an explicitly review-required RFQ draft when purchase data is complete | `burn_plate_*.dxf`, RFQ-ready status | +| `blocked` | 3 | Run manifest and field-specific QA report; safe reference outputs when inputs permit | `burn_plate_*.dxf`, RFQ workbook, RFQ-ready status | +| `dependency_missing` | 4 | Run manifest naming the missing capability | Outputs that depend on the missing capability | +| `usage_or_internal_error` | 1 | Best-effort machine-readable diagnostic | Any artifact lacking a manifest allow-list entry | + +`burn_plate_N.dxf` exists only when the nest is complete and every profile and hole is verified. It contains only `PROFILE` and `HOLES` entities. `reference_plate_N.dxf` and `reference_nest.dxf` may contain `PLATE`, `BOUNDS`, `HOLES`, and `NOTES`, but unresolved irregular bounds never appear on `PROFILE`. A successful run followed by a blocked run publishes two isolated directories; the latest-run pointer identifies the blocked run, so stale successful files cannot masquerade as current output. + +--- + +## Output Structure + +```text +skills/ + _shared/ + pi_steel/ + schemas/ + steel-estimate/ + SKILL.md + scripts/ + references/ + steel-nest/ + steel-rfq/ + steel-takeoff/ +tests/ + fixtures/ + golden/ +docs/ + plans/ +``` + +The exact Python module split may change during implementation, but the shared contract must remain shipped with the npm package and usable by each skill without a network service. + +--- + +## Implementation Units + +### U8. Suppress unsafe burn output immediately + +- **Goal:** ship the smallest fail-closed correction before broader platform work begins. +- **Requirements:** R1, R4, R17. +- **Dependencies:** None. +- **Files:** `skills/steel-nest/scripts/nest.py`, `skills/steel-nest/SKILL.md`, `README.md`, `tests/test_nest_burn_guard.py`. +- **Approach:** add a conservative guard to the current renderer: produce no `burn_plate_*.dxf` when any part is irregular, any required part is unplaced, or basic supported-hole containment fails. Keep PDF/PNG, text, JSON, and clearly named reference outputs available. Update user-facing claims in the same unit. +- **Execution note:** start with characterization tests proving the current rectangular `PROFILE` behavior and the unsafe irregular case. +- **Patterns to follow:** retain the current no-G-code boundary and use current structured results for the temporary guard rather than designing the full v0.3 contract here. +- **Test scenarios:** + - A complete rectangular example continues to emit its current per-sheet burn DXFs. + - Any irregular part suppresses every `burn_plate_*.dxf` for the affected job and produces an explicit warning. + - Any unplaced part suppresses fabrication-style DXFs even when other parts fit. + - A supported hole outside its part suppresses fabrication-style DXFs. +- **Verification:** the known unsafe path is closed independently of U7, U1, or the canonical schema work. + +### U9. Validate the v0.3 workflow premise + +- **Goal:** confirm that safety, artifact recreation, and stage handoffs are the dominant next problems before freezing the canonical contract. +- **Requirements:** R6, R10, R12, R15, R18. +- **Dependencies:** U8. +- **Files:** `docs/research/v0.3-workflow-evidence.md`, `tests/fixtures/workflows/README.md`. +- **Approach:** exercise several materially different synthetic estimate families through the current takeoff, nest, and RFQ workflow. Record unsupported layouts, manual reinterpretation, blockers, rework, and the CAM product/version available for import acceptance. Do not use private production, customer, vendor, employee, project, pricing, contract, or operational data in this public repository. Confirm which of U2–U6 remain necessary for v0.3 and revise the plan if drawing ingestion or long-product optimization is the actual dominant blocker. +- **Patterns to follow:** preserve source artifacts only as approved sanitized fixtures; report decision-relevant workflow evidence without customer or commercial details. +- **Test scenarios:** + - The corpus includes different section layouts, scope markers, member/plate mixes, purchase-stock relationships, and at least one blocked or ambiguous case. + - Each observed layout is either supported by the proposed canonical model or explicitly rejected with a user-visible reason. + - The selected CAM acceptance target records product and version, or the plan records that v0.3 will make no named compatibility claim. +- **Verification:** the evidence note supports the v0.3 investment and defines the representative fixture families U2–U6 must pass. + +### U7. Establish the shipped Python runtime and test bootstrap + +- **Goal:** make shared modules, dependencies, and tests work from a source checkout and the installed npm tarball before feature units depend on them. +- **Requirements:** R13, R14, R15, R16. +- **Dependencies:** U9. +- **Files:** `pyproject.toml`, `requirements.txt`, `requirements-tested.txt`, `requirements-dev.txt`, `skills/_shared/bootstrap.py`, `skills/_shared/schemas/run-manifest.schema.json`, `skills/_shared/pi_steel/__init__.py`, `skills/_shared/pi_steel/run_manifest.py`, `scripts/doctor.py`, `package.json`, `DATA_PROVENANCE.md`, `tests/test_runtime_bootstrap.py`, `tests/test_run_manifests.py`, `tests/test_installed_scripts.py`. +- **Approach:** declare Python 3.11–3.13 and compatible runtime/test dependency ranges, record the exact dependency set exercised by CI, establish one file-relative import bootstrap for every shipped Python entry point, and implement the shared outcome, QA, atomic publication, allow-list, byte-hash, and semantic-hash primitives before feature units use them. Resolve and record the AISC data source, edition, checksum, transformation history, licensing evidence, and redistribution decision as an early release-viability gate. Add a minimal test command plus packed-artifact smoke fixture. Use JSON Schema Draft 2020-12 through `jsonschema`; do not require a global `PYTHONPATH` or editable install for normal Pi use. +- **Execution note:** first prove that current scripts can be invoked from an arbitrary working directory, then route new shared imports through the bootstrap. +- **Patterns to follow:** retain npm as the distribution mechanism and keep skill-local executable wrappers as the user-facing commands. +- **Test scenarios:** + - Each shipped Python entry point locates package assets and shared modules when invoked outside the repository root. + - The packed npm artifact contains the bootstrap, runtime requirements, and shared module package. + - An unsupported Python version or missing required dependency returns `dependency_missing` with a machine-readable diagnostic. + - Optional rendering and LibreOffice capabilities are reported separately from base calculation requirements. + - A ready run followed by a blocked run against the same destination publishes isolated manifests, and the latest-run pointer identifies only the blocked run. +- **Verification:** U1–U6 can import and test shared code from both the source tree and an unpacked npm tarball without environment-specific path setup. + +### U1. Make burn outputs safe for irregular parts + +- **Goal:** remove the current possibility that bounding-box geometry is presented as a verified irregular-part profile. +- **Requirements:** R1, R3, R4, R17, R18. +- **Dependencies:** U8, U9, and U7. +- **Files:** `skills/steel-nest/scripts/nest.py`, `skills/steel-nest/SKILL.md`, `README.md`, `tests/test_nest_outputs.py`, `tests/fixtures/nest/irregular-reference-only.json`. +- **Approach:** consume U7's shared Stage Outcome Contract and publication primitives. A complete verified rectangular nest may emit `burn_plate_N.dxf` containing only `PROFILE` and `HOLES`. A complete nest with unresolved irregular outlines is `review_required`, emits no `burn_plate_*.dxf`, and may emit `reference_plate_N.dxf` plus `reference_nest.dxf`. Invalid geometry or a partial nest is `blocked`; it preserves only safe reference/diagnostic artifacts. +- **Execution note:** add characterization coverage for existing rectangular DXF entities, layers, units, and origins before changing the renderer. +- **Patterns to follow:** preserve the existing `PROFILE` and `HOLES` conventions for verified rectangular parts; retain `PLATE` and `NOTES` only in clearly separate reference outputs; preserve the documented no-G-code boundary. +- **Test scenarios:** + - A rectangular plate part with valid holes produces a closed `PROFILE`, the expected `HOLES`, inch units, and `geometry_verified` status. + - An irregular gusset with only width, height, and area never places its bounding rectangle on `PROFILE`; the result and report identify it as `reference_only`. + - A mixed plate containing rectangular and unresolved irregular parts emits no fabrication-style DXF and cannot be reported as geometry-verified. + - A geometry-verified per-sheet DXF contains no plate-outline or label entities that a CAM importer could mistake for cuts. + - Requesting geometry-verified-only output for unresolved geometry exits unsuccessfully while still producing a diagnostic QA result. + - A successful run followed by a blocked rerun to the same requested destination publishes a new isolated run and leaves no stale burn file in the blocked run. +- **Verification:** no user-facing message can represent unresolved geometry as ready for CAM import. A named CAM compatibility claim is published only after U9's selected product/version successfully preserves units, origin, closed profiles, and hole classification; otherwise documentation describes the DXF layer contract without a product-support claim. + +### U2. Define the canonical estimate package and validation gates + +- **Goal:** establish one versioned, traceable contract shared by takeoff, nesting, purchasing, and RFQ generation. +- **Requirements:** R2, R3, R4, R6, R7, R10, R11, R12, R13, R14. +- **Dependencies:** U8, U9, U7, and U1. +- **Files:** `skills/_shared/schemas/estimate-package.schema.json`, `skills/_shared/schemas/nest-result.schema.json`, `skills/_shared/pi_steel/contracts.py`, `skills/_shared/pi_steel/validation.py`, `skills/_shared/pi_steel/parsing.py`, `skills/_shared/pi_steel/geometry_verify.py`, `skills/steel-takeoff/assets/bom-template.csv`, `skills/steel-takeoff/scripts/validate-bom.py`, `skills/steel-takeoff/scripts/calculate-weight.sh`, `skills/steel-nest/references/job_template.json`, `skills/steel-nest/references/example_job.json`, `tests/test_contracts.py`, `tests/fixtures/contracts/`. +- **Approach:** model project and revision metadata, source page/row evidence, estimator and as-of date, member items, plate parts, on-hand and purchasable stock, explicit allowances, commercial basis, review findings, stage status, and artifact lineage. Use discriminated item intent so fabricated parts, purchase stock, hardware, allowances, exclusions, and by-others scope cannot be conflated. Define three identifier levels: source IDs supplied by structured inputs or revision-scoped by legacy adapters, normalized item IDs derived from project plus stable source/mark identity, and deterministic per-instance placement IDs derived from the normalized item ID plus canonical instance order. Ambiguous duplicate legacy rows remain review-required rather than receiving false-stable IDs. Normalize legacy BOM CSV and nest JSON through adapters rather than breaking direct workflows. +- **Execution note:** implement new contract and semantic-validation behavior test-first; add compatibility fixtures before changing existing templates. +- **Patterns to follow:** reuse AISC designation and grade knowledge from `skills/steel-takeoff/scripts/validate-bom.py`; preserve stable `rfq_nesting` concepts while versioning their schema. +- **Test scenarios:** + - A legacy member-only BOM CSV normalizes to the canonical model without losing marks, grades, lengths, or weights. + - Duplicate IDs, zero or negative quantities, non-finite dimensions, unsupported shapes, out-of-bounds holes, and net area at or below zero return field-specific blocking errors. + - Parts with different material, grade, or thickness cannot enter the same nest group. + - An estimate allowance contributes to estimate totals but never becomes a fabricated part or vendor purchase row. + - Confirmed on-hand stock can satisfy a nest requirement and appears as inventory consumption, not a vendor purchase quantity. + - On-hand stock without a stable inventory ID, measured dimensions, source and as-of date, available/reserved status, and reviewer confirmation bound to the estimate hash cannot reduce RFQ purchase quantities. + - Missing drawing evidence creates a visible review warning rather than invented source data. + - A warning acknowledgement records a stable finding ID, actor, timestamp, disposition, and relevant input hash; changing source or configuration invalidates it. + - A legacy nest job may inherit one explicit job-level material, grade, thickness, and unit basis; missing compatibility facts remain `review_required` and block burn/RFQ readiness rather than being invented from a stock name. + - Explicit cost inputs retain currency, unit basis, and effective context; absent cost inputs produce no fabricated price. + - A contract version unknown to the installed tool is rejected with a migration-oriented diagnostic. + - Explicit source IDs and normalized item IDs remain stable when rows reorder; quantity expansion creates predictable instance IDs without renaming existing instances. + - A legacy row without a stable mark or source key receives a revision-scoped ID and warning; indistinguishable duplicates cannot be silently merged. +- **Verification:** every downstream stage consumes a validated versioned model or a documented legacy adapter, and blockers versus warnings have one consistent meaning. + +### U3. Harden nesting calculations, grouping, and verification + +- **Goal:** make the nester a validated domain engine whose metrics and outputs reconcile. +- **Requirements:** R2, R3, R4, R5, R12, R13, R15. +- **Dependencies:** U2. +- **Files:** `skills/steel-nest/scripts/nest.py`, `skills/steel-nest/SKILL.md`, `skills/steel-nest/references/job_template.json`, `skills/steel-nest/references/example_job.json`, `tests/test_nest_engine.py`, `tests/test_nest_invariants.py`, `tests/test_nest_cli_contract.py`, `tests/fixtures/nest/`. +- **Approach:** validate before placement, segregate material groups, aggregate unplaced quantities, and send normalized placement results through the shared pure geometry verifier rather than MaxRects internals. Include algorithm version plus normalized input hash. Replace generic yield with packing utilization and net material yield, each carrying an approximation status. Report leftover free rectangles as remnant candidates rather than certified reusable drops. Define kerf, inter-part gap, and edge-margin ownership so exact-fit boundary behavior is intentional. +- **Execution note:** characterize current valid layouts first, then change one invariant or metric family at a time. +- **Patterns to follow:** keep `run_job` as the computation boundary and retain structured `result.json` plus human `report.txt`; version the RFQ nesting handoff rather than silently changing fields. +- **Test scenarios:** + - Exact-fit and rotated-fit parts honor the documented edge and spacing contract without false rejection. + - Non-rotatable parts remain oriented and oversized parts are aggregated as unplaced with quantities. + - Finite stock exhaustion blocks RFQ readiness; unlimited stock opens only compatible plates. + - Stock entries with the same display name but different sizes, grades, or thicknesses remain distinct in results and RFQ rows. + - Every generated placement is inside usable bounds and maintains required clearance from every other placement. + - Cost-per-sheet and cost-per-pound cases reconcile to plate counts and weights; conflicting cost bases are rejected. + - Holes and declared irregular areas affect net utilization but not packing coverage, and approximation labels remain visible. + - Direct nesting commands emit the shared run manifest and obey the outcome/exit/artifact matrix for ready, review-required, and blocked cases. +- **Verification:** independently checked geometry, material grouping, cost, and metrics reconcile in structured and human-readable outputs for all fixtures. + +### U4. Implement the deterministic RFQ compiler + +- **Goal:** replace model-improvised workbook creation with a repeatable parser, normalizer, renderer, and verifier. +- **Requirements:** R2, R7, R8, R9, R10, R11, R12, R14, R15. +- **Dependencies:** U2 and the versioned nest handoff from U3. +- **Files:** `skills/steel-rfq/scripts/generate-rfq.py`, `skills/steel-rfq/scripts/recalc.py`, `skills/steel-rfq/SKILL.md`, `skills/steel-rfq/assets/company-profile.example.json`, `skills/steel-rfq/references/rfq-input.md`, `tests/test_rfq_generator.py`, `tests/test_rfq_workbook_contract.py`, `tests/fixtures/rfq/`, `tests/golden/rfq/`. +- **Approach:** separate spreadsheet interpretation from deterministic generation. Adapters normalize estimate XLSX or the canonical package into typed purchase items; the compiler owns grouping, styles, formulas, nesting references, approved term templates, injected prepared/issued dates, naming, print setup, and draft status. Each term template records its hash, approver, approval date, and status; any content edit invalidates approval and returns the workbook to draft review. Resolve company configuration from an explicit `PI_STEEL_CONFIG` path, then project-local `.pi-steel/company-profile.json`, then the platform user-config directory, with the chosen source recorded in the manifest. Golden comparison uses a normalized semantic projection of cells, formulas, styles, ranges, print properties, and selected workbook metadata rather than XLSX ZIP bytes. +- **Execution note:** build workbook contract tests before moving the formatting rules out of `SKILL.md`. +- **Patterns to follow:** preserve the documented workbook structure and `recalc.py` distinction between formula recalculation-on-open and values baked by LibreOffice. +- **Test scenarios:** + - A valid canonical package produces the expected groups, merges, styles, widths, print settings, vendor input cells, and exact total formulas. + - The legacy adapter maps an unambiguous `BY OTHERS` marker and zero quantity to typed scope intent while valid mixed-grade rows remain traceable to source IDs. + - Structured scope intent controls exclusions; a description that merely contains similar words cannot change scope. + - Consolidated purchase stock replaces individual plate/flat-bar pieces only when the validated input explicitly identifies that relationship. + - Missing company identity blocks generation; a missing optional logo falls back to the text header. + - Nesting rows remain separated by material, grade, thickness, and sheet size and show reference-only warnings. + - A system without LibreOffice reports cached formula values as deferred rather than claiming they were baked. + - The compiler writes a draft workbook only and exposes no send or award action. + - Direct RFQ commands emit the shared run manifest and obey the outcome/exit/artifact matrix. +- **Verification:** reopening a generated workbook with `openpyxl` proves its structural contract without relying on visual inspection or model judgment. + +### U5. Add the end-to-end `steel-estimate` orchestrator + +- **Goal:** make the advertised pipeline executable as one review-gated workflow with stable artifacts and lineage. +- **Requirements:** R4, R6, R8, R9, R10, R11, R12, R13, R14, R17. +- **Dependencies:** U2, U3, and U4. +- **Files:** `skills/steel-estimate/SKILL.md`, `skills/steel-estimate/scripts/build-estimate-package.py`, `skills/steel-estimate/references/estimate-package-example.json`, `skills/steel-estimate/references/output-contract.md`, `tests/test_estimate_pipeline.py`, `tests/fixtures/pipeline/`, `tests/golden/pipeline/`. +- **Approach:** orchestrate normalization, validation, BOM calculation, compatible nest-group creation, safe rendering, RFQ compilation, and final manifest/QA reporting. Preserve partial diagnostic artifacts when blocked. Complete reference-only nests may produce an explicitly review-required RFQ draft, but validation failures, unplaced parts, or missing required company data produce no workbook, and unresolved cut readiness never receives RFQ-ready status. +- **Execution note:** start with a failing end-to-end contract test that describes the complete package inventory and stop conditions. +- **Patterns to follow:** compose the existing skill engines rather than duplicating their calculations; use the current `rfq_nesting.json` intent as the initial lineage seam. +- **Test scenarios:** + - A representative project with W-shapes, long products, plate parts, and purchase stock produces a normalized BOM, separated nests, draft RFQ, manifest, and QA report with stable IDs. + - Re-running the same normalized input, configuration, and explicit dates yields the same canonical JSON, ordering, formulas, semantic workbook projection, and semantic manifest hashes; volatile run metadata does not affect them. + - Changing the source revision or a quantity changes the manifest hash and identifies affected downstream artifacts. + - A validation error produces a blocked package with diagnostics and no RFQ workbook. + - Unplaced parts preserve nest reports but prevent RFQ-ready status. + - Warnings and approximations flow into the QA report and workbook notes instead of disappearing between stages. +- **Verification:** one agent request can produce a complete review package without ad hoc calculations or spreadsheet rendering, and every blocked flow stops at the documented gate. + +### U6. Establish release, dependency, and provenance gates + +- **Goal:** make clean-install behavior, generated-data provenance, and npm contents reproducible. +- **Requirements:** R15, R16, R17. +- **Dependencies:** U8, U9, U7, and U1 through U5. +- **Files:** `package.json`, `.github/workflows/ci.yml`, `DATA_PROVENANCE.md`, `README.md`, `tests/test_package_contents.py`, `tests/test_data_provenance.py`, `tests/test_full_render_smoke.py`. +- **Approach:** add project-local commands for unit/integration tests, example contracts, dependency diagnosis, and tarball inspection. Separate base no-render tests from full optional-render tests. Enforce U7's resolved AISC provenance and redistribution decision rather than deferring that decision until release. +- **Patterns to follow:** retain npm as the Pi distribution mechanism and keep generated outputs, profiles, caches, and credentials out of the tarball. +- **Test scenarios:** + - Dependency diagnosis distinguishes missing required tools from optional rendering or formula-baking capabilities. + - The package-content test includes all skills, schemas, shared modules, examples, and runtime requirements while excluding profiles, outputs, caches, and test-only artifacts as intended. + - A clean environment runs the contract and no-render test tier; the full environment additionally verifies PDF/PNG/DXF and workbook baking. + - The optional full tier renders a representative RFQ to PDF through LibreOffice and checks that key regions are present without obvious clipping or logo overlap. + - Shape data counts, required fields, uniqueness, declared edition, and checked-in checksums remain consistent. + - Documentation examples reference only shipped files and describe burn/RFQ safety states accurately. +- **Verification:** CI and the local release gate fail on behavioral regressions, unsafe packaging, missing runtime assets, or undocumented data changes. + +--- + +## Acceptance Examples + +- AE1. Given an irregular gusset with only a bounding box and declared area, when nesting outputs are generated, then no rectangular cutting profile is emitted for that gusset and the plate is `reference_only`. +- AE2. Given a hole whose edge falls outside its part, when the estimate is validated, then nesting does not start and the QA report identifies the part and hole path. +- AE3. Given two plate parts with different grade or thickness, when the package is built, then they are assigned to separate compatible nest groups and RFQ purchase rows. +- AE4. Given valid material with insufficient stock, when nesting completes with unplaced parts, then diagnostic nest artifacts remain available but no RFQ-ready status is produced. +- AE5. Given a valid estimate and complete company profile, when the package is built twice, then the normalized data, workbook structure, and semantic manifest contents are reproducible. +- AE6. Given unresolved takeoff assumptions, when a package is generated, then each assumption remains traceable in the QA report and relevant RFQ notes until reviewed. +- AE7. Given no explicit cost basis, when BOM totals are calculated, then weight and tonnage are reported without presenting illustrative rates as current prices. +- AE8. Given LibreOffice is unavailable, when the RFQ workbook is generated, then the result says formulas recalculate on open and does not claim cached values were computed. +- AE9. Given a confirmed available remnant with measured dimensions and an approval bound to the current estimate hash satisfies a plate group, when purchase quantities are compiled, then the manifest records the inventory consumption and the RFQ omits that stock. + +--- + +## System-Wide Impact + +- **Estimators:** receive a reviewable package with explicit assumptions and blockers instead of three loosely connected artifacts. +- **Fabrication and CAM users:** gain a reliable distinction between verified rectangular cut geometry and reference-only irregular bounds. +- **Procurement users:** receive deterministic draft RFQs but retain control over substitutions, sending, and awards. +- **Agents and developers:** move interpretation to the edges and calculations into tested code, reducing prompt drift and duplicated logic. +- **Distribution:** npm remains the delivery mechanism, but the shipped surface expands to include shared schemas, Python modules, an orchestrator skill, and documented dependency tiers. + +--- + +## Risks and Dependencies + +- **Backward compatibility:** renaming yield fields and versioning handoffs can break consumers. Keep legacy input adapters and deprecation warnings through the v0.3 line; write only the new output schema and document that no downgrade path is provided. +- **Spreadsheet diversity:** real estimate workbooks vary widely. Keep heuristic parsing in adapters and require confirmation when mapping is ambiguous; never let the deterministic compiler infer columns. +- **Geometry scope:** this milestone does not implement true-shape nesting. Safe reference-only behavior must not be mistaken for a placeholder that quietly becomes geometry-verified. +- **Commercial language:** RFQ terms can create commitments when sent. The package generates drafts only, and terms/profile changes require human review. +- **Python distribution:** npm cannot install Python dependencies automatically. The doctor and documented supported environment must make that limitation visible. +- **AISC data rights and provenance:** provenance work may uncover redistribution constraints. Treat the audit as a prerequisite to stronger licensing claims, not as proof in advance. + +--- + +## Scope Boundaries + +### Included + +- Immediate irregular-profile burn-output safety. +- Canonical estimate and output contracts with legacy adapters. +- Strict validation, nesting hardening, deterministic RFQ compilation, orchestration, and release gates. +- Evidence fields and review-state propagation needed for later drawing revision workflows. + +### Deferred to Follow-Up Work + +- **v0.4:** drawing/PDF ingestion, evidence review queue, and addendum-aware estimate deltas. +- **v0.5:** one-dimensional stock optimization for beams, HSS, channels, angles, and flat bar, followed by remnant inventory. +- **v0.6:** returned vendor quote normalization and comparison with commercial approval gates. +- True-shape polygon/DXF nesting may move earlier if users need irregular burn geometry before purchasing intelligence. + +### Outside This Milestone + +- Machine-specific toolpaths, kerf compensation, lead-ins, pierce strategy, NC, or G-code. +- Automated RFQ email, vendor selection, substitution approval, purchase authorization, or award. +- Structural engineering, member design, connection design, or code-compliance certification. +- Real-time market pricing, a hosted service, or a web UI. + +--- + +## Sources and Research + +- `README.md` defines the advertised three-skill pipeline and current operational claims. +- `skills/steel-nest/scripts/nest.py` contains the MaxRects engine, metric calculations, RFQ handoff, and DXF renderers. +- `skills/steel-nest/SKILL.md` defines input expectations, safety boundaries, and the claimed verification behavior. +- `skills/steel-rfq/SKILL.md` is the current workbook contract; `skills/steel-rfq/scripts/recalc.py` is the only deterministic RFQ helper. +- `skills/steel-takeoff/scripts/validate-bom.py` and `skills/steel-takeoff/scripts/calculate-weight.sh` expose the duplicated parsing, validation, and hard-coded cost-sensitivity behavior to consolidate. diff --git a/skills/steel-nest/SKILL.md b/skills/steel-nest/SKILL.md index 4f9195e..5e49d39 100644 --- a/skills/steel-nest/SKILL.md +++ b/skills/steel-nest/SKILL.md @@ -1,6 +1,6 @@ --- name: steel-nest -description: "Nest steel parts onto stock plates and estimate material — the plate-layout / cutting step that CAM software (SigmaNEST, Hypertherm, FANUC) does. Use this skill whenever someone mentions nesting, plate layout, plate optimization, cut list, cutting plan, yield, drop/remnant, how many sheets/plates a job needs, how much plate to buy, or laying parts out on a sheet. Also trigger when a new order/SO comes in and someone asks 'how much material', 'how many plates', 'what's the yield', or 'lay these parts out' — even if they don't say the word 'nest'. Produces a nesting layout (PDF + PNG), a cut list, yield/scrap/remnant numbers, material cost, and a DXF of the nest. Rectangular parts nest exactly; irregular parts nest by bounding box." +description: "Nest steel parts onto stock plates and estimate material — the plate-layout / cutting step that CAM software does. Use this skill whenever someone mentions nesting, plate layout, plate optimization, cut list, cutting plan, yield, drop/remnant, how many sheets/plates a job needs, how much plate to buy, or laying parts out on a sheet. Also trigger when a new order/SO comes in and someone asks 'how much material', 'how many plates', 'what's the yield', or 'lay these parts out' — even if they don't say the word 'nest'. Produces a nesting layout (PDF + PNG), a cut list, yield/scrap/remnant numbers, material cost, and guarded reference/burn DXFs. Rectangular parts nest exactly; irregular parts nest by bounding box." --- # Steel Plate Nesting & Estimate @@ -22,13 +22,15 @@ Be honest with the user about the boundary — it protects the shop from over-tr - Yield %, scrap weight, largest reusable **drop** per plate. - Material weight and cost (by $/lb — which also values scrap — or by $/sheet). - Labeled layout (PDF + one PNG per plate). -- **Burn-table files**: one DXF **per sheet** (`burn_plate_N.dxf`) with part outlines on layer `PROFILE` and holes on layer `HOLES`, origin at the sheet corner — ready to import into the table's CAM. Plus `nest.dxf`, an all-sheets overview. +- **Reference file**: `reference_nest.dxf`, an all-sheets estimating/layout reference. +- **Guarded burn-table files**: one DXF per sheet (`burn_plate_N.dxf`) with part outlines on `PROFILE` and holes on `HOLES`, origin at the sheet corner, but only when every part is rectangular, every required part fits, and every supported hole stays inside its part. **Approximate — always flag it:** - **Irregular parts** (gussets, brackets, curved profiles, parts with holes) are nested by their **bounding box**, not true shape. Real yield is a little better than reported. For exact weight/cost on those, get the true cut area (in²) into the part's `area` field. This is NOT true-shape nesting like a dedicated CAM engine. +- Any irregular part suppresses all fabrication-style DXFs for that job. The remaining PDF, PNG, report, JSON, and `reference_nest.dxf` outputs are estimating aids, not cutting instructions. **Do NOT pretend to do:** -- **Machine-ready G-code / NC** with kerf compensation, pierce points, and lead-ins for a specific controller. That is machine-specific and safety-critical and must come from the real post-processor. The burn DXF from this skill is a geometry/import file — the table's CAM (Hypertherm ProNest, FastCAM, SigmaNEST, Lantek, or the controller's own importer) applies kerf comp, lead-ins and pierce. Say so plainly if asked for G-code; offer the DXF as the correct hand-off. +- **Machine-ready G-code / NC** with kerf compensation, pierce points, and lead-ins for a specific controller. That is machine-specific and safety-critical and must come from the real post-processor. When a burn DXF is emitted, it is still an import geometry file that requires operator and CAM verification. Say so plainly if asked for G-code. ## Inputs to Gather @@ -36,7 +38,7 @@ Everything drives a single job JSON (schema in `references/job_template.json`; a Gather three things: -1. **Parts** — for each unique part: name, width × height (inches; use the bounding box for odd shapes), quantity, whether it's `rect` or `irregular`, and whether rotation is allowed (`rotatable: false` locks grain/rolling direction for anisotropic material or directional finish). If a part has **holes or cutouts** and you want them cut in the burn file (and netted out of weight), add a `holes` list — each hole's `x,y` is its center from the part's lower-left corner: round = `{"dia":, "x":, "y":}`, rectangular cutout = `{"w":, "h":, "x":, "y":}`. Holes are optional; skip them if you only need the layout/estimate. +1. **Parts** — for each unique part: name, width × height (inches; use the bounding box for odd shapes), quantity, whether it's `rect` or `irregular`, and whether rotation is allowed (`rotatable: false` locks grain/rolling direction for anisotropic material or directional finish). If a rectangular part has **holes or cutouts**, add a `holes` list — each hole's `x,y` is its center from the part's lower-left corner: round = `{"dia":, "x":, "y":}`, rectangular cutout = `{"w":, "h":, "x":, "y":}`. A supported hole must remain fully inside its part or fabrication-style DXFs are suppressed. Holes are optional; skip them if you only need the layout/estimate. 2. **Stock** — plate size(s) on hand (width × height × thickness), how many sheets are available (or `unlimited` to buy as needed), and price (`cost_per_lb` preferred; `cost_per_sheet` works too). 3. **Cut settings** — kerf, part gap, edge margin, material density. Sensible defaults are in the template; only ask if the user hasn't implied them. Common kerf: plasma ~0.06", oxy-fuel ~0.10", laser ~0.02", waterjet ~0.03". @@ -58,8 +60,8 @@ python3 scripts/nest.py --job --out Outputs land in `/`: - `layout.pdf` — every plate drawn (holes shown) + a summary page (the main deliverable) - `plate_1.png`, `plate_2.png`, … — one image per plate -- `burn_plate_1.dxf`, `burn_plate_2.dxf`, … — **one DXF per sheet for the burn table** (PROFILE + HOLES layers, origin at sheet corner) -- `nest.dxf` — all sheets side-by-side, one overview file +- `burn_plate_1.dxf`, `burn_plate_2.dxf`, … — guarded per-sheet import geometry (PROFILE + HOLES layers, origin at sheet corner); absent when the safety guard blocks them +- `reference_nest.dxf` — all sheets side-by-side, reference only - `rfq_nesting.json` — the Material / Nesting Plan / Drop Notes block for the `steel-rfq` hand-off (see below) - `report.txt` — the text report - `result.json` — structured result (plates, placements, holes, yield, cost) for downstream use @@ -68,7 +70,7 @@ The engine has no third-party build dependencies beyond `ezdxf`, `matplotlib`, a ## What to Deliver -Always deliver the **PDF layout** and give the headline numbers in the message: plates used, overall yield %, total material cost, and any parts that **did not fit** (the report flags these — never hide them; it means they need more or bigger stock). Offer the DXF and per-plate PNGs. If the parts were irregular, restate the bounding-box caveat so the quote isn't over-trusted. +Always deliver the **PDF layout** and give the headline numbers in the message: plates used, overall yield %, total material cost, and any parts that **did not fit** (the report flags these — never hide them; it means they need more or bigger stock). Offer the reference DXF and per-plate PNGs. Offer burn DXFs only when they were emitted by the guard, and still state that CAM/operator verification is required. If burn DXFs were suppressed, report every reason. If the parts were irregular, restate the bounding-box caveat so the quote isn't over-trusted. Verify before presenting: the engine already checks that no parts overlap and all fit in-bounds, but sanity-check the yield and cost against the plate count (e.g., cost = plates × sheet cost, or plate weight × $/lb). If a plate shows very low yield, mention it — it's usually the tail plate and may be worth holding parts for the next order. diff --git a/skills/steel-nest/scripts/nest.py b/skills/steel-nest/scripts/nest.py index 2531e9a..3c13668 100644 --- a/skills/steel-nest/scripts/nest.py +++ b/skills/steel-nest/scripts/nest.py @@ -14,10 +14,10 @@ * Yield / scrap / largest reusable drop, part weight, material cost. * Labeled layout (PNG per plate + combined PDF). * DXF outputs: - - nest.dxf all plates side-by-side (overview/reference) - - burn_plate_N.dxf ONE FILE PER SHEET for the burn table: - part profiles on layer PROFILE, holes on - layer HOLES, origin at the sheet corner. + - reference_nest.dxf all plates side-by-side (reference only) + - burn_plate_N.dxf ONE FILE PER SHEET for the burn table when + every part is rectangular, every required part + fits, and supported holes stay inside the part. Deliberately NOT done: * True-shape nesting of irregular parts (they nest by BOUNDING BOX, @@ -186,6 +186,62 @@ def hole_local(pc, hole): return hx, hy +def burn_dxf_warnings(job, unplaced): + """Return reasons the current job is unsafe for fabrication-style DXF output.""" + warnings = [] + + if any(part.get("shape", "rect") == "irregular" for part in job["parts"]): + warnings.append( + "Irregular parts use approximate bounding boxes; burn DXFs are suppressed." + ) + + if unplaced: + warnings.append( + f"{len(unplaced)} required part(s) did not fit; burn DXFs are suppressed." + ) + + eps = 1e-9 + for part in job["parts"]: + width = float(part["width"]) + height = float(part["height"]) + for index, hole in enumerate(part.get("holes", []) or [], 1): + try: + x = float(hole["x"]) + y = float(hole["y"]) + if hole.get("dia") is not None: + radius = float(hole["dia"]) / 2.0 + contained = ( + radius > 0 + and x - radius >= -eps + and y - radius >= -eps + and x + radius <= width + eps + and y + radius <= height + eps + ) + elif hole.get("w") is not None and hole.get("h") is not None: + hole_width = float(hole["w"]) + hole_height = float(hole["h"]) + contained = ( + hole_width > 0 + and hole_height > 0 + and x - hole_width / 2.0 >= -eps + and y - hole_height / 2.0 >= -eps + and x + hole_width / 2.0 <= width + eps + and y + hole_height / 2.0 <= height + eps + ) + else: + contained = False + except (KeyError, TypeError, ValueError): + contained = False + + if not contained: + warnings.append( + f"Part '{part['name']}' hole {index} is unsupported or extends " + "outside the part; burn DXFs are suppressed." + ) + + return warnings + + # -------------------------------------------------------------------------- # Job runner # -------------------------------------------------------------------------- @@ -353,6 +409,7 @@ def _summarize(job, used_plates, unplaced, density, margin, kerf, gap): overall_yield = round(100 * tot_part_area_bbox / tot_plate_area, 1) if tot_plate_area else 0.0 + burn_warnings = burn_dxf_warnings(job, unplaced) res = { "meta": { "job_name": job.get("job_name", "Nesting job"), @@ -373,6 +430,8 @@ def _summarize(job, used_plates, unplaced, density, margin, kerf, gap): "unplaced": [{"label": u["label"], "size": f'{_fmt(u["w"])} x {_fmt(u["h"])}'} for u in unplaced], "has_irregular": any(p.get("shape") == "irregular" for p in job["parts"]), + "burn_dxf_eligible": not burn_warnings, + "burn_dxf_warnings": burn_warnings, } res["rfq_nesting"] = rfq_nesting_block(res) return res @@ -468,6 +527,12 @@ def render_text(res): if res["has_irregular"]: L.append(" NOTE: irregular parts are nested by BOUNDING BOX. Supply a") L.append(" true `area` per irregular part for exact weight/cost.") + if res["burn_dxf_warnings"]: + L.append("") + L.append(" BURN DXF SUPPRESSED:") + for warning in res["burn_dxf_warnings"]: + L.append(f" - {warning}") + L.append(" Reference layouts are estimating aids, not cutting instructions.") L.append("=" * 64) return "\n".join(L) @@ -584,13 +649,16 @@ def render_dxf_overview(res, outdir): for pc in pr["placements"]: _draw_part_dxf(msp, pc, x_off + margin, margin, "PROFILE", "HOLES", "NOTES") x_off += W + 10.0 - path = os.path.join(outdir, "nest.dxf") + path = os.path.join(outdir, "reference_nest.dxf") doc.saveas(path) return path def render_burn_dxfs(res, outdir): """One DXF per sheet for the burn table. Origin at sheet corner.""" + if not res.get("burn_dxf_eligible", False): + return [] + import ezdxf margin = res["meta"]["edge_margin_in"] paths = [] @@ -642,9 +710,13 @@ def main(): overview = render_dxf_overview(res, args.out) burns = render_burn_dxfs(res, args.out) print(f"\nWrote: {pdf}") - print(f" {overview} (overview)") + print(f" {overview} (reference only)") for b in burns: print(f" {b} (burn table — one per sheet)") + if not burns: + print(" burn DXFs suppressed:") + for warning in res["burn_dxf_warnings"]: + print(f" - {warning}") print(f" {len(pngs)} PNG(s), report.txt, result.json, rfq_nesting.json") diff --git a/tests/test_nest_burn_guard.py b/tests/test_nest_burn_guard.py new file mode 100644 index 0000000..2730c0d --- /dev/null +++ b/tests/test_nest_burn_guard.py @@ -0,0 +1,151 @@ +import importlib.util +import sys +import tempfile +import unittest +from pathlib import Path + +import ezdxf + + +ROOT = Path(__file__).resolve().parents[1] +NEST_SCRIPT = ROOT / "skills" / "steel-nest" / "scripts" / "nest.py" +SPEC = importlib.util.spec_from_file_location("pi_steel_nest", NEST_SCRIPT) +nest = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = nest +SPEC.loader.exec_module(nest) + + +def job_with(part, *, stock_qty=1): + return { + "job_name": "SYNTHETIC-BURN-GUARD", + "settings": { + "kerf_in": 0.06, + "part_gap_in": 0.25, + "edge_margin_in": 0.5, + "density_lb_in3": 0.2836, + }, + "stock": [ + { + "name": "A36 Plate 1/2", + "width": 20, + "height": 20, + "thickness": 0.5, + "qty": stock_qty, + } + ], + "parts": [part], + } + + +class BurnDxfGuardTests(unittest.TestCase): + def render_burns(self, job): + result = nest.run_job(job) + output_dir = tempfile.TemporaryDirectory() + self.addCleanup(output_dir.cleanup) + paths = nest.render_burn_dxfs(result, output_dir.name) + return result, [Path(path) for path in paths] + + def test_complete_rectangular_part_emits_closed_profile(self): + result, paths = self.render_burns( + job_with( + { + "name": "Base Plate", + "width": 8, + "height": 6, + "qty": 1, + "shape": "rect", + "holes": [{"dia": 1, "x": 4, "y": 3}], + } + ) + ) + + self.assertEqual(result["unplaced"], []) + self.assertTrue(result["burn_dxf_eligible"]) + self.assertEqual(result["burn_dxf_warnings"], []) + self.assertEqual(len(paths), 1) + document = ezdxf.readfile(paths[0]) + profiles = list(document.modelspace().query('*[layer=="PROFILE"]')) + holes = list(document.modelspace().query('*[layer=="HOLES"]')) + self.assertEqual(len(profiles), 1) + self.assertTrue(profiles[0].closed) + self.assertEqual(len(holes), 1) + + def test_irregular_part_suppresses_all_burn_dxfs(self): + result, paths = self.render_burns( + job_with( + { + "name": "Gusset", + "width": 8, + "height": 6, + "qty": 1, + "shape": "irregular", + "area": 24, + } + ) + ) + + self.assertEqual(paths, []) + self.assertFalse(result["burn_dxf_eligible"]) + self.assertTrue( + any("irregular" in warning.lower() for warning in result["burn_dxf_warnings"]) + ) + self.assertIn("BURN DXF SUPPRESSED", nest.render_text(result)) + + output_dir = tempfile.TemporaryDirectory() + self.addCleanup(output_dir.cleanup) + reference_path = Path(nest.render_dxf_overview(result, output_dir.name)) + self.assertEqual(reference_path.name, "reference_nest.dxf") + self.assertTrue(reference_path.exists()) + + def test_unplaced_part_suppresses_burn_dxfs_for_the_whole_job(self): + job = job_with( + { + "name": "Fitting Part", + "width": 8, + "height": 6, + "qty": 1, + "shape": "rect", + } + ) + job["parts"].append( + { + "name": "Oversize Part", + "width": 30, + "height": 30, + "qty": 1, + "shape": "rect", + } + ) + + result, paths = self.render_burns(job) + + self.assertEqual(paths, []) + self.assertFalse(result["burn_dxf_eligible"]) + self.assertEqual(len(result["unplaced"]), 1) + self.assertTrue( + any("did not fit" in warning.lower() for warning in result["burn_dxf_warnings"]) + ) + + def test_hole_outside_part_suppresses_burn_dxfs(self): + result, paths = self.render_burns( + job_with( + { + "name": "Plate With Invalid Hole", + "width": 8, + "height": 6, + "qty": 1, + "shape": "rect", + "holes": [{"dia": 2, "x": 0.5, "y": 3}], + } + ) + ) + + self.assertEqual(paths, []) + self.assertFalse(result["burn_dxf_eligible"]) + self.assertTrue( + any("outside" in warning.lower() for warning in result["burn_dxf_warnings"]) + ) + + +if __name__ == "__main__": + unittest.main() From c6eaa47dad922007b06b438017ac0d0ef9e2db61 Mon Sep 17 00:00:00 2001 From: Victor Garcia Date: Tue, 28 Jul 2026 11:55:17 -0600 Subject: [PATCH 03/15] docs(research): validate synthetic workflow premise --- docs/research/v0.3-workflow-evidence.md | 126 ++++++++++++++++++ tests/fixtures/workflows/README.md | 32 +++++ tests/fixtures/workflows/ambiguous_scope.csv | 3 + .../fixtures/workflows/blocked_plate_job.json | 28 ++++ tests/fixtures/workflows/member_only.csv | 4 + tests/fixtures/workflows/mixed_plate_job.json | 43 ++++++ 6 files changed, 236 insertions(+) create mode 100644 docs/research/v0.3-workflow-evidence.md create mode 100644 tests/fixtures/workflows/README.md create mode 100644 tests/fixtures/workflows/ambiguous_scope.csv create mode 100644 tests/fixtures/workflows/blocked_plate_job.json create mode 100644 tests/fixtures/workflows/member_only.csv create mode 100644 tests/fixtures/workflows/mixed_plate_job.json diff --git a/docs/research/v0.3-workflow-evidence.md b/docs/research/v0.3-workflow-evidence.md new file mode 100644 index 0000000..6c831bf --- /dev/null +++ b/docs/research/v0.3-workflow-evidence.md @@ -0,0 +1,126 @@ +# v0.3 Workflow Evidence + +## Decision + +Proceed with U2–U6 as a reliability milestone. Synthetic workflow trials confirm +that the dominant current failures are incompatible stage contracts, missing +process-level stop conditions, model-improvised RFQ generation, and absent artifact +lineage. Drawing ingestion and one-dimensional stock optimization remain valuable +follow-up work, but neither can make the advertised takeoff → nest → RFQ chain +reproducible without this foundation. + +This evidence uses only fixtures created from scratch under +`tests/fixtures/workflows/`. No private production, customer, vendor, project, +pricing, contract, inventory, or employee data was used. + +## Trials + +### 1. Member-only takeoff + +Input: `member_only.csv`, containing synthetic W-shape and HSS rows. + +Observed: + +- `validate-bom.py` accepts the designations, quantities, lengths, weights, and + grades. +- `calculate-weight.sh` produces deterministic weight totals and allowances. +- Neither command emits a reusable versioned handoff, source-revision evidence, + stable normalized IDs, or a run manifest. + +Implication: the takeoff calculations are useful, but U2, U5, and U6 remain needed +to make their result a traceable pipeline input. + +### 2. Mixed rectangular and irregular plate work + +Input: `mixed_plate_job.json`, containing exact rectangular geometry plus one +unresolved irregular profile. + +Observed: + +- The nester can place both part families by rectangular footprint. +- The U8 safety guard correctly marks burn output ineligible and reports the + irregular approximation. +- The current nest result is not the same model as the BOM CSV and carries no + source-row/revision lineage. +- The current RFQ handoff contains summary strings rather than a versioned purchase + contract. + +Implication: U1–U3 remain necessary. The product must preserve reference-only +geometry without allowing it to become geometry-verified downstream. + +### 3. Finite stock with an oversized part + +Input: `blocked_plate_job.json`, containing a required part larger than its only +available stock. + +Observed: + +- The nester reports the part as unplaced and the U8 guard suppresses burn DXFs. +- The direct CLI still completes without a shared blocked exit/outcome contract. +- With no plate opened, the current summary can report a zero-dollar known material + cost even though required material is unplaced. +- Nothing currently prevents an agent from manually continuing into an RFQ step + using incomplete material data. + +Implication: U3 and U5 remain necessary for independent verification and a +process-level stop gate. + +### 4. Ambiguous scope and purchasing intent + +Input: `ambiguous_scope.csv`, containing valid material rows whose notes do not +unambiguously define fabricated, purchased, reference-only, or by-others intent. + +Observed: + +- The current BOM validator checks material facts but has no typed scope or + purchase relationship. +- The RFQ instructions rely on description matching and agent interpretation. +- There is no deterministic RFQ generator; `recalc.py` only recalculates an + already-created workbook. + +Implication: U2 and U4 remain necessary. Ambiguous scope must become a visible +review finding rather than a guessed inclusion or exclusion. + +## Manual Reinterpretation Still Required + +Today an agent must manually: + +1. transform BOM rows into the unrelated nesting job format; +2. decide which items are fabricated parts, purchase stock, allowances, exclusions, + or by-others work; +3. decide whether on-hand stock should reduce purchase quantities; +4. reconstruct an RFQ workbook from prose; +5. carry warnings and assumptions between artifacts without a common identifier. + +These are the highest-risk recreation points because they affect quantities, +commercial drafts, and whether incomplete geometry appears ready for downstream +use. + +## CAM Compatibility + +No CAM product/version import acceptance was performed. v0.3 therefore makes no +named CAM compatibility claim. Documentation may describe the DXF layer, unit, and +origin contract only. A future claim requires a recorded synthetic import test that +verifies units, origin, closed profiles, hole classification, and rejection of +reference-only geometry for a specific product and version. + +## Required v0.3 Units + +- **U2:** required for stable identity, typed scope, source evidence, and shared + validation. +- **U3:** required for independent placement verification, material grouping, and + meaningful metrics. +- **U4:** required because no deterministic RFQ compiler exists. +- **U5:** required to enforce stop conditions and preserve lineage across stages. +- **U6:** required to keep tests, package contents, provenance, and public-data + controls reproducible at release. + +## Deferred Findings + +- Drawing/PDF ingestion remains deferred because the tested handoff failures occur + even with already-structured inputs. +- One-dimensional member optimization remains deferred because it is a distinct + engine; v0.3 must represent long-product purchasing intent without pretending to + optimize it. +- True-profile irregular nesting remains deferred. Irregular geometry stays + reference-only until a dedicated geometry workflow exists. diff --git a/tests/fixtures/workflows/README.md b/tests/fixtures/workflows/README.md new file mode 100644 index 0000000..453bb77 --- /dev/null +++ b/tests/fixtures/workflows/README.md @@ -0,0 +1,32 @@ +# Synthetic Workflow Fixtures + +These fixtures were created from scratch to test pi-steel's public workflow. They +are not copied, transformed, rounded, renamed, or anonymized from production, +customer, vendor, bid, drawing, takeoff, inventory, or RFQ data. + +| Fixture | Synthetic family | Purpose | +|---|---|---| +| `member_only.csv` | Mixed W-shape and HSS member takeoff | Exercises supported member validation and weight calculation. | +| `mixed_plate_job.json` | Rectangular and unresolved irregular plate parts | Exercises reference-only geometry and approximation propagation. | +| `blocked_plate_job.json` | Part larger than finite stock | Exercises unplaced-part diagnostics and the missing process-level stop contract. | +| `ambiguous_scope.csv` | Member rows with ambiguous scope and purchasing intent | Proves that current CSV validation cannot determine scope or purchase relationships. | + +The fixtures deliberately contain: + +- different section families and material grades; +- member and plate workflows; +- explicit synthetic scope text; +- both complete and blocked nesting cases; +- no company identity, contact details, pricing, commercial terms, or real project + identifiers. + +No fixture establishes CAM compatibility. A named CAM product/version may be added +only with a separate, recorded import acceptance check using synthetic geometry. + +## Provenance + +- Creator: StructuPath pi-steel maintainers +- Creation method: deliberately invented values and geometry +- Public-data review: 2026-07-28 +- Private source artifacts: none +- Release approval: required through the repository public-data check diff --git a/tests/fixtures/workflows/ambiguous_scope.csv b/tests/fixtures/workflows/ambiguous_scope.csv new file mode 100644 index 0000000..236374d --- /dev/null +++ b/tests/fixtures/workflows/ambiguous_scope.csv @@ -0,0 +1,3 @@ +Mark,Qty,Size,Grade,Length_ft,Unit_Wt_plf,Total_Wt_lbs,Connections,Notes +SYN-A1,1,L4X4X3/8,A36,8,9.8,,,REFERENCE ONLY +SYN-A2,2,C8X11.5,A36,10,11.5,,,Purchase relationship intentionally unspecified diff --git a/tests/fixtures/workflows/blocked_plate_job.json b/tests/fixtures/workflows/blocked_plate_job.json new file mode 100644 index 0000000..3b8825c --- /dev/null +++ b/tests/fixtures/workflows/blocked_plate_job.json @@ -0,0 +1,28 @@ +{ + "job_name": "SYNTHETIC-WORKFLOW-BLOCKED-PLATE", + "customer": "Example Customer", + "settings": { + "kerf_in": 0.05, + "part_gap_in": 0.2, + "edge_margin_in": 0.4, + "density_lb_in3": 0.2836 + }, + "stock": [ + { + "name": "Synthetic Plate B", + "width": 24, + "height": 18, + "thickness": 0.25, + "qty": 1 + } + ], + "parts": [ + { + "name": "SYN-OVERSIZE-1", + "width": 30, + "height": 20, + "qty": 1, + "shape": "rect" + } + ] +} diff --git a/tests/fixtures/workflows/member_only.csv b/tests/fixtures/workflows/member_only.csv new file mode 100644 index 0000000..a477de2 --- /dev/null +++ b/tests/fixtures/workflows/member_only.csv @@ -0,0 +1,4 @@ +Mark,Qty,Size,Grade,Length_ft,Unit_Wt_plf,Total_Wt_lbs,Connections,Notes +SYN-M1,2,W14X30,A992,18,30,,,Synthetic member +SYN-M2,3,W12X26,A992,12.5,26,,,Synthetic member +SYN-B1,4,HSS6X6X3/8,A500 Gr. C,9,17.27,,,Synthetic brace diff --git a/tests/fixtures/workflows/mixed_plate_job.json b/tests/fixtures/workflows/mixed_plate_job.json new file mode 100644 index 0000000..4bf3e2b --- /dev/null +++ b/tests/fixtures/workflows/mixed_plate_job.json @@ -0,0 +1,43 @@ +{ + "job_name": "SYNTHETIC-WORKFLOW-MIXED-PLATE", + "customer": "Example Customer", + "settings": { + "kerf_in": 0.05, + "part_gap_in": 0.2, + "edge_margin_in": 0.4, + "density_lb_in3": 0.2836 + }, + "stock": [ + { + "name": "Synthetic Plate A", + "width": 42, + "height": 24, + "thickness": 0.375, + "qty": 2 + } + ], + "parts": [ + { + "name": "SYN-RECT-1", + "width": 10, + "height": 6, + "qty": 3, + "shape": "rect", + "holes": [ + { + "dia": 0.75, + "x": 5, + "y": 3 + } + ] + }, + { + "name": "SYN-IRREGULAR-1", + "width": 8, + "height": 7, + "qty": 2, + "shape": "irregular", + "area": 31 + } + ] +} From c0be4b4b9269f94d2edae25a08f0f3a3f60ea6f1 Mon Sep 17 00:00:00 2001 From: Victor Garcia Date: Tue, 28 Jul 2026 12:02:30 -0600 Subject: [PATCH 04/15] feat(runtime): add shared manifest foundation --- DATA_PROVENANCE.md | 38 +++ package.json | 12 + pyproject.toml | 24 ++ requirements-dev.txt | 7 + requirements-tested.txt | 8 + requirements.txt | 4 + scripts/doctor.py | 127 ++++++++ skills/_shared/bootstrap.py | 32 ++ skills/_shared/pi_steel/__init__.py | 27 ++ skills/_shared/pi_steel/run_manifest.py | 306 ++++++++++++++++++ .../_shared/schemas/run-manifest.schema.json | 145 +++++++++ tests/test_installed_scripts.py | 56 ++++ tests/test_run_manifests.py | 142 ++++++++ tests/test_runtime_bootstrap.py | 123 +++++++ 14 files changed, 1051 insertions(+) create mode 100644 DATA_PROVENANCE.md create mode 100644 pyproject.toml create mode 100644 requirements-dev.txt create mode 100644 requirements-tested.txt create mode 100644 requirements.txt create mode 100644 scripts/doctor.py create mode 100644 skills/_shared/bootstrap.py create mode 100644 skills/_shared/pi_steel/__init__.py create mode 100644 skills/_shared/pi_steel/run_manifest.py create mode 100644 skills/_shared/schemas/run-manifest.schema.json create mode 100644 tests/test_installed_scripts.py create mode 100644 tests/test_run_manifests.py create mode 100644 tests/test_runtime_bootstrap.py diff --git a/DATA_PROVENANCE.md b/DATA_PROVENANCE.md new file mode 100644 index 0000000..865a452 --- /dev/null +++ b/DATA_PROVENANCE.md @@ -0,0 +1,38 @@ +# Data Provenance + +This file records the evidence and release decision for third-party datasets +shipped by pi-steel. The repository's MIT license covers StructuPath-authored +code and documentation; it must not be interpreted as granting rights in +third-party data. + +## AISC shapes database + +| Field | Recorded value | +| --- | --- | +| Shipped file | `skills/steel-takeoff/assets/aisc-shapes-database.json` | +| Claimed edition | AISC Shapes Database v16.0, consistent with the 16th Edition Steel Construction Manual | +| Rows in shipped JSON | 477 | +| SHA-256 | `5a7c975c4c290c34df6f7df3b4d0d0d13a00ef7c2b1f45097a49a78d245dcc91` | +| Upstream description | [AISC Shapes Database v16.0](https://www.aisc.org/aisc/publications/steel-construction-manual/aisc-shapes-database-v160/) | +| Release evidence | [AISC's August 14, 2023 companion-material announcement](https://www.aisc.org/news/aisc-releases-complementary-materials-for-the-16th-edition-steel-construction-manual/) | +| Repository introduction | Initial repository commit | +| Transformation history | Unknown; no source workbook, conversion script, field map, or contemporaneous checksum is present in repository history | +| Upstream file checksum | Not recorded; the checked-in JSON cannot currently be byte- or row-reconciled to a preserved source workbook | +| Redistribution permission | Unverified | +| Release gate | Blocked pending affirmative redistribution evidence or replacement with a dataset whose redistribution terms are documented | + +The upstream pages establish that AISC publishes the v16.0 spreadsheet as a +downloadable digital supplement and describe its relationship to the 16th +Edition Manual. They do not, based on the evidence recorded here, grant +permission to redistribute a transformed copy in another public package. +Availability without charge is not treated as redistribution permission. + +Until the release gate is resolved: + +- Do not claim that the repository's MIT license covers the shapes data. +- Do not claim that the checked-in JSON is an independently reproducible + transformation of the official workbook. +- Do not publish a new release containing this file without documented approval + or a documented replacement decision. +- Continue verifying the recorded checksum so an unexplained data change cannot + pass unnoticed. diff --git a/package.json b/package.json index 8f16237..90e4a01 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,12 @@ "engines": { "node": ">=20.6.0" }, + "scripts": { + "doctor": "python3 scripts/doctor.py", + "test": "python3 -m pytest", + "test:runtime": "python3 -m pytest tests/test_runtime_bootstrap.py tests/test_run_manifests.py tests/test_installed_scripts.py", + "pack:dry-run": "npm pack --dry-run" + }, "pi": { "skills": [ "./skills" @@ -40,6 +46,12 @@ "!skills/**/__pycache__", "!skills/**/*.pyc", "!skills/**/company-profile.json", + "scripts/doctor.py", + "pyproject.toml", + "requirements.txt", + "requirements-tested.txt", + "requirements-dev.txt", + "DATA_PROVENANCE.md", "README.md", "PUBLIC_DATA_POLICY.md", "LICENSE" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..7b69f59 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,24 @@ +[project] +name = "pi-steel-runtime" +version = "0.2.2" +description = "Python runtime dependencies and test configuration for pi-steel" +requires-python = ">=3.11,<3.14" +dependencies = [ + "jsonschema>=4.23,<5", + "openpyxl>=3.1,<4", + "pandas>=2.2,<3", +] + +[project.optional-dependencies] +render = [ + "ezdxf>=1.3,<2", + "matplotlib>=3.9,<4", + "numpy>=2.1,<3", +] +test = [ + "pytest>=8.3,<10", +] + +[tool.pytest.ini_options] +addopts = "-ra" +testpaths = ["tests"] diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..229facc --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,7 @@ +-r requirements.txt + +# Test and optional rendering capabilities exercised by the development suite. +pytest>=8.3,<10 +ezdxf>=1.3,<2 +matplotlib>=3.9,<4 +numpy>=2.1,<3 diff --git a/requirements-tested.txt b/requirements-tested.txt new file mode 100644 index 0000000..f7ff4d3 --- /dev/null +++ b/requirements-tested.txt @@ -0,0 +1,8 @@ +# Exact environment exercised during the v0.3 runtime bootstrap work. +jsonschema==4.26.0 +openpyxl==3.1.5 +pandas==2.3.3 +pytest==9.0.2 +ezdxf==1.4.4 +matplotlib==3.10.9 +numpy==2.4.1 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..5a1e10e --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +# Required by the validated estimate and RFQ runtime. +jsonschema>=4.23,<5 +openpyxl>=3.1,<4 +pandas>=2.2,<3 diff --git a/scripts/doctor.py b/scripts/doctor.py new file mode 100644 index 0000000..d08495a --- /dev/null +++ b/scripts/doctor.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +"""Diagnose the supported pi-steel Python runtime and optional capabilities.""" + +from __future__ import annotations + +import argparse +import importlib.util +import json +import shutil +import sys +from importlib.metadata import PackageNotFoundError, version +from pathlib import Path + + +SHARED_ROOT = Path(__file__).resolve().parents[1] / "skills" / "_shared" +sys.path.insert(0, str(SHARED_ROOT)) +from bootstrap import bootstrap_shared # noqa: E402 + +bootstrap_shared(__file__) +from pi_steel import outcome_exit_code # noqa: E402 + + +REQUIRED_MODULES = { + "jsonschema": "contract validation", + "openpyxl": "RFQ workbook generation", + "pandas": "spreadsheet adapters", +} +OPTIONAL_CAPABILITIES = { + "nest_rendering": ("ezdxf", "matplotlib", "numpy"), +} + + +def _module_status(name, module_finder): + available = module_finder(name) is not None + item = {"name": name, "kind": "python_module", "available": available} + if available: + try: + item["version"] = version(name) + except PackageNotFoundError: + item["version"] = "unknown" + return item + + +def diagnose( + *, + version_info=None, + module_finder=importlib.util.find_spec, + command_finder=shutil.which, +): + current = version_info or sys.version_info + python_supported = (3, 11) <= tuple(current[:2]) < (3, 14) + required = [ + { + "name": "python", + "kind": "runtime", + "available": python_supported, + "version": ".".join(str(value) for value in current[:3]), + "supported": ">=3.11,<3.14", + } + ] + for name, purpose in REQUIRED_MODULES.items(): + item = _module_status(name, module_finder) + item["purpose"] = purpose + required.append(item) + + jq_path = command_finder("jq") + required.append( + { + "name": "jq", + "kind": "command", + "available": jq_path is not None, + "path": jq_path, + "purpose": "AISC shape lookup helpers", + } + ) + + optional = {} + for capability, modules in OPTIONAL_CAPABILITIES.items(): + checks = [_module_status(name, module_finder) for name in modules] + optional[capability] = { + "available": all(check["available"] for check in checks), + "dependencies": checks, + } + + office_path = command_finder("soffice") or command_finder("libreoffice") + optional["formula_baking"] = { + "available": office_path is not None, + "dependencies": [ + { + "name": "libreoffice", + "kind": "command", + "available": office_path is not None, + "path": office_path, + } + ], + } + + missing = [item["name"] for item in required if not item["available"]] + outcome = "dependency_missing" if missing else "ready" + return { + "schema_version": "1.0.0", + "run_outcome": outcome, + "exit_code": outcome_exit_code(outcome), + "required": required, + "optional_capabilities": optional, + "missing_required": missing, + } + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--json", + action="store_true", + help="emit compact JSON instead of indented JSON", + ) + args = parser.parse_args(argv) + report = diagnose() + if args.json: + print(json.dumps(report, sort_keys=True, separators=(",", ":"))) + else: + print(json.dumps(report, indent=2, sort_keys=True)) + return report["exit_code"] + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/_shared/bootstrap.py b/skills/_shared/bootstrap.py new file mode 100644 index 0000000..42058d9 --- /dev/null +++ b/skills/_shared/bootstrap.py @@ -0,0 +1,32 @@ +"""Relocatable import bootstrap for scripts shipped inside the npm package.""" + +from __future__ import annotations + +import sys +from pathlib import Path + + +class BootstrapError(RuntimeError): + """Raised when an entry point cannot locate the shipped skills tree.""" + + +def find_skills_root(entry_file: str | Path) -> Path: + """Find ``skills/`` relative to an entry point, without using the cwd.""" + entry = Path(entry_file).resolve() + for parent in entry.parents: + if parent.name == "skills" and (parent / "_shared").is_dir(): + return parent + candidate = parent / "skills" + if (candidate / "_shared").is_dir(): + return candidate + raise BootstrapError(f"cannot locate the shipped skills directory from {entry.name}") + + +def bootstrap_shared(entry_file: str | Path) -> Path: + """Put the shipped shared module directory on ``sys.path`` and return skills root.""" + skills_root = find_skills_root(entry_file) + shared_root = skills_root / "_shared" + shared_text = str(shared_root) + if shared_text not in sys.path: + sys.path.insert(0, shared_text) + return skills_root diff --git a/skills/_shared/pi_steel/__init__.py b/skills/_shared/pi_steel/__init__.py new file mode 100644 index 0000000..e61e72c --- /dev/null +++ b/skills/_shared/pi_steel/__init__.py @@ -0,0 +1,27 @@ +"""Shared deterministic runtime primitives for pi-steel skills.""" + +from .run_manifest import ( + ARTIFACT_READINESS, + OUTCOME_EXIT_CODES, + PACKAGE_STATUSES, + RUN_OUTCOMES, + ManifestError, + RunPublisher, + canonical_json_bytes, + outcome_exit_code, + sha256_bytes, + sha256_file, +) + +__all__ = [ + "ARTIFACT_READINESS", + "OUTCOME_EXIT_CODES", + "PACKAGE_STATUSES", + "RUN_OUTCOMES", + "ManifestError", + "RunPublisher", + "canonical_json_bytes", + "outcome_exit_code", + "sha256_bytes", + "sha256_file", +] diff --git a/skills/_shared/pi_steel/run_manifest.py b/skills/_shared/pi_steel/run_manifest.py new file mode 100644 index 0000000..d467b52 --- /dev/null +++ b/skills/_shared/pi_steel/run_manifest.py @@ -0,0 +1,306 @@ +"""Run manifests and isolated atomic artifact publication.""" + +from __future__ import annotations + +import copy +import hashlib +import json +import os +import re +import shutil +import tempfile +import uuid +from datetime import datetime, timezone +from pathlib import Path, PurePosixPath +from typing import Any + + +MANIFEST_SCHEMA_VERSION = "1.0.0" +RUN_OUTCOMES = frozenset( + { + "ready", + "review_required", + "blocked", + "dependency_missing", + "usage_or_internal_error", + } +) +OUTCOME_EXIT_CODES = { + "ready": 0, + "review_required": 2, + "blocked": 3, + "dependency_missing": 4, + "usage_or_internal_error": 1, +} +PACKAGE_STATUSES = frozenset( + { + "draft", + "review_required", + "validated", + "nested_partial", + "nest_verified", + "rfq_draft_review_required", + "rfq_ready_for_review", + } +) +ARTIFACT_READINESS = frozenset( + {"geometry_verified", "reference_only", "draft", "diagnostic"} +) +_RUN_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$") +_HASH = re.compile(r"^[0-9a-f]{64}$") + + +class ManifestError(RuntimeError): + """Raised when a run cannot be published safely.""" + + +def canonical_json_bytes(value: Any) -> bytes: + """Return a stable UTF-8 JSON representation suitable for hashing.""" + return ( + json.dumps( + value, + ensure_ascii=False, + allow_nan=False, + separators=(",", ":"), + sort_keys=True, + ) + + "\n" + ).encode("utf-8") + + +def sha256_bytes(value: bytes) -> str: + return hashlib.sha256(value).hexdigest() + + +def sha256_file(path: str | Path) -> str: + digest = hashlib.sha256() + with Path(path).open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def outcome_exit_code(outcome: str) -> int: + try: + return OUTCOME_EXIT_CODES[outcome] + except KeyError as exc: + raise ManifestError(f"unknown run outcome: {outcome}") from exc + + +def semantic_manifest_projection(manifest: dict[str, Any]) -> dict[str, Any]: + """Remove volatile fields and the self-referential hash before semantic hashing.""" + projection = copy.deepcopy(manifest) + for field in ("semantic_hash", "run_id", "created_at"): + projection.pop(field, None) + return projection + + +def _safe_relative_path(value: str | Path) -> PurePosixPath: + text = Path(value).as_posix() + path = PurePosixPath(text) + if ( + text in {"", "."} + or path.is_absolute() + or ".." in path.parts + or path.name in {"run-manifest.json", "latest-run.json"} + ): + raise ManifestError(f"unsafe or reserved artifact path: {text}") + return path + + +def _utc_now() -> str: + return datetime.now(timezone.utc).isoformat().replace("+00:00", "Z") + + +class RunPublisher: + """Stage one isolated run and publish it with an integrity manifest.""" + + def __init__( + self, + destination: str | Path, + *, + stage: str, + run_outcome: str, + package_status: str, + input_hash: str, + configuration_hash: str, + schema_versions: dict[str, str], + tool_versions: dict[str, str], + explicit_dates: dict[str, str], + warnings: list[Any] | None = None, + approximations: list[Any] | None = None, + run_id: str | None = None, + created_at: str | None = None, + ) -> None: + if run_outcome not in RUN_OUTCOMES: + raise ManifestError(f"unknown run outcome: {run_outcome}") + if package_status not in PACKAGE_STATUSES: + raise ManifestError(f"unknown package status: {package_status}") + if not _HASH.fullmatch(input_hash) or not _HASH.fullmatch(configuration_hash): + raise ManifestError("input and configuration hashes must be SHA-256 hex") + + self.destination = Path(destination).resolve() + self.run_id = run_id or ( + datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S.%fZ") + + "-" + + uuid.uuid4().hex[:12] + ) + if not _RUN_ID.fullmatch(self.run_id): + raise ManifestError(f"invalid run id: {self.run_id}") + + self._metadata = { + "schema_version": MANIFEST_SCHEMA_VERSION, + "run_id": self.run_id, + "stage": stage, + "run_outcome": run_outcome, + "package_status": package_status, + "created_at": created_at or _utc_now(), + "schema_versions": dict(schema_versions), + "tool_versions": dict(tool_versions), + "input_hash": input_hash, + "configuration_hash": configuration_hash, + "explicit_dates": dict(explicit_dates), + "warnings": list(warnings or []), + "approximations": list(approximations or []), + } + self._artifacts: dict[str, dict[str, Any]] = {} + self._published = False + + staging_parent = self.destination / ".staging" + staging_parent.mkdir(parents=True, exist_ok=True) + self.staging_path = Path( + tempfile.mkdtemp(prefix=f"{self.run_id}-", dir=staging_parent) + ) + self.final_path = self.destination / "runs" / self.run_id + + def __enter__(self) -> "RunPublisher": + return self + + def __exit__(self, exc_type, exc, traceback) -> None: + if not self._published and self.staging_path.exists(): + shutil.rmtree(self.staging_path) + + def path_for(self, relative_path: str | Path) -> Path: + relative = _safe_relative_path(relative_path) + target = self.staging_path.joinpath(*relative.parts) + target.parent.mkdir(parents=True, exist_ok=True) + return target + + def register_artifact( + self, + relative_path: str | Path, + *, + readiness: str, + media_type: str | None = None, + ) -> Path: + if readiness not in ARTIFACT_READINESS: + raise ManifestError(f"unknown artifact readiness: {readiness}") + relative = _safe_relative_path(relative_path) + text = relative.as_posix() + if text in self._artifacts: + raise ManifestError(f"artifact already registered: {text}") + record: dict[str, Any] = {"readiness": readiness} + if media_type: + record["media_type"] = media_type + self._artifacts[text] = record + return self.path_for(relative) + + def write_bytes( + self, + relative_path: str | Path, + value: bytes, + *, + readiness: str, + media_type: str | None = None, + ) -> Path: + target = self.register_artifact( + relative_path, readiness=readiness, media_type=media_type + ) + target.write_bytes(value) + return target + + def write_json( + self, + relative_path: str | Path, + value: Any, + *, + readiness: str, + ) -> Path: + return self.write_bytes( + relative_path, + canonical_json_bytes(value), + readiness=readiness, + media_type="application/json", + ) + + def write_qa_report(self, value: Any) -> Path: + """Register the standard machine-readable QA artifact for a stage run.""" + return self.write_json("qa-report.json", value, readiness="diagnostic") + + def publish(self) -> Path: + if self._published: + raise ManifestError("run already published") + if self.final_path.exists(): + raise ManifestError(f"run already exists: {self.run_id}") + if ( + self._metadata["run_outcome"] in {"ready", "review_required", "blocked"} + and "qa-report.json" not in self._artifacts + ): + raise ManifestError( + f"{self._metadata['run_outcome']} runs require qa-report.json" + ) + + actual_files = set() + for path in self.staging_path.rglob("*"): + if path.is_symlink(): + raise ManifestError("artifact symlinks are not allowed") + if path.is_file(): + actual_files.add(path.relative_to(self.staging_path).as_posix()) + registered_files = set(self._artifacts) + if actual_files != registered_files: + missing = sorted(registered_files - actual_files) + extra = sorted(actual_files - registered_files) + raise ManifestError( + f"artifact allow-list mismatch; missing={missing}, unregistered={extra}" + ) + + artifact_records = [] + for relative in sorted(self._artifacts): + path = self.staging_path / relative + record = { + "path": relative, + "sha256": sha256_file(path), + "size_bytes": path.stat().st_size, + **self._artifacts[relative], + } + artifact_records.append(record) + + manifest = {**self._metadata, "artifacts": artifact_records} + manifest["semantic_hash"] = sha256_bytes( + canonical_json_bytes(semantic_manifest_projection(manifest)) + ) + manifest_path = self.staging_path / "run-manifest.json" + manifest_path.write_bytes(canonical_json_bytes(manifest)) + + self.final_path.parent.mkdir(parents=True, exist_ok=True) + os.replace(self.staging_path, self.final_path) + manifest_hash = sha256_file(self.final_path / "run-manifest.json") + pointer = { + "schema_version": MANIFEST_SCHEMA_VERSION, + "run_id": self.run_id, + "run_directory": f"runs/{self.run_id}", + "manifest_sha256": manifest_hash, + } + pointer_fd, pointer_name = tempfile.mkstemp( + prefix=".latest-run-", dir=self.destination + ) + try: + with os.fdopen(pointer_fd, "wb") as stream: + stream.write(canonical_json_bytes(pointer)) + os.replace(pointer_name, self.destination / "latest-run.json") + finally: + if os.path.exists(pointer_name): + os.unlink(pointer_name) + + self._published = True + return self.final_path diff --git a/skills/_shared/schemas/run-manifest.schema.json b/skills/_shared/schemas/run-manifest.schema.json new file mode 100644 index 0000000..b03873c --- /dev/null +++ b/skills/_shared/schemas/run-manifest.schema.json @@ -0,0 +1,145 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://structupath.ai/schemas/pi-steel/run-manifest-1.0.0.json", + "title": "Pi Steel Run Manifest", + "type": "object", + "additionalProperties": false, + "required": [ + "schema_version", + "run_id", + "stage", + "run_outcome", + "package_status", + "created_at", + "schema_versions", + "tool_versions", + "input_hash", + "configuration_hash", + "explicit_dates", + "warnings", + "approximations", + "artifacts", + "semantic_hash" + ], + "properties": { + "schema_version": { + "const": "1.0.0" + }, + "run_id": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9._-]*$" + }, + "stage": { + "type": "string", + "minLength": 1 + }, + "run_outcome": { + "enum": [ + "ready", + "review_required", + "blocked", + "dependency_missing", + "usage_or_internal_error" + ] + }, + "package_status": { + "enum": [ + "draft", + "review_required", + "validated", + "nested_partial", + "nest_verified", + "rfq_draft_review_required", + "rfq_ready_for_review" + ] + }, + "created_at": { + "type": "string", + "format": "date-time" + }, + "schema_versions": { + "$ref": "#/$defs/versionMap" + }, + "tool_versions": { + "$ref": "#/$defs/versionMap" + }, + "input_hash": { + "$ref": "#/$defs/sha256" + }, + "configuration_hash": { + "$ref": "#/$defs/sha256" + }, + "explicit_dates": { + "type": "object", + "additionalProperties": { + "type": "string", + "format": "date" + } + }, + "warnings": { + "type": "array", + "items": {} + }, + "approximations": { + "type": "array", + "items": {} + }, + "artifacts": { + "type": "array", + "items": { + "$ref": "#/$defs/artifact" + } + }, + "semantic_hash": { + "$ref": "#/$defs/sha256" + } + }, + "$defs": { + "sha256": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "versionMap": { + "type": "object", + "additionalProperties": { + "type": "string", + "minLength": 1 + } + }, + "artifact": { + "type": "object", + "additionalProperties": false, + "required": [ + "path", + "sha256", + "size_bytes", + "readiness" + ], + "properties": { + "path": { + "type": "string", + "minLength": 1 + }, + "sha256": { + "$ref": "#/$defs/sha256" + }, + "size_bytes": { + "type": "integer", + "minimum": 0 + }, + "readiness": { + "enum": [ + "geometry_verified", + "reference_only", + "draft", + "diagnostic" + ] + }, + "media_type": { + "type": "string", + "minLength": 1 + } + } + } + } +} diff --git a/tests/test_installed_scripts.py b/tests/test_installed_scripts.py new file mode 100644 index 0000000..14729f1 --- /dev/null +++ b/tests/test_installed_scripts.py @@ -0,0 +1,56 @@ +import json +import os +import subprocess +import sys +import tarfile +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_packed_npm_artifact_contains_runtime_and_runs_doctor(tmp_path): + pack = subprocess.run( + ["npm", "pack", "--json", "--pack-destination", str(tmp_path)], + cwd=ROOT, + capture_output=True, + text=True, + ) + assert pack.returncode == 0, pack.stdout + pack.stderr + metadata = json.loads(pack.stdout) + archive = tmp_path / metadata[0]["filename"] + + with tarfile.open(archive) as package: + members = set(package.getnames()) + package.extractall(tmp_path / "unpacked", filter="data") + + expected = { + "package/scripts/doctor.py", + "package/skills/_shared/bootstrap.py", + "package/skills/_shared/pi_steel/__init__.py", + "package/skills/_shared/pi_steel/run_manifest.py", + "package/skills/_shared/schemas/run-manifest.schema.json", + "package/pyproject.toml", + "package/requirements.txt", + "package/requirements-tested.txt", + "package/requirements-dev.txt", + "package/DATA_PROVENANCE.md", + } + assert expected <= members + assert not any(name.startswith("package/tests/") for name in members) + assert not any("__pycache__" in name or name.endswith(".pyc") for name in members) + assert not any("company-profile.json" in name for name in members) + + environment = os.environ.copy() + environment.pop("PYTHONPATH", None) + installed_root = tmp_path / "unpacked" / "package" + doctor = subprocess.run( + [sys.executable, installed_root / "scripts" / "doctor.py", "--json"], + cwd=tmp_path, + env=environment, + capture_output=True, + text=True, + ) + + assert doctor.returncode == 0, doctor.stdout + doctor.stderr + assert json.loads(doctor.stdout)["run_outcome"] == "ready" diff --git a/tests/test_run_manifests.py b/tests/test_run_manifests.py new file mode 100644 index 0000000..30c4aa3 --- /dev/null +++ b/tests/test_run_manifests.py @@ -0,0 +1,142 @@ +import json +import sys +from pathlib import Path + +import jsonschema +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +SHARED = ROOT / "skills" / "_shared" +sys.path.insert(0, str(SHARED)) + +from pi_steel.run_manifest import ( + ManifestError, + RunPublisher, + canonical_json_bytes, + semantic_manifest_projection, + sha256_bytes, + sha256_file, +) + + +INPUT_HASH = sha256_bytes(b"SYNTHETIC-INPUT") +CONFIGURATION_HASH = sha256_bytes(b"SYNTHETIC-CONFIGURATION") +CREATED_AT = "2026-07-28T12:00:00Z" + + +def publisher( + destination, + run_id, + *, + outcome="ready", + status="validated", + created_at=CREATED_AT, +): + return RunPublisher( + destination, + stage="runtime_test", + run_outcome=outcome, + package_status=status, + input_hash=INPUT_HASH, + configuration_hash=CONFIGURATION_HASH, + schema_versions={"run_manifest": "1.0.0"}, + tool_versions={"pi_steel": "0.3.0"}, + explicit_dates={"as_of": "2026-07-28"}, + run_id=run_id, + created_at=created_at, + ) + + +def load_json(path): + return json.loads(Path(path).read_text(encoding="utf-8")) + + +def test_published_manifest_validates_and_hashes_allow_listed_artifacts(tmp_path): + with publisher(tmp_path, "SYNTHETIC-RUN-READY") as run: + run.write_qa_report( + {"project_id": "SYNTHETIC-RUNTIME", "findings": []} + ) + final_path = run.publish() + + manifest_path = final_path / "run-manifest.json" + manifest = load_json(manifest_path) + schema = load_json( + ROOT / "skills" / "_shared" / "schemas" / "run-manifest.schema.json" + ) + jsonschema.Draft202012Validator( + schema, format_checker=jsonschema.FormatChecker() + ).validate(manifest) + + assert [artifact["path"] for artifact in manifest["artifacts"]] == [ + "qa-report.json" + ] + artifact = manifest["artifacts"][0] + assert artifact["sha256"] == sha256_file(final_path / artifact["path"]) + assert manifest["semantic_hash"] == sha256_bytes( + canonical_json_bytes(semantic_manifest_projection(manifest)) + ) + + +def test_ready_then_blocked_runs_publish_isolated_and_latest_points_to_blocked(tmp_path): + with publisher(tmp_path, "SYNTHETIC-RUN-READY") as ready: + ready.write_qa_report({"outcome": "ready"}) + ready_path = ready.publish() + + with publisher( + tmp_path, + "SYNTHETIC-RUN-BLOCKED", + outcome="blocked", + status="draft", + ) as blocked: + blocked.write_qa_report( + {"outcome": "blocked", "errors": ["synthetic blocker"]} + ) + blocked_path = blocked.publish() + + pointer = load_json(tmp_path / "latest-run.json") + assert ready_path.exists() + assert blocked_path.exists() + assert ready_path != blocked_path + assert pointer["run_id"] == "SYNTHETIC-RUN-BLOCKED" + assert pointer["run_directory"] == "runs/SYNTHETIC-RUN-BLOCKED" + assert pointer["manifest_sha256"] == sha256_file( + blocked_path / "run-manifest.json" + ) + assert load_json(ready_path / "qa-report.json")["outcome"] == "ready" + assert load_json(blocked_path / "qa-report.json")["outcome"] == "blocked" + + +def test_semantic_hash_excludes_volatile_run_identity(tmp_path): + semantic_hashes = [] + runs = ( + ("SYNTHETIC-RUN-A", "2026-07-28T12:00:00Z"), + ("SYNTHETIC-RUN-B", "2026-07-28T12:01:00Z"), + ) + for run_id, created_at in runs: + with publisher(tmp_path, run_id, created_at=created_at) as run: + run.write_qa_report({"project_id": "SYNTHETIC-RUNTIME"}) + final_path = run.publish() + semantic_hashes.append(load_json(final_path / "run-manifest.json")["semantic_hash"]) + + assert semantic_hashes[0] == semantic_hashes[1] + + +def test_publication_rejects_unregistered_and_unsafe_artifacts(tmp_path): + with publisher(tmp_path, "SYNTHETIC-RUN-UNREGISTERED") as run: + run.write_qa_report({"findings": []}) + run.path_for("unregistered.txt").write_text("synthetic", encoding="utf-8") + with pytest.raises(ManifestError, match="allow-list mismatch"): + run.publish() + + with publisher(tmp_path, "SYNTHETIC-RUN-TRAVERSAL") as run: + with pytest.raises(ManifestError, match="unsafe"): + run.write_bytes( + "../outside.txt", b"synthetic", readiness="diagnostic" + ) + + +def test_ready_publication_requires_qa_report(tmp_path): + with publisher(tmp_path, "SYNTHETIC-RUN-NO-QA") as run: + with pytest.raises(ManifestError, match="require qa-report.json"): + run.publish() diff --git a/tests/test_runtime_bootstrap.py b/tests/test_runtime_bootstrap.py new file mode 100644 index 0000000..526b441 --- /dev/null +++ b/tests/test_runtime_bootstrap.py @@ -0,0 +1,123 @@ +import importlib.util +import json +import os +import subprocess +import sys +from pathlib import Path + +import openpyxl + + +ROOT = Path(__file__).resolve().parents[1] +SHARED = ROOT / "skills" / "_shared" +sys.path.insert(0, str(SHARED)) + +from bootstrap import bootstrap_shared, find_skills_root + + +def run_from(cwd, *command): + environment = os.environ.copy() + environment.pop("PYTHONPATH", None) + return subprocess.run( + [str(value) for value in command], + cwd=cwd, + env=environment, + capture_output=True, + text=True, + ) + + +def load_doctor(): + path = ROOT / "scripts" / "doctor.py" + spec = importlib.util.spec_from_file_location("pi_steel_doctor", path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +def test_bootstrap_finds_skills_from_skill_and_root_entrypoints(): + nest_script = ROOT / "skills" / "steel-nest" / "scripts" / "nest.py" + doctor_script = ROOT / "scripts" / "doctor.py" + + assert find_skills_root(nest_script) == ROOT / "skills" + assert find_skills_root(doctor_script) == ROOT / "skills" + assert bootstrap_shared(nest_script) == ROOT / "skills" + + import pi_steel + + assert pi_steel.outcome_exit_code("dependency_missing") == 4 + + +def test_current_python_entrypoints_work_from_an_arbitrary_cwd(tmp_path): + bom_path = tmp_path / "synthetic-bom.csv" + bom_path.write_text( + "Mark,Qty,Size,Grade,Length_ft,Unit_Wt_plf,Total_Wt_lbs,Connections,Notes\n" + "SYNTHETIC-M1,1,W14X30,A992,10,30,300,None,synthetic fixture\n", + encoding="utf-8", + ) + workbook_path = tmp_path / "synthetic-workbook.xlsx" + workbook = openpyxl.Workbook() + workbook.active["A1"] = "SYNTHETIC-RUNTIME" + workbook.save(workbook_path) + + nest = run_from( + tmp_path, + sys.executable, + ROOT / "skills" / "steel-nest" / "scripts" / "nest.py", + "--help", + ) + validate = run_from( + tmp_path, + sys.executable, + ROOT / "skills" / "steel-takeoff" / "scripts" / "validate-bom.py", + bom_path, + ) + recalc = run_from( + tmp_path, + sys.executable, + ROOT / "skills" / "steel-rfq" / "scripts" / "recalc.py", + workbook_path, + ) + doctor = run_from( + tmp_path, + sys.executable, + ROOT / "scripts" / "doctor.py", + "--json", + ) + + assert nest.returncode == 0, nest.stderr + assert validate.returncode == 0, validate.stdout + validate.stderr + assert recalc.returncode == 0, recalc.stdout + recalc.stderr + assert doctor.returncode == 0, doctor.stdout + doctor.stderr + assert json.loads(doctor.stdout)["run_outcome"] == "ready" + + +def test_doctor_returns_machine_readable_dependency_missing_for_unsupported_python(): + doctor = load_doctor() + report = doctor.diagnose( + version_info=(3, 10, 9), + module_finder=lambda _name: object(), + command_finder=lambda name: "synthetic-jq" if name == "jq" else None, + ) + + assert report["run_outcome"] == "dependency_missing" + assert report["exit_code"] == 4 + assert "python" in report["missing_required"] + + +def test_doctor_reports_optional_capabilities_without_blocking_base_runtime(): + doctor = load_doctor() + + def module_finder(name): + return object() if name in doctor.REQUIRED_MODULES else None + + report = doctor.diagnose( + version_info=(3, 12, 1), + module_finder=module_finder, + command_finder=lambda name: "synthetic-jq" if name == "jq" else None, + ) + + assert report["run_outcome"] == "ready" + assert report["optional_capabilities"]["nest_rendering"]["available"] is False + assert report["optional_capabilities"]["formula_baking"]["available"] is False From 542b24be328ea8e462a6e20ee1ca868a123259a1 Mon Sep 17 00:00:00 2001 From: Victor Garcia Date: Tue, 28 Jul 2026 12:09:08 -0600 Subject: [PATCH 05/15] feat(nest): publish safe manifested DXF runs --- README.md | 6 +- skills/steel-nest/SKILL.md | 41 +- skills/steel-nest/scripts/nest.py | 397 +++++++++++++++--- tests/fixtures/nest/README.md | 9 + .../nest/irregular-reference-only.json | 29 ++ tests/test_nest_outputs.py | 284 +++++++++++++ 6 files changed, 687 insertions(+), 79 deletions(-) create mode 100644 tests/fixtures/nest/README.md create mode 100644 tests/fixtures/nest/irregular-reference-only.json create mode 100644 tests/test_nest_outputs.py diff --git a/README.md b/README.md index 8c3daf7..ce1f664 100644 --- a/README.md +++ b/README.md @@ -32,10 +32,14 @@ Ask your agent things like: The plate-layout step CAM software does, minus the CAM seat: MaxRects bin-packing of parts onto stock plates with kerf/gap/edge-margin spacing, holes and rectangular cutouts, yield/scrap/reusable-drop numbers, and material cost. Outputs include a labeled layout (PDF + PNG per plate), a cut list, and an explicitly named all-sheets reference DXF. -Per-sheet `burn_plate_N.dxf` files are emitted only for a complete rectangular nest whose supported holes remain inside their parts. Any irregular part, unplaced part, or out-of-bounds hole suppresses burn DXFs for the whole job and leaves the estimating/reference artifacts available with an explicit warning. +Per-sheet `burn_plate_N.dxf` files are emitted only for a complete rectangular nest whose supported holes remain inside their parts. Those files contain cut entities only: closed outlines on `PROFILE` and holes/cutouts on `HOLES`. Sheet outlines and labels remain in clearly named reference files. Any irregular part, unplaced part, or out-of-bounds hole suppresses burn DXFs for the whole job and leaves the safe estimating/reference artifacts available with an explicit warning. Honest about its limits: rectangular parts nest exactly; irregular parts nest by bounding box (flagged, never hidden); reference DXFs are not cutting instructions; and the package deliberately does **not** emit G-code. Kerf compensation, lead-ins, pierce points, and machine-specific verification belong to the table's real CAM and post-processor. +Each command publishes an isolated run under the requested output root and updates +`latest-run.json`. Exit `0` is geometry-verified, `2` requires review, and `3` is +blocked. No named CAM compatibility is claimed. + > "How many sheets does this job need?" > "Nest these parts on 96×48 plate and give me the yield" > "Lay this out for the burn table" diff --git a/skills/steel-nest/SKILL.md b/skills/steel-nest/SKILL.md index 5e49d39..019c471 100644 --- a/skills/steel-nest/SKILL.md +++ b/skills/steel-nest/SKILL.md @@ -7,9 +7,9 @@ description: "Nest steel parts onto stock plates and estimate material — the p ## What This Skill Does -This is the in-house version of the plate-nesting step a CAM package (SigmaNEST / Hypertherm / FANUC) performs: it takes a list of parts and the plate stock on hand, packs the parts onto as few plates as possible, and tells you the yield, the drops you can reuse, the material weight, and the cost. It also draws the layout and exports a DXF. +This is an estimating-oriented version of the plate-nesting step performed by CAM software: it takes a list of parts and the plate stock on hand, packs the parts onto as few plates as possible, and tells you the yield, the drops you can reuse, the material weight, and the cost. It also draws the layout and exports guarded reference or cut-geometry DXFs. -It exists so that the moment an order lands, anyone in the shop can get a fast, repeatable material number for quoting and a layout the cutter can follow — without waiting on the CAM seat. +It exists so an estimator can get a fast, repeatable material number and a reviewable layout. It does not replace CAM setup or operator verification. ## What It Does Well vs. What It Doesn't @@ -18,12 +18,12 @@ Be honest with the user about the boundary — it protects the shop from over-tr **Reliable:** - Rectangular / plate-blank parts nest **exactly** (MaxRects bin-packing with rotation). - Multiple plate sizes, kerf + gap spacing, edge margin (grip/clamp keep-out). -- **Holes and rectangular cutouts** on any part — subtracted from weight/cost, rotated with the part, and cut as real geometry in the output. +- **Holes and rectangular cutouts** on verified rectangular parts — subtracted from weight/cost, rotated with the part, and emitted as cut geometry only when the complete job passes the output gate. - Yield %, scrap weight, largest reusable **drop** per plate. - Material weight and cost (by $/lb — which also values scrap — or by $/sheet). - Labeled layout (PDF + one PNG per plate). -- **Reference file**: `reference_nest.dxf`, an all-sheets estimating/layout reference. -- **Guarded burn-table files**: one DXF per sheet (`burn_plate_N.dxf`) with part outlines on `PROFILE` and holes on `HOLES`, origin at the sheet corner, but only when every part is rectangular, every required part fits, and every supported hole stays inside its part. +- **Reference files**: `reference_nest.dxf` plus `reference_plate_N.dxf`, with sheet outlines, bounding boxes, holes, and labels for estimating review. +- **Guarded cut-geometry files**: one DXF per sheet (`burn_plate_N.dxf`) containing only closed part outlines on `PROFILE` and holes/cutouts on `HOLES`, with origin at the sheet corner. They exist only when every part is rectangular, every required part fits, and every supported hole stays inside its part. **Approximate — always flag it:** - **Irregular parts** (gussets, brackets, curved profiles, parts with holes) are nested by their **bounding box**, not true shape. Real yield is a little better than reported. For exact weight/cost on those, get the true cut area (in²) into the part's `area` field. This is NOT true-shape nesting like a dedicated CAM engine. @@ -57,16 +57,37 @@ Write the job JSON, then run the engine: python3 scripts/nest.py --job --out ``` -Outputs land in `/`: +The legacy job JSON and command arguments remain accepted. `` is now a +publication root rather than a flat artifact directory. Every invocation creates +`/runs//` and atomically updates `/latest-run.json`; follow +that pointer to find the current run. This prevents an older burn file from appearing +current after a blocked rerun. + +Use `--geometry-verified-only` when reference-only geometry does not satisfy the +request. The command still publishes its QA diagnostics, but exits unsuccessfully. + +Each run contains: + +- `run-manifest.json` and `qa-report.json` — outcome, readiness, hashes, warnings, and the exact artifact allow-list - `layout.pdf` — every plate drawn (holes shown) + a summary page (the main deliverable) - `plate_1.png`, `plate_2.png`, … — one image per plate -- `burn_plate_1.dxf`, `burn_plate_2.dxf`, … — guarded per-sheet import geometry (PROFILE + HOLES layers, origin at sheet corner); absent when the safety guard blocks them -- `reference_nest.dxf` — all sheets side-by-side, reference only +- `burn_plate_1.dxf`, `burn_plate_2.dxf`, … — geometry-verified cut entities only; absent for review-required or blocked runs +- `reference_nest.dxf` and `reference_plate_N.dxf` — explicitly reference-only layouts - `rfq_nesting.json` — the Material / Nesting Plan / Drop Notes block for the `steel-rfq` hand-off (see below) - `report.txt` — the text report - `result.json` — structured result (plates, placements, holes, yield, cost) for downstream use -The engine has no third-party build dependencies beyond `ezdxf`, `matplotlib`, and `numpy`. Install once if missing: `pip install --break-system-packages ezdxf matplotlib numpy`. +Exit meanings: + +- `0` — `ready`; requested artifacts were published and cut geometry is verified within the supported rectangular scope +- `2` — `review_required`; safe reference outputs were published, but no burn DXF exists +- `3` — `blocked`; validation or unplaced material prevented fabrication-style output +- `4` — a required runtime capability is missing +- `1` — usage or internal error + +Install the declared dependencies from the package root with +`python3 -m pip install -r requirements-dev.txt`. Formula baking remains an optional +capability reported by `python3 scripts/doctor.py`. ## What to Deliver @@ -76,7 +97,7 @@ Verify before presenting: the engine already checks that no parts overlap and al ## Integration with steel-rfq -The `steel-rfq` skill has a "Nesting / Drop Reference" table. This engine writes exactly that data to `rfq_nesting.json` on every run — one row per plate material with `material`, `nesting_plan`, `drop_notes`, plus `sheets_needed` (for cross-checking the estimate's assumed sheet count) and `total_cost`. When an RFQ involves plate, `steel-rfq` builds a nest job from the plate parts, runs this engine, and reads `rfq_nesting.json` straight into its table. Keep this JSON shape stable — the RFQ skill depends on those field names. +The `steel-rfq` skill has a "Nesting / Drop Reference" table. This engine writes exactly that data to `rfq_nesting.json` inside each isolated run — one row per plate material with `material`, `nesting_plan`, `drop_notes`, plus `sheets_needed` (for cross-checking the estimate's assumed sheet count) and `total_cost`. Resolve the current run through `latest-run.json` before reading it. Keep this JSON shape stable — the RFQ skill depends on those field names. ## Common Variations diff --git a/skills/steel-nest/scripts/nest.py b/skills/steel-nest/scripts/nest.py index 3c13668..4f5103d 100644 --- a/skills/steel-nest/scripts/nest.py +++ b/skills/steel-nest/scripts/nest.py @@ -8,13 +8,14 @@ * Nests parts onto stock plates with a MaxRects bin-packing algorithm (rotation, kerf + gap spacing, edge margin, multiple plate sizes, greedy multi-plate fill). Rectangular parts nest exactly. - * Parts can carry HOLES (round) and rectangular CUTOUTS -- subtracted - from weight/cost, rotated with the part, drawn in the layout, and cut - as real geometry in the DXF output. + * Rectangular parts can carry HOLES (round) and rectangular CUTOUTS -- + subtracted from weight/cost, rotated with the part, drawn in the layout, + and emitted as cut geometry only after the complete job passes its gate. * Yield / scrap / largest reusable drop, part weight, material cost. * Labeled layout (PNG per plate + combined PDF). * DXF outputs: - reference_nest.dxf all plates side-by-side (reference only) + - reference_plate_N.dxf one reference-only file per used plate - burn_plate_N.dxf ONE FILE PER SHEET for the burn table when every part is rectangular, every required part fits, and supported holes stay inside the part. @@ -27,16 +28,45 @@ machine's own CAM/post applies those (that is where they belong). Usage: - python3 nest.py --job job.json --out out/ + python3 nest.py --job job.json --out published/ + +The output root receives isolated runs// directories plus a +latest-run.json pointer. Exit 0 is ready, 2 requires review, and 3 is blocked. """ import argparse +import importlib.util import json import math import os +import sys from dataclasses import dataclass, field +from pathlib import Path + + +SHARED_ROOT = Path(__file__).resolve().parents[2] / "_shared" +if str(SHARED_ROOT) not in sys.path: + sys.path.insert(0, str(SHARED_ROOT)) +from bootstrap import bootstrap_shared # noqa: E402 + +bootstrap_shared(__file__) +from pi_steel import ( # noqa: E402 + RunPublisher, + canonical_json_bytes, + outcome_exit_code, + sha256_bytes, +) STEEL_DENSITY = 0.2836 # lb/in^3, A36 mild steel +NEST_ALGORITHM_VERSION = "maxrects-bssf-u1" + + +class StageArgumentParser(argparse.ArgumentParser): + """Use the shared stage contract's exit 1 for command usage errors.""" + + def error(self, message): + self.print_usage(sys.stderr) + self.exit(1, f"{self.prog}: error: {message}\n") # -------------------------------------------------------------------------- @@ -186,20 +216,9 @@ def hole_local(pc, hole): return hx, hy -def burn_dxf_warnings(job, unplaced): - """Return reasons the current job is unsafe for fabrication-style DXF output.""" +def hole_containment_warnings(job): + """Return supported-hole geometry failures for the current legacy job.""" warnings = [] - - if any(part.get("shape", "rect") == "irregular" for part in job["parts"]): - warnings.append( - "Irregular parts use approximate bounding boxes; burn DXFs are suppressed." - ) - - if unplaced: - warnings.append( - f"{len(unplaced)} required part(s) did not fit; burn DXFs are suppressed." - ) - eps = 1e-9 for part in job["parts"]: width = float(part["width"]) @@ -242,6 +261,24 @@ def burn_dxf_warnings(job, unplaced): return warnings +def burn_dxf_warnings(job, unplaced): + """Return reasons the current job is unsafe for fabrication-style DXF output.""" + warnings = [] + + if any(part.get("shape", "rect") == "irregular" for part in job["parts"]): + warnings.append( + "Irregular parts use approximate bounding boxes; burn DXFs are suppressed." + ) + + if unplaced: + warnings.append( + f"{len(unplaced)} required part(s) did not fit; burn DXFs are suppressed." + ) + + warnings.extend(hole_containment_warnings(job)) + return warnings + + # -------------------------------------------------------------------------- # Job runner # -------------------------------------------------------------------------- @@ -409,7 +446,15 @@ def _summarize(job, used_plates, unplaced, density, margin, kerf, gap): overall_yield = round(100 * tot_part_area_bbox / tot_plate_area, 1) if tot_plate_area else 0.0 + invalid_holes = hole_containment_warnings(job) burn_warnings = burn_dxf_warnings(job, unplaced) + has_irregular = any(p.get("shape") == "irregular" for p in job["parts"]) + if unplaced or invalid_holes: + geometry_readiness = "diagnostic" + elif has_irregular: + geometry_readiness = "reference_only" + else: + geometry_readiness = "geometry_verified" res = { "meta": { "job_name": job.get("job_name", "Nesting job"), @@ -429,7 +474,9 @@ def _summarize(job, used_plates, unplaced, density, margin, kerf, gap): "plate_reports": plate_reports, "unplaced": [{"label": u["label"], "size": f'{_fmt(u["w"])} x {_fmt(u["h"])}'} for u in unplaced], - "has_irregular": any(p.get("shape") == "irregular" for p in job["parts"]), + "has_irregular": has_irregular, + "invalid_hole_warnings": invalid_holes, + "geometry_readiness": geometry_readiness, "burn_dxf_eligible": not burn_warnings, "burn_dxf_warnings": burn_warnings, } @@ -605,14 +652,17 @@ def render_layout(res, outdir): # -------------------------------------------------------------------------- -# DXF: overview (all plates) + one burn file per sheet +# DXF: clearly separated reference and geometry-verified burn files # -------------------------------------------------------------------------- def _draw_part_dxf(msp, pc, x0, y0, profile_layer, holes_layer, notes_layer, label=True): import ezdxf x, y = x0 + pc["x"], y0 + pc["y"] w, h = pc["w"], pc["h"] - msp.add_lwpolyline([(x, y), (x + w, y), (x + w, y + h), (x, y + h), (x, y)], - dxfattribs={"layer": profile_layer, "closed": True}) + msp.add_lwpolyline( + [(x, y), (x + w, y), (x + w, y + h), (x, y + h)], + close=True, + dxfattribs={"layer": profile_layer}, + ) for hole in pc.get("holes", []): lx, ly = hole_local(pc, hole) cx, cy = x + lx, y + ly @@ -623,39 +673,84 @@ def _draw_part_dxf(msp, pc, x0, y0, profile_layer, holes_layer, notes_layer, lab if pc["rotated"]: hw, hh = hh, hw msp.add_lwpolyline( - [(cx - hw / 2, cy - hh / 2), (cx + hw / 2, cy - hh / 2), - (cx + hw / 2, cy + hh / 2), (cx - hw / 2, cy + hh / 2), (cx - hw / 2, cy - hh / 2)], - dxfattribs={"layer": holes_layer, "closed": True}) - if label: + [ + (cx - hw / 2, cy - hh / 2), + (cx + hw / 2, cy - hh / 2), + (cx + hw / 2, cy + hh / 2), + (cx - hw / 2, cy + hh / 2), + ], + close=True, + dxfattribs={"layer": holes_layer}, + ) + if label and notes_layer: msp.add_text(pc["label"], height=min(1.0, max(0.25, min(w, h) * 0.18)), dxfattribs={"layer": notes_layer}).set_placement( (x + w / 2, y + h / 2), align=ezdxf.enums.TextEntityAlignment.MIDDLE_CENTER) def render_dxf_overview(res, outdir): + """Write one explicitly reference-only overview with bounding-box outlines.""" import ezdxf margin = res["meta"]["edge_margin_in"] doc = ezdxf.new("R2010") doc.units = ezdxf.units.IN msp = doc.modelspace() - for lyr, col in [("PLATE", 5), ("PROFILE", 3), ("HOLES", 1), ("NOTES", 7)]: + for lyr, col in [("PLATE", 5), ("BOUNDS", 2), ("HOLES", 1), ("NOTES", 7)]: if lyr not in doc.layers: doc.layers.add(lyr, color=col) x_off = 0.0 for pr in res["plate_reports"]: W, H = pr["W"], pr["H"] - msp.add_lwpolyline([(x_off, 0), (x_off + W, 0), (x_off + W, H), (x_off, H), (x_off, 0)], - dxfattribs={"layer": "PLATE", "closed": True}) + msp.add_lwpolyline( + [(x_off, 0), (x_off + W, 0), (x_off + W, H), (x_off, H)], + close=True, + dxfattribs={"layer": "PLATE"}, + ) for pc in pr["placements"]: - _draw_part_dxf(msp, pc, x_off + margin, margin, "PROFILE", "HOLES", "NOTES") + _draw_part_dxf( + msp, pc, x_off + margin, margin, "BOUNDS", "HOLES", "NOTES" + ) x_off += W + 10.0 path = os.path.join(outdir, "reference_nest.dxf") doc.saveas(path) return path +def render_reference_plate_dxfs(res, outdir): + """Write one clearly named reference-only DXF per used plate.""" + import ezdxf + + margin = res["meta"]["edge_margin_in"] + paths = [] + for pr in res["plate_reports"]: + doc = ezdxf.new("R2010") + doc.units = ezdxf.units.IN + msp = doc.modelspace() + for layer, color in [ + ("PLATE", 5), + ("BOUNDS", 2), + ("HOLES", 1), + ("NOTES", 7), + ]: + doc.layers.add(layer, color=color) + width, height = pr["W"], pr["H"] + msp.add_lwpolyline( + [(0, 0), (width, 0), (width, height), (0, height)], + close=True, + dxfattribs={"layer": "PLATE"}, + ) + for placement in pr["placements"]: + _draw_part_dxf( + msp, placement, margin, margin, "BOUNDS", "HOLES", "NOTES" + ) + path = os.path.join(outdir, f"reference_plate_{pr['index']}.dxf") + doc.saveas(path) + paths.append(path) + return paths + + def render_burn_dxfs(res, outdir): - """One DXF per sheet for the burn table. Origin at sheet corner.""" + """Write cut-geometry-only DXFs for a fully verified rectangular nest.""" if not res.get("burn_dxf_eligible", False): return [] @@ -666,59 +761,225 @@ def render_burn_dxfs(res, outdir): doc = ezdxf.new("R2010") doc.units = ezdxf.units.IN msp = doc.modelspace() - for lyr, col in [("PLATE", 5), ("PROFILE", 3), ("HOLES", 1), ("NOTES", 7)]: + for lyr, col in [("PROFILE", 3), ("HOLES", 1)]: doc.layers.add(lyr, color=col) - W, H = pr["W"], pr["H"] - # sheet outline for reference (delete on the table if not wanted) - msp.add_lwpolyline([(0, 0), (W, 0), (W, H), (0, H), (0, 0)], - dxfattribs={"layer": "PLATE", "closed": True}) for pc in pr["placements"]: - _draw_part_dxf(msp, pc, margin, margin, "PROFILE", "HOLES", "NOTES") + _draw_part_dxf( + msp, pc, margin, margin, "PROFILE", "HOLES", None, label=False + ) path = os.path.join(outdir, f"burn_plate_{pr['index']}.dxf") doc.saveas(path) paths.append(path) return paths +def stage_decision(res, geometry_verified_only=False): + """Map the temporary legacy nest result onto the shared stage contract.""" + findings = [] + if res["unplaced"]: + findings.append({ + "code": "UNPLACED_PARTS", + "severity": "error", + "message": f"{len(res['unplaced'])} required part(s) remain unplaced.", + }) + for warning in res["invalid_hole_warnings"]: + findings.append({ + "code": "INVALID_HOLE_GEOMETRY", + "severity": "error", + "message": warning, + }) + + if findings: + outcome = "blocked" + package_status = "nested_partial" if res["unplaced"] else "draft" + elif res["has_irregular"]: + outcome = "review_required" + package_status = "review_required" + findings.append({ + "code": "APPROXIMATE_PROFILE_GEOMETRY", + "severity": "warning", + "message": ( + "Irregular profiles are represented by bounding boxes in " + "reference-only artifacts." + ), + }) + else: + outcome = "ready" + package_status = "nest_verified" + + if geometry_verified_only and outcome != "ready": + findings.append({ + "code": "GEOMETRY_VERIFIED_REQUIRED", + "severity": "error", + "message": "The requested geometry-verified output is unavailable.", + }) + + return outcome, package_status, findings + + +def package_version(): + package_path = Path(__file__).resolve().parents[3] / "package.json" + try: + return json.loads(package_path.read_text(encoding="utf-8"))["version"] + except (OSError, KeyError, json.JSONDecodeError): + return "unknown" + + +def missing_render_dependencies(): + """Return optional render modules unavailable to this interpreter.""" + modules = ("ezdxf", "matplotlib", "numpy") + return [name for name in modules if importlib.util.find_spec(name) is None] + + +def publish_nest_run(job, args): + """Run a legacy nest job and publish one isolated, manifested artifact set.""" + result = run_job(job) + missing_dependencies = [] if args.no_render else missing_render_dependencies() + if missing_dependencies: + outcome = "dependency_missing" + package_status = "draft" + findings = [{ + "code": "RENDER_DEPENDENCY_MISSING", + "severity": "error", + "message": ( + "Rendering requires the missing module(s): " + + ", ".join(missing_dependencies) + ), + }] + else: + outcome, package_status, findings = stage_decision( + result, args.geometry_verified_only + ) + result["run_outcome"] = outcome + result["package_status"] = package_status + report = render_text(result) + + configuration = { + "algorithm_version": NEST_ALGORITHM_VERSION, + "geometry_verified_only": args.geometry_verified_only, + "render": not args.no_render, + } + approximations = [] + if result["has_irregular"]: + approximations.append({ + "code": "BOUNDING_BOX_NESTING", + "message": "One or more irregular profiles use bounding-box placement.", + }) + qa_report = { + "schema_version": "1.0.0", + "stage": "steel-nest", + "run_outcome": outcome, + "package_status": package_status, + "geometry_readiness": result["geometry_readiness"], + "geometry_verified_only_requested": args.geometry_verified_only, + "findings": findings, + } + + with RunPublisher( + args.out, + stage="steel-nest", + run_outcome=outcome, + package_status=package_status, + input_hash=sha256_bytes(canonical_json_bytes(job)), + configuration_hash=sha256_bytes(canonical_json_bytes(configuration)), + schema_versions={ + "run_manifest": "1.0.0", + "nest_result": "legacy-u1", + }, + tool_versions={ + "pi_steel": package_version(), + "nest_algorithm": NEST_ALGORITHM_VERSION, + }, + explicit_dates={}, + warnings=result["burn_dxf_warnings"] + + [finding["message"] for finding in findings if finding["severity"] == "error"], + approximations=approximations, + run_id=args.run_id, + ) as publisher: + publisher.write_qa_report(qa_report) + publisher.write_bytes( + "report.txt", + report.encode("utf-8"), + readiness="diagnostic", + media_type="text/plain", + ) + publisher.write_json("result.json", result, readiness="diagnostic") + publisher.write_json( + "rfq_nesting.json", result["rfq_nesting"], readiness="diagnostic" + ) + + if not args.no_render and not missing_dependencies: + publisher.register_artifact( + "layout.pdf", readiness="reference_only", media_type="application/pdf" + ) + for plate in result["plate_reports"]: + publisher.register_artifact( + f"plate_{plate['index']}.png", + readiness="reference_only", + media_type="image/png", + ) + render_layout(result, publisher.staging_path) + + if result["plate_reports"]: + publisher.register_artifact( + "reference_nest.dxf", + readiness="reference_only", + media_type="image/vnd.dxf", + ) + for plate in result["plate_reports"]: + publisher.register_artifact( + f"reference_plate_{plate['index']}.dxf", + readiness="reference_only", + media_type="image/vnd.dxf", + ) + render_dxf_overview(result, publisher.staging_path) + render_reference_plate_dxfs(result, publisher.staging_path) + + if outcome == "ready": + for plate in result["plate_reports"]: + publisher.register_artifact( + f"burn_plate_{plate['index']}.dxf", + readiness="geometry_verified", + media_type="image/vnd.dxf", + ) + render_burn_dxfs(result, publisher.staging_path) + + final_path = publisher.publish() + + return result, qa_report, report, final_path + + # -------------------------------------------------------------------------- # Main # -------------------------------------------------------------------------- -def main(): - ap = argparse.ArgumentParser(description="Steel plate nesting engine") +def main(argv=None): + ap = StageArgumentParser(description="Steel plate nesting engine") ap.add_argument("--job", required=True) - ap.add_argument("--out", default="out") + ap.add_argument( + "--out", + default="out", + help="Publication root; each invocation writes an isolated runs//", + ) ap.add_argument("--no-render", action="store_true", help="Skip PDF/PNG/DXF") - args = ap.parse_args() - - with open(args.job) as f: + ap.add_argument( + "--geometry-verified-only", + action="store_true", + help="Require geometry-verified output; unresolved jobs still publish QA", + ) + ap.add_argument("--run-id", help=argparse.SUPPRESS) + args = ap.parse_args(argv) + + with open(args.job, encoding="utf-8") as f: job = json.load(f) - os.makedirs(args.out, exist_ok=True) - - res = run_job(job) - - report = render_text(res) + result, qa_report, report, final_path = publish_nest_run(job, args) print(report) - with open(os.path.join(args.out, "report.txt"), "w") as f: - f.write(report) - with open(os.path.join(args.out, "result.json"), "w") as f: - json.dump(res, f, indent=2) - with open(os.path.join(args.out, "rfq_nesting.json"), "w") as f: - json.dump(res["rfq_nesting"], f, indent=2) - - if not args.no_render: - pdf, pngs = render_layout(res, args.out) - overview = render_dxf_overview(res, args.out) - burns = render_burn_dxfs(res, args.out) - print(f"\nWrote: {pdf}") - print(f" {overview} (reference only)") - for b in burns: - print(f" {b} (burn table — one per sheet)") - if not burns: - print(" burn DXFs suppressed:") - for warning in res["burn_dxf_warnings"]: - print(f" - {warning}") - print(f" {len(pngs)} PNG(s), report.txt, result.json, rfq_nesting.json") + print(f"\nPublished {qa_report['run_outcome']} run: {final_path}") + if result["burn_dxf_warnings"]: + print("Burn DXFs suppressed:") + for warning in result["burn_dxf_warnings"]: + print(f" - {warning}") + return outcome_exit_code(qa_report["run_outcome"]) if __name__ == "__main__": - main() + raise SystemExit(main()) diff --git a/tests/fixtures/nest/README.md b/tests/fixtures/nest/README.md new file mode 100644 index 0000000..78a9bc0 --- /dev/null +++ b/tests/fixtures/nest/README.md @@ -0,0 +1,9 @@ +# Synthetic Nest Fixtures + +The files in this directory were created from scratch for pi-steel tests. They +are not copied, transformed, rounded, renamed, or anonymized from production, +customer, vendor, drawing, takeoff, inventory, or commercial data. + +`irregular-reference-only.json` uses invented geometry to prove that an +unresolved irregular outline remains reference-only and cannot produce a burn +DXF. It establishes no compatibility claim for any CAM product or version. diff --git a/tests/fixtures/nest/irregular-reference-only.json b/tests/fixtures/nest/irregular-reference-only.json new file mode 100644 index 0000000..e632b06 --- /dev/null +++ b/tests/fixtures/nest/irregular-reference-only.json @@ -0,0 +1,29 @@ +{ + "job_name": "SYNTHETIC-IRREGULAR-REFERENCE", + "customer": "Example Customer", + "settings": { + "kerf_in": 0.06, + "part_gap_in": 0.25, + "edge_margin_in": 0.5, + "density_lb_in3": 0.2836 + }, + "stock": [ + { + "name": "Synthetic A36 Plate", + "width": 24, + "height": 24, + "thickness": 0.5, + "qty": 1 + } + ], + "parts": [ + { + "name": "Synthetic Irregular Gusset", + "width": 8, + "height": 6, + "qty": 1, + "shape": "irregular", + "area": 24 + } + ] +} diff --git a/tests/test_nest_outputs.py b/tests/test_nest_outputs.py new file mode 100644 index 0000000..b300ee8 --- /dev/null +++ b/tests/test_nest_outputs.py @@ -0,0 +1,284 @@ +import importlib.util +import json +import os +import subprocess +import sys +from copy import deepcopy +from pathlib import Path +from types import SimpleNamespace + +import ezdxf + + +ROOT = Path(__file__).resolve().parents[1] +NEST_SCRIPT = ROOT / "skills" / "steel-nest" / "scripts" / "nest.py" +SPEC = importlib.util.spec_from_file_location("pi_steel_nest_outputs", NEST_SCRIPT) +nest = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = nest +SPEC.loader.exec_module(nest) + + +def rectangular_job(): + return { + "job_name": "SYNTHETIC-DXF-CHARACTERIZATION", + "settings": { + "kerf_in": 0.06, + "part_gap_in": 0.25, + "edge_margin_in": 0.5, + "density_lb_in3": 0.2836, + }, + "stock": [ + { + "name": "Synthetic A36 Plate", + "width": 20, + "height": 20, + "thickness": 0.5, + "qty": 1, + } + ], + "parts": [ + { + "name": "Synthetic Base Plate", + "width": 8, + "height": 6, + "qty": 1, + "shape": "rect", + "holes": [{"dia": 1, "x": 4, "y": 3}], + } + ], + } + + +IRREGULAR_FIXTURE = ( + ROOT / "tests" / "fixtures" / "nest" / "irregular-reference-only.json" +) + + +def write_job(tmp_path, name, job): + path = tmp_path / name + path.write_text(json.dumps(job), encoding="utf-8") + return path + + +def run_cli(tmp_path, job_path, output_root, run_id, *extra): + environment = os.environ.copy() + environment.pop("PYTHONPATH", None) + return subprocess.run( + [ + sys.executable, + NEST_SCRIPT, + "--job", + job_path, + "--out", + output_root, + "--run-id", + run_id, + *extra, + ], + cwd=tmp_path, + env=environment, + capture_output=True, + text=True, + ) + + +def latest_run(output_root): + pointer = json.loads( + (output_root / "latest-run.json").read_text(encoding="utf-8") + ) + return output_root / pointer["run_directory"] + + +def load_json(path): + return json.loads(Path(path).read_text(encoding="utf-8")) + + +def test_verified_rectangular_burn_preserves_units_origin_and_cut_entities_only( + tmp_path, +): + result = nest.run_job(rectangular_job()) + paths = nest.render_burn_dxfs(result, tmp_path) + + assert len(paths) == 1 + document = ezdxf.readfile(paths[0]) + assert document.units == ezdxf.units.IN + + entities = list(document.modelspace()) + assert {entity.dxf.layer for entity in entities} == {"PROFILE", "HOLES"} + assert { + entity.dxftype() for entity in entities if entity.dxf.layer == "PROFILE" + } == {"LWPOLYLINE"} + assert { + entity.dxftype() for entity in entities if entity.dxf.layer == "HOLES" + } <= {"CIRCLE", "LWPOLYLINE"} + assert not any(entity.dxftype() == "TEXT" for entity in entities) + profile = next(entity for entity in entities if entity.dxf.layer == "PROFILE") + points = list(profile.get_points("xy")) + assert min(point[0] for point in points) == 0.5 + assert min(point[1] for point in points) == 0.5 + assert profile.closed + + +def test_irregular_job_publishes_review_required_reference_outputs(tmp_path): + output_root = tmp_path / "published" + completed = run_cli( + tmp_path, + IRREGULAR_FIXTURE, + output_root, + "SYNTHETIC-IRREGULAR-RUN", + ) + + assert completed.returncode == 2, completed.stdout + completed.stderr + run_path = latest_run(output_root) + assert not list(run_path.glob("burn_plate_*.dxf")) + assert (run_path / "reference_nest.dxf").exists() + assert (run_path / "reference_plate_1.dxf").exists() + + reference = ezdxf.readfile(run_path / "reference_plate_1.dxf") + reference_layers = {entity.dxf.layer for entity in reference.modelspace()} + assert "BOUNDS" in reference_layers + assert "PROFILE" not in reference_layers + + result = load_json(run_path / "result.json") + qa = load_json(run_path / "qa-report.json") + manifest = load_json(run_path / "run-manifest.json") + assert result["geometry_readiness"] == "reference_only" + assert qa["run_outcome"] == "review_required" + assert manifest["run_outcome"] == "review_required" + assert {artifact["readiness"] for artifact in manifest["artifacts"]} <= { + "reference_only", + "diagnostic", + } + + +def test_geometry_verified_only_request_for_irregular_job_exits_unsuccessfully( + tmp_path, +): + output_root = tmp_path / "published" + completed = run_cli( + tmp_path, + IRREGULAR_FIXTURE, + output_root, + "SYNTHETIC-GEOMETRY-ONLY-RUN", + "--geometry-verified-only", + ) + + assert completed.returncode == 2, completed.stdout + completed.stderr + run_path = latest_run(output_root) + qa = load_json(run_path / "qa-report.json") + assert any( + finding["code"] == "GEOMETRY_VERIFIED_REQUIRED" + for finding in qa["findings"] + ) + assert not list(run_path.glob("burn_plate_*.dxf")) + + +def test_blocked_rerun_is_isolated_and_cannot_expose_stale_burn_output(tmp_path): + output_root = tmp_path / "published" + ready_job = write_job(tmp_path, "ready.json", rectangular_job()) + ready = run_cli( + tmp_path, + ready_job, + output_root, + "SYNTHETIC-READY-RUN", + ) + assert ready.returncode == 0, ready.stdout + ready.stderr + ready_path = latest_run(output_root) + assert (ready_path / "burn_plate_1.dxf").exists() + ready_manifest = load_json(ready_path / "run-manifest.json") + assert ready_manifest["run_outcome"] == "ready" + burn_artifact = next( + artifact + for artifact in ready_manifest["artifacts"] + if artifact["path"] == "burn_plate_1.dxf" + ) + assert burn_artifact["readiness"] == "geometry_verified" + + blocked_job = deepcopy(rectangular_job()) + blocked_job["parts"].append( + { + "name": "Synthetic Oversize Plate", + "width": 30, + "height": 30, + "qty": 1, + "shape": "rect", + } + ) + blocked_path_input = write_job(tmp_path, "blocked.json", blocked_job) + blocked = run_cli( + tmp_path, + blocked_path_input, + output_root, + "SYNTHETIC-BLOCKED-RUN", + ) + + assert blocked.returncode == 3, blocked.stdout + blocked.stderr + blocked_path = latest_run(output_root) + assert blocked_path.name == "SYNTHETIC-BLOCKED-RUN" + assert not list(blocked_path.glob("burn_plate_*.dxf")) + assert (blocked_path / "reference_nest.dxf").exists() + assert (ready_path / "burn_plate_1.dxf").exists() + assert load_json(blocked_path / "run-manifest.json")["run_outcome"] == "blocked" + assert load_json(blocked_path / "qa-report.json")["package_status"] == ( + "nested_partial" + ) + + +def test_invalid_hole_is_blocked_with_only_safe_reference_and_diagnostics(tmp_path): + invalid_job = rectangular_job() + invalid_job["parts"][0]["holes"] = [{"dia": 2, "x": 0.5, "y": 3}] + job_path = write_job(tmp_path, "invalid-hole.json", invalid_job) + output_root = tmp_path / "published" + + completed = run_cli( + tmp_path, + job_path, + output_root, + "SYNTHETIC-INVALID-HOLE-RUN", + ) + + assert completed.returncode == 3, completed.stdout + completed.stderr + run_path = latest_run(output_root) + assert not list(run_path.glob("burn_plate_*.dxf")) + assert (run_path / "reference_nest.dxf").exists() + manifest = load_json(run_path / "run-manifest.json") + assert manifest["run_outcome"] == "blocked" + assert "geometry_verified" not in { + artifact["readiness"] for artifact in manifest["artifacts"] + } + + +def test_missing_render_dependency_publishes_diagnostic_run(tmp_path, monkeypatch): + monkeypatch.setattr( + nest, "missing_render_dependencies", lambda: ["synthetic-render-module"] + ) + args = SimpleNamespace( + out=tmp_path / "published", + no_render=False, + geometry_verified_only=False, + run_id="SYNTHETIC-DEPENDENCY-RUN", + ) + + _, qa, _, run_path = nest.publish_nest_run(rectangular_job(), args) + + assert qa["run_outcome"] == "dependency_missing" + assert nest.outcome_exit_code(qa["run_outcome"]) == 4 + assert not (run_path / "layout.pdf").exists() + assert not list(run_path.glob("burn_plate_*.dxf")) + assert load_json(run_path / "run-manifest.json")["run_outcome"] == ( + "dependency_missing" + ) + + +def test_cli_usage_error_uses_stage_contract_exit_one(tmp_path): + environment = os.environ.copy() + environment.pop("PYTHONPATH", None) + completed = subprocess.run( + [sys.executable, NEST_SCRIPT], + cwd=tmp_path, + env=environment, + capture_output=True, + text=True, + ) + + assert completed.returncode == 1 From 8d7af95aa550d7ee4d011d19483ee6b929d006ed Mon Sep 17 00:00:00 2001 From: Victor Garcia Date: Tue, 28 Jul 2026 12:17:49 -0600 Subject: [PATCH 06/15] feat(contracts): add canonical estimate validation --- skills/_shared/pi_steel/__init__.py | 16 + skills/_shared/pi_steel/contracts.py | 87 ++++ skills/_shared/pi_steel/geometry_verify.py | 75 +++ skills/_shared/pi_steel/parsing.py | 205 ++++++++ skills/_shared/pi_steel/validation.py | 458 ++++++++++++++++++ .../schemas/estimate-package.schema.json | 361 ++++++++++++++ .../_shared/schemas/nest-result.schema.json | 90 ++++ skills/steel-nest/references/example_job.json | 15 +- .../steel-nest/references/job_template.json | 9 +- skills/steel-takeoff/assets/bom-template.csv | 2 +- .../steel-takeoff/scripts/calculate-weight.sh | 6 +- skills/steel-takeoff/scripts/validate-bom.py | 347 ++++++------- tests/fixtures/contracts/README.md | 5 + tests/fixtures/contracts/legacy-bom.csv | 3 + tests/fixtures/contracts/legacy-nest.json | 32 ++ tests/test_contracts.py | 367 ++++++++++++++ 16 files changed, 1870 insertions(+), 208 deletions(-) create mode 100644 skills/_shared/pi_steel/contracts.py create mode 100644 skills/_shared/pi_steel/geometry_verify.py create mode 100644 skills/_shared/pi_steel/parsing.py create mode 100644 skills/_shared/pi_steel/validation.py create mode 100644 skills/_shared/schemas/estimate-package.schema.json create mode 100644 skills/_shared/schemas/nest-result.schema.json create mode 100644 tests/fixtures/contracts/README.md create mode 100644 tests/fixtures/contracts/legacy-bom.csv create mode 100644 tests/fixtures/contracts/legacy-nest.json create mode 100644 tests/test_contracts.py diff --git a/skills/_shared/pi_steel/__init__.py b/skills/_shared/pi_steel/__init__.py index e61e72c..fb0bdfd 100644 --- a/skills/_shared/pi_steel/__init__.py +++ b/skills/_shared/pi_steel/__init__.py @@ -1,5 +1,14 @@ """Shared deterministic runtime primitives for pi-steel skills.""" +from .contracts import ( + ESTIMATE_PACKAGE_VERSION, + ITEM_INTENTS, + NEST_RESULT_VERSION, + estimate_input_hash, + instance_ids, + item_id_for, + placement_ids, +) from .run_manifest import ( ARTIFACT_READINESS, OUTCOME_EXIT_CODES, @@ -15,12 +24,19 @@ __all__ = [ "ARTIFACT_READINESS", + "ESTIMATE_PACKAGE_VERSION", + "ITEM_INTENTS", + "NEST_RESULT_VERSION", "OUTCOME_EXIT_CODES", "PACKAGE_STATUSES", "RUN_OUTCOMES", "ManifestError", "RunPublisher", "canonical_json_bytes", + "estimate_input_hash", + "instance_ids", + "item_id_for", + "placement_ids", "outcome_exit_code", "sha256_bytes", "sha256_file", diff --git a/skills/_shared/pi_steel/contracts.py b/skills/_shared/pi_steel/contracts.py new file mode 100644 index 0000000..4ef62d8 --- /dev/null +++ b/skills/_shared/pi_steel/contracts.py @@ -0,0 +1,87 @@ +"""Canonical contract constants and deterministic identity helpers.""" + +from __future__ import annotations + +import hashlib +import json +from typing import Any + + +ESTIMATE_PACKAGE_VERSION = "1.0.0" +NEST_RESULT_VERSION = "1.0.0" +ITEM_INTENTS = frozenset( + { + "fabricated_part", + "purchased_stock", + "hardware", + "allowance", + "exclusion", + "by_others", + } +) + + +def canonical_json_bytes(value: Any) -> bytes: + return json.dumps( + value, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ).encode("utf-8") + + +def content_hash(value: Any) -> str: + return hashlib.sha256(canonical_json_bytes(value)).hexdigest() + + +def item_id_for(project_id: str, revision_id: str, source_id: str) -> str: + """Derive normalized identity from project and stable source identity. + + ``revision_id`` remains in the signature for adapter compatibility. Legacy + source IDs already embed their revision when no stable source key exists; + explicit source IDs therefore remain stable across revisions. + """ + digest = content_hash( + { + "project_id": project_id, + "source_id": source_id, + } + ) + return f"item:{digest[:24]}" + + +def instance_ids(item_id: str, quantity: int) -> list[str]: + """Expand quantity predictably; increasing quantity never renames old instances.""" + if quantity < 0: + raise ValueError("quantity must be non-negative") + return [f"{item_id}:instance:{index:04d}" for index in range(1, quantity + 1)] + + +def placement_ids(item_id: str, quantity: int) -> list[str]: + """Derive stable per-instance placement identities from canonical item order.""" + if quantity < 0: + raise ValueError("quantity must be non-negative") + return [f"{item_id}:placement:{index:04d}" for index in range(1, quantity + 1)] + + +def fallback_source_id(revision_id: str, identity: Any) -> str: + """Create a revision-scoped legacy identity from stable row content.""" + return f"legacy:{revision_id}:{content_hash(identity)[:20]}" + + +def estimate_input_projection(package: dict[str, Any]) -> dict[str, Any]: + """Return source/config semantics, excluding review and confirmation bookkeeping.""" + projection = { + key: value + for key, value in package.items() + if key not in {"review"} + } + projection = json.loads(json.dumps(projection)) + for stock in projection.get("stock", []): + stock.pop("reviewer_confirmation", None) + return projection + + +def estimate_input_hash(package: dict[str, Any]) -> str: + return content_hash(estimate_input_projection(package)) + + +def finding_id_for(code: str, path: str) -> str: + return f"finding:{content_hash({'code': code, 'path': path})[:24]}" diff --git a/skills/_shared/pi_steel/geometry_verify.py b/skills/_shared/pi_steel/geometry_verify.py new file mode 100644 index 0000000..61312b6 --- /dev/null +++ b/skills/_shared/pi_steel/geometry_verify.py @@ -0,0 +1,75 @@ +"""Geometry checks shared by contract validation and nesting.""" + +from __future__ import annotations + +import math +from collections import defaultdict +from typing import Any + + +SUPPORTED_SHAPES = frozenset({"rect", "irregular"}) + + +def finite_positive(value: Any) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) and math.isfinite(value) and value > 0 + + +def hole_area(hole: dict[str, Any]) -> float: + if hole.get("kind") == "round": + diameter = hole.get("diameter", 0) + return math.pi * (diameter / 2) ** 2 + if hole.get("kind") == "rect": + return hole.get("width", 0) * hole.get("height", 0) + return 0 + + +def hole_within_bounds(hole: dict[str, Any], width: float, height: float) -> bool: + x, y = hole.get("x"), hole.get("y") + if not all(isinstance(value, (int, float)) and math.isfinite(value) for value in (x, y)): + return False + if hole.get("kind") == "round": + diameter = hole.get("diameter") + if not finite_positive(diameter): + return False + radius = diameter / 2 + return radius <= x <= width - radius and radius <= y <= height - radius + if hole.get("kind") == "rect": + hole_width, hole_height = hole.get("width"), hole.get("height") + if not finite_positive(hole_width) or not finite_positive(hole_height): + return False + return ( + hole_width / 2 <= x <= width - hole_width / 2 + and hole_height / 2 <= y <= height - hole_height / 2 + ) + return False + + +def gross_area(geometry: dict[str, Any]) -> float: + if geometry.get("shape") == "irregular" and geometry.get("area") is not None: + return geometry["area"] + return geometry.get("width", 0) * geometry.get("height", 0) + + +def net_area(geometry: dict[str, Any]) -> float: + return gross_area(geometry) - sum(hole_area(hole) for hole in geometry.get("holes", [])) + + +def plate_group_key(item: dict[str, Any]) -> tuple[Any, Any, Any]: + geometry = item.get("geometry", {}) + return item.get("material"), item.get("grade"), geometry.get("thickness") + + +def group_plate_items(items: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Partition parts strictly by material, grade, and thickness.""" + groups: dict[tuple[Any, Any, Any], list[dict[str, Any]]] = defaultdict(list) + for item in items: + groups[plate_group_key(item)].append(item) + return [ + { + "material": key[0], + "grade": key[1], + "thickness": key[2], + "items": grouped, + } + for key, grouped in sorted(groups.items(), key=lambda pair: repr(pair[0])) + ] diff --git a/skills/_shared/pi_steel/parsing.py b/skills/_shared/pi_steel/parsing.py new file mode 100644 index 0000000..5f7fb42 --- /dev/null +++ b/skills/_shared/pi_steel/parsing.py @@ -0,0 +1,205 @@ +"""Adapters from the supported legacy BOM and nesting inputs.""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path +from typing import Any + +from .contracts import ( + ESTIMATE_PACKAGE_VERSION, + content_hash, + fallback_source_id, + item_id_for, +) + + +def parse_length_ft(raw: str) -> float: + value = raw.strip() + if "'" not in value: + return float(value) + feet, _, inches = value.replace('"', "").partition("'") + return float(feet.replace("-", "").strip()) + ( + float(inches.replace("-", "").strip()) / 12 if inches.strip() else 0 + ) + + +def _package( + *, + project_id: str, + revision_id: str, + source_type: str, + source_hash: str, + items: list[dict[str, Any]], + stock: list[dict[str, Any]] | None = None, + unit_system: str = "imperial", +) -> dict[str, Any]: + return { + "schema_version": ESTIMATE_PACKAGE_VERSION, + "project": { + "project_id": project_id, + "revision": {"revision_id": revision_id}, + }, + "unit_system": unit_system, + "items": items, + "stock": stock or [], + "commercial_basis": {"currency": "USD", "costs": []}, + "review": {"status": "draft", "findings": [], "acknowledgements": []}, + "lineage": { + "source_type": source_type, + "source_hash": source_hash, + "configuration_hash": content_hash({"adapter": source_type, "version": 1}), + }, + } + + +def adapt_legacy_bom_csv( + path: str | Path, *, project_id: str, revision_id: str +) -> dict[str, Any]: + csv_path = Path(path) + raw_bytes = csv_path.read_bytes() + items: list[dict[str, Any]] = [] + with csv_path.open(newline="", encoding="utf-8-sig") as handle: + for row in csv.DictReader(handle): + mark = row.get("Mark", "").strip() + if mark.startswith("#") or not any((value or "").strip() for value in row.values()): + continue + explicit_source = row.get("Source_ID", "").strip() + stable_identity = { + key: (value or "").strip() + for key, value in row.items() + if key not in {"Total_Wt_lbs"} + } + source_id = explicit_source or ( + f"legacy:{revision_id}:mark:{mark}" + if mark + else fallback_source_id(revision_id, stable_identity) + ) + item: dict[str, Any] = { + "intent": row.get("Intent", "").strip() or "fabricated_part", + "source_id": source_id, + "item_id": item_id_for(project_id, revision_id, source_id), + "quantity": int(row.get("Qty", "0").strip()), + "mark": mark, + "designation": row.get("Size", "").strip(), + "grade": row.get("Grade", "").strip(), + "length_ft": parse_length_ft(row.get("Length_ft", "0")), + "unit_weight_plf": float(row.get("Unit_Wt_plf", "0").strip()), + "connections": row.get("Connections", "").strip(), + "notes": row.get("Notes", "").strip(), + } + total = row.get("Total_Wt_lbs", "").strip() + if total: + item["total_weight_lbs"] = float(total) + sheet = row.get("Source_Sheet", "").strip() + detail = row.get("Source_Detail", "").strip() + if sheet or detail: + item["source_evidence"] = [ + { + "source": sheet or "legacy_bom", + "locator": detail or mark or source_id, + } + ] + if not explicit_source and not mark: + item["identity_warning"] = True + items.append(item) + return _package( + project_id=project_id, + revision_id=revision_id, + source_type="legacy_bom_csv", + source_hash=content_hash({"bytes_hex": raw_bytes.hex()}), + items=items, + ) + + +def adapt_legacy_nest( + data: dict[str, Any] | str | Path, *, project_id: str, revision_id: str +) -> dict[str, Any]: + if isinstance(data, (str, Path)): + value = json.loads(Path(data).read_text(encoding="utf-8")) + else: + value = json.loads(json.dumps(data)) + material = value.get("material") + grade = value.get("grade") + unit_system = value.get("unit_system") + thickness = value.get("thickness_in", value.get("settings", {}).get("thickness_in")) + items = [] + for part in value.get("parts", []): + explicit_source = part.get("source_id") + identity = { + key: part.get(key) + for key in ("name", "width", "height", "shape", "area") + } + source_id = explicit_source or fallback_source_id(revision_id, identity) + geometry = { + "shape": part.get("shape", "rect"), + "width": part.get("width"), + "height": part.get("height"), + "thickness": thickness, + "holes": [ + ( + { + "kind": "round", + "diameter": hole.get("dia"), + "x": hole.get("x"), + "y": hole.get("y"), + } + if "dia" in hole + else { + "kind": "rect", + "width": hole.get("w"), + "height": hole.get("h"), + "x": hole.get("x"), + "y": hole.get("y"), + } + ) + for hole in part.get("holes", []) + ], + "rotatable": part.get("rotatable", True), + } + if "area" in part: + geometry["area"] = part["area"] + item = { + "intent": "fabricated_part", + "source_id": source_id, + "item_id": item_id_for(project_id, revision_id, source_id), + "quantity": part.get("qty", 0), + "mark": part.get("name", ""), + "geometry": geometry, + } + if material is not None: + item["material"] = material + if grade is not None: + item["grade"] = grade + if not explicit_source: + item["identity_warning"] = True + items.append(item) + + stock = [] + if material and grade and thickness is not None: + for index, legacy_stock in enumerate(value.get("stock", []), start=1): + stock.append( + { + "stock_kind": "purchasable", + "inventory_id": f"legacy-stock:{revision_id}:{index:04d}", + "material": material, + "grade": grade, + "width": legacy_stock.get("width"), + "height": legacy_stock.get("height"), + "thickness": legacy_stock.get("thickness", thickness), + "quantity": legacy_stock.get( + "qty", 1 if legacy_stock.get("unlimited") else 0 + ), + "status": "available", + } + ) + return _package( + project_id=project_id, + revision_id=revision_id, + source_type="legacy_nest_json", + source_hash=content_hash(value), + items=items, + stock=stock, + unit_system=unit_system or "imperial", + ) diff --git a/skills/_shared/pi_steel/validation.py b/skills/_shared/pi_steel/validation.py new file mode 100644 index 0000000..3d0b880 --- /dev/null +++ b/skills/_shared/pi_steel/validation.py @@ -0,0 +1,458 @@ +"""Schema and domain validation for canonical estimate packages.""" + +from __future__ import annotations + +import json +import math +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import jsonschema + +from .contracts import ( + ESTIMATE_PACKAGE_VERSION, + estimate_input_hash, + finding_id_for, +) +from .geometry_verify import ( + SUPPORTED_SHAPES, + finite_positive, + gross_area, + hole_within_bounds, + net_area, +) + + +_SCHEMA_PATH = ( + Path(__file__).resolve().parents[1] / "schemas" / "estimate-package.schema.json" +) +_REVIEWABLE_BLOCKERS = frozenset( + { + "missing_material_basis", + "unconfirmed_on_hand_stock", + } +) + + +@dataclass(frozen=True) +class ValidationResult: + status: str + input_hash: str + findings: list[dict[str, Any]] + active_findings: list[dict[str, Any]] + + @property + def blockers(self) -> list[dict[str, Any]]: + return [ + finding + for finding in self.active_findings + if finding["severity"] == "blocker" + ] + + @property + def warnings(self) -> list[dict[str, Any]]: + return [ + finding + for finding in self.active_findings + if finding["severity"] == "warning" + ] + + +def _path(parts: Any) -> str: + result = "$" + for part in parts: + result += f"[{part}]" if isinstance(part, int) else f".{part}" + return result + + +def _finding( + *, + code: str, + severity: str, + path: str, + message: str, + relevant_hash: str, +) -> dict[str, Any]: + return { + "finding_id": finding_id_for(code, path), + "code": code, + "severity": severity, + "path": path, + "message": message, + "relevant_hash": relevant_hash, + } + + +def _add( + findings: list[dict[str, Any]], + input_hash: str, + code: str, + severity: str, + path: str, + message: str, +) -> None: + findings.append( + _finding( + code=code, + severity=severity, + path=path, + message=message, + relevant_hash=input_hash, + ) + ) + + +def _schema_findings( + package: dict[str, Any], input_hash: str +) -> list[dict[str, Any]]: + schema = json.loads(_SCHEMA_PATH.read_text(encoding="utf-8")) + validator = jsonschema.Draft202012Validator( + schema, format_checker=jsonschema.FormatChecker() + ) + findings = [] + for error in sorted(validator.iter_errors(package), key=lambda item: list(item.path)): + _add( + findings, + input_hash, + "schema_validation", + "blocker", + _path(error.absolute_path), + error.message, + ) + return findings + + +def _geometry_findings( + item: dict[str, Any], + index: int, + input_hash: str, + findings: list[dict[str, Any]], +) -> None: + geometry = item.get("geometry") + if not isinstance(geometry, dict): + return + base = f"$.items[{index}].geometry" + shape = geometry.get("shape") + if shape not in SUPPORTED_SHAPES: + _add( + findings, + input_hash, + "unsupported_shape", + "blocker", + f"{base}.shape", + f"Unsupported shape {shape!r}; expected rect or irregular.", + ) + dimensions = ("width", "height", "thickness") + for field in dimensions: + value = geometry.get(field) + if isinstance(value, (int, float)) and not math.isfinite(value): + _add( + findings, + input_hash, + "nonfinite_dimension", + "blocker", + f"{base}.{field}", + f"{field} must be finite.", + ) + elif not finite_positive(value): + _add( + findings, + input_hash, + "nonpositive_dimension", + "blocker", + f"{base}.{field}", + f"{field} must be greater than zero.", + ) + width, height = geometry.get("width"), geometry.get("height") + if finite_positive(width) and finite_positive(height): + for hole_index, hole in enumerate(geometry.get("holes", [])): + if not hole_within_bounds(hole, width, height): + _add( + findings, + input_hash, + "hole_out_of_bounds", + "blocker", + f"{base}.holes[{hole_index}]", + "Hole geometry must be positive and contained by the part.", + ) + try: + area = net_area(geometry) + except (TypeError, ValueError, OverflowError): + area = math.nan + if not math.isfinite(area) or area <= 0: + _add( + findings, + input_hash, + "nonpositive_net_area", + "blocker", + base, + "Part net area after holes must be greater than zero.", + ) + if shape == "irregular" and not finite_positive(geometry.get("area")): + _add( + findings, + input_hash, + "invalid_irregular_area", + "blocker", + f"{base}.area", + "Irregular parts require a positive true-cut area.", + ) + + +def validate_estimate_package(package: dict[str, Any]) -> ValidationResult: + input_hash = estimate_input_hash(package) + version = package.get("schema_version") + if version != ESTIMATE_PACKAGE_VERSION: + finding = _finding( + code="unsupported_contract_version", + severity="blocker", + path="$.schema_version", + message=( + f"Unsupported estimate package version {version!r}; " + f"migrate to {ESTIMATE_PACKAGE_VERSION} before processing." + ), + relevant_hash=input_hash, + ) + return ValidationResult("invalid", input_hash, [finding], [finding]) + + findings = _schema_findings(package, input_hash) + source_ids: dict[str, int] = {} + item_ids: dict[str, int] = {} + for index, item in enumerate(package.get("items", [])): + base = f"$.items[{index}]" + quantity = item.get("quantity") + if ( + not isinstance(quantity, int) + or isinstance(quantity, bool) + or quantity <= 0 + ): + _add( + findings, + input_hash, + "invalid_quantity", + "blocker", + f"{base}.quantity", + "Quantity must be a positive integer.", + ) + for field, seen, code in ( + ("source_id", source_ids, "duplicate_source_id"), + ("item_id", item_ids, "duplicate_item_id"), + ): + identity = item.get(field) + if identity in seen: + _add( + findings, + input_hash, + code, + "blocker", + f"{base}.{field}", + f"{field} duplicates item {seen[identity]}; rows were not merged.", + ) + elif identity is not None: + seen[identity] = index + + if item.get("identity_warning"): + _add( + findings, + input_hash, + "unstable_source_identity", + "warning", + f"{base}.source_id", + "No explicit source ID or stable mark was supplied; identity is revision-scoped.", + ) + if not item.get("source_evidence"): + _add( + findings, + input_hash, + "missing_source_evidence", + "warning", + f"{base}.source_evidence", + "No drawing or structured-source locator was supplied; none was invented.", + ) + if item.get("intent") == "fabricated_part": + _geometry_findings(item, index, input_hash, findings) + if item.get("geometry") and ( + not item.get("material") + or not item.get("grade") + or not finite_positive(item.get("geometry", {}).get("thickness")) + ): + _add( + findings, + input_hash, + "missing_material_basis", + "blocker", + base, + "Plate material, grade, and thickness must be explicit before nesting, RFQ, or burn.", + ) + if not item.get("geometry"): + if not item.get("designation"): + _add( + findings, + input_hash, + "missing_designation", + "blocker", + f"{base}.designation", + "A legacy member requires a section designation.", + ) + for field, code, label in ( + ("length_ft", "invalid_member_length", "Member length"), + ( + "unit_weight_plf", + "invalid_unit_weight", + "Member unit weight", + ), + ): + if not finite_positive(item.get(field)): + _add( + findings, + input_hash, + code, + "blocker", + f"{base}.{field}", + f"{label} must be finite and greater than zero.", + ) + if not item.get("grade"): + _add( + findings, + input_hash, + "missing_grade", + "warning", + f"{base}.grade", + "No material grade was supplied; none was inferred.", + ) + + for index, stock in enumerate(package.get("stock", [])): + if stock.get("stock_kind") != "on_hand": + continue + required = ( + stock.get("inventory_id"), + stock.get("measured_at"), + stock.get("source"), + stock.get("status") in {"available", "reserved"}, + ) + confirmation = stock.get("reviewer_confirmation", {}) + confirmed = ( + all(required) + and confirmation.get("estimate_hash") == input_hash + and confirmation.get("actor") + and confirmation.get("timestamp") + ) + if not confirmed: + _add( + findings, + input_hash, + "unconfirmed_on_hand_stock", + "blocker", + f"$.stock[{index}]", + "On-hand stock cannot reduce purchasing without traceable measurements and hash-bound reviewer confirmation.", + ) + + basis = package.get("commercial_basis", {}) + for index, cost in enumerate(basis.get("costs", [])): + if cost.get("currency") != basis.get("currency") or not all( + cost.get(field) + for field in ("unit_basis", "effective_date", "source") + ): + _add( + findings, + input_hash, + "invalid_cost_basis", + "blocker", + f"$.commercial_basis.costs[{index}]", + "Cost currency, unit basis, effective date, and source must be preserved.", + ) + + acknowledgements = package.get("review", {}).get("acknowledgements", []) + active = [] + for finding in findings: + acknowledged = ( + finding["severity"] != "blocker" + and any( + acknowledgement.get("finding_id") == finding["finding_id"] + and acknowledgement.get("input_hash") == finding["relevant_hash"] + and acknowledgement.get("disposition") == "accepted" + for acknowledgement in acknowledgements + ) + ) + if not acknowledged: + active.append(finding) + + blockers = [f for f in active if f["severity"] == "blocker"] + if blockers: + status = ( + "review_required" + if all(f["code"] in _REVIEWABLE_BLOCKERS for f in blockers) + else "invalid" + ) + elif any(f["severity"] == "warning" for f in active): + status = "review_required" + else: + status = "validated" + return ValidationResult(status, input_hash, findings, active) + + +def acknowledge_finding( + package: dict[str, Any], + finding: dict[str, Any], + *, + actor: str, + timestamp: str, + disposition: str, +) -> dict[str, Any]: + if disposition not in {"accepted", "rejected", "deferred"}: + raise ValueError("unsupported acknowledgement disposition") + acknowledgement = { + "finding_id": finding["finding_id"], + "actor": actor, + "timestamp": timestamp, + "disposition": disposition, + "input_hash": finding["relevant_hash"], + } + package.setdefault("review", {}).setdefault("acknowledgements", []).append( + acknowledgement + ) + return acknowledgement + + +def eligible_on_hand_stock(package: dict[str, Any]) -> list[dict[str, Any]]: + input_hash = estimate_input_hash(package) + eligible = [] + for stock in package.get("stock", []): + confirmation = stock.get("reviewer_confirmation", {}) + if ( + stock.get("stock_kind") == "on_hand" + and stock.get("inventory_id") + and finite_positive(stock.get("width")) + and finite_positive(stock.get("height")) + and finite_positive(stock.get("thickness")) + and stock.get("measured_at") + and stock.get("source") + and stock.get("status") == "available" + and confirmation.get("actor") + and confirmation.get("timestamp") + and confirmation.get("estimate_hash") == input_hash + ): + eligible.append(stock) + return eligible + + +def purchasable_items(package: dict[str, Any]) -> list[dict[str, Any]]: + return [ + item + for item in package.get("items", []) + if item.get("intent") in {"fabricated_part", "purchased_stock", "hardware"} + ] + + +def stock_requiring_purchase(package: dict[str, Any]) -> list[dict[str, Any]]: + """Return only stock explicitly modeled as purchasable. + + On-hand entries that are unavailable or not confirmed remain validation + findings; they never silently become vendor demand. + """ + return [ + stock + for stock in package.get("stock", []) + if stock.get("stock_kind") == "purchasable" + ] diff --git a/skills/_shared/schemas/estimate-package.schema.json b/skills/_shared/schemas/estimate-package.schema.json new file mode 100644 index 0000000..12b6281 --- /dev/null +++ b/skills/_shared/schemas/estimate-package.schema.json @@ -0,0 +1,361 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://structupath.ai/schemas/pi-steel/estimate-package-1.0.0.json", + "title": "pi-steel canonical estimate package", + "type": "object", + "required": [ + "schema_version", + "project", + "unit_system", + "items", + "stock", + "commercial_basis", + "review", + "lineage" + ], + "properties": { + "schema_version": { "const": "1.0.0" }, + "project": { "$ref": "#/$defs/project" }, + "unit_system": { "enum": ["imperial", "metric"] }, + "items": { + "type": "array", + "items": { "$ref": "#/$defs/item" } + }, + "stock": { + "type": "array", + "items": { "$ref": "#/$defs/stock" } + }, + "commercial_basis": { "$ref": "#/$defs/commercialBasis" }, + "review": { "$ref": "#/$defs/review" }, + "lineage": { "$ref": "#/$defs/lineage" } + }, + "additionalProperties": false, + "$defs": { + "project": { + "type": "object", + "required": ["project_id", "revision"], + "properties": { + "project_id": { "type": "string", "minLength": 1 }, + "name": { "type": "string" }, + "revision": { + "type": "object", + "required": ["revision_id"], + "properties": { + "revision_id": { "type": "string", "minLength": 1 }, + "source_document": { "type": "string" }, + "issued_date": { "type": "string", "format": "date" } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "evidence": { + "type": "object", + "required": ["source", "locator"], + "properties": { + "source": { "type": "string", "minLength": 1 }, + "locator": { "type": "string", "minLength": 1 }, + "source_hash": { "type": "string", "pattern": "^[0-9a-f]{64}$" } + }, + "additionalProperties": false + }, + "hole": { + "oneOf": [ + { + "type": "object", + "required": ["kind", "diameter", "x", "y"], + "properties": { + "kind": { "const": "round" }, + "diameter": { "type": "number" }, + "x": { "type": "number" }, + "y": { "type": "number" } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": ["kind", "width", "height", "x", "y"], + "properties": { + "kind": { "const": "rect" }, + "width": { "type": "number" }, + "height": { "type": "number" }, + "x": { "type": "number" }, + "y": { "type": "number" } + }, + "additionalProperties": false + } + ] + }, + "geometry": { + "type": "object", + "required": ["shape", "width", "height", "thickness"], + "properties": { + "shape": { "type": "string" }, + "width": { "type": "number" }, + "height": { "type": "number" }, + "thickness": { "type": "number" }, + "area": { "type": "number" }, + "holes": { + "type": "array", + "items": { "$ref": "#/$defs/hole" }, + "default": [] + }, + "rotatable": { "type": "boolean", "default": true } + }, + "additionalProperties": false + }, + "itemBase": { + "type": "object", + "required": ["intent", "source_id", "item_id", "quantity"], + "properties": { + "intent": { + "enum": [ + "fabricated_part", + "purchased_stock", + "hardware", + "allowance", + "exclusion", + "by_others" + ] + }, + "source_id": { "type": "string", "minLength": 1 }, + "item_id": { "type": "string", "minLength": 1 }, + "quantity": { "type": "integer" }, + "mark": { "type": "string" }, + "description": { "type": "string" }, + "source_evidence": { + "type": "array", + "items": { "$ref": "#/$defs/evidence" } + } + } + }, + "fabricatedPart": { + "allOf": [ + { "$ref": "#/$defs/itemBase" }, + { + "type": "object", + "required": ["intent"], + "properties": { + "intent": { "const": "fabricated_part" }, + "material": { "type": "string" }, + "grade": { "type": "string" }, + "geometry": { "$ref": "#/$defs/geometry" }, + "designation": { "type": "string" }, + "length_ft": { "type": "number" }, + "unit_weight_plf": { "type": "number" }, + "total_weight_lbs": { "type": "number" }, + "connections": { "type": "string" }, + "notes": { "type": "string" }, + "identity_warning": { "type": "boolean" } + } + } + ], + "unevaluatedProperties": false + }, + "purchased": { + "allOf": [ + { "$ref": "#/$defs/itemBase" }, + { + "type": "object", + "required": ["intent"], + "properties": { + "intent": { "enum": ["purchased_stock", "hardware"] }, + "material": { "type": "string" }, + "grade": { "type": "string" }, + "specification": { "type": "string" }, + "dimensions": { "type": "object" } + } + } + ], + "unevaluatedProperties": false + }, + "allowance": { + "allOf": [ + { "$ref": "#/$defs/itemBase" }, + { + "type": "object", + "required": ["intent", "allowance_basis"], + "properties": { + "intent": { "const": "allowance" }, + "allowance_basis": { + "type": "object", + "required": ["kind", "value", "applies_to"], + "properties": { + "kind": { "enum": ["percent", "fixed_weight"] }, + "value": { "type": "number" }, + "applies_to": { "type": "string", "minLength": 1 } + }, + "additionalProperties": false + } + } + } + ], + "unevaluatedProperties": false + }, + "nonSupply": { + "allOf": [ + { "$ref": "#/$defs/itemBase" }, + { + "type": "object", + "required": ["intent", "reason"], + "properties": { + "intent": { "enum": ["exclusion", "by_others"] }, + "reason": { "type": "string", "minLength": 1 } + } + } + ], + "unevaluatedProperties": false + }, + "item": { + "oneOf": [ + { "$ref": "#/$defs/fabricatedPart" }, + { "$ref": "#/$defs/purchased" }, + { "$ref": "#/$defs/allowance" }, + { "$ref": "#/$defs/nonSupply" } + ] + }, + "stock": { + "type": "object", + "required": [ + "stock_kind", + "material", + "grade", + "width", + "height", + "thickness", + "quantity" + ], + "properties": { + "stock_kind": { "enum": ["on_hand", "purchasable"] }, + "inventory_id": { "type": "string", "minLength": 1 }, + "material": { "type": "string", "minLength": 1 }, + "grade": { "type": "string", "minLength": 1 }, + "width": { "type": "number" }, + "height": { "type": "number" }, + "thickness": { "type": "number" }, + "quantity": { "type": "integer" }, + "status": { "enum": ["available", "reserved", "unavailable"] }, + "measured_at": { "type": "string", "format": "date" }, + "source": { "type": "string" }, + "reviewer_confirmation": { + "type": "object", + "required": ["actor", "timestamp", "estimate_hash"], + "properties": { + "actor": { "type": "string", "minLength": 1 }, + "timestamp": { "type": "string", "format": "date-time" }, + "estimate_hash": { "type": "string", "pattern": "^[0-9a-f]{64}$" } + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "commercialBasis": { + "type": "object", + "required": ["currency", "costs"], + "properties": { + "currency": { "type": "string", "pattern": "^[A-Z]{3}$" }, + "costs": { + "type": "array", + "items": { + "type": "object", + "required": [ + "cost_id", + "amount", + "currency", + "unit_basis", + "effective_date", + "source" + ], + "properties": { + "cost_id": { "type": "string", "minLength": 1 }, + "amount": { "type": "number" }, + "currency": { "type": "string", "pattern": "^[A-Z]{3}$" }, + "unit_basis": { "type": "string", "minLength": 1 }, + "effective_date": { "type": "string", "format": "date" }, + "source": { "type": "string", "minLength": 1 }, + "synthetic": { "type": "boolean" } + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + }, + "finding": { + "type": "object", + "required": [ + "finding_id", + "code", + "severity", + "path", + "message", + "relevant_hash" + ], + "properties": { + "finding_id": { "type": "string", "minLength": 1 }, + "code": { "type": "string", "minLength": 1 }, + "severity": { "enum": ["blocker", "warning", "info"] }, + "path": { "type": "string", "minLength": 1 }, + "message": { "type": "string", "minLength": 1 }, + "relevant_hash": { "type": "string", "pattern": "^[0-9a-f]{64}$" } + }, + "additionalProperties": false + }, + "review": { + "type": "object", + "required": ["status", "findings", "acknowledgements"], + "properties": { + "status": { + "enum": ["draft", "review_required", "validated", "invalid"] + }, + "findings": { + "type": "array", + "items": { "$ref": "#/$defs/finding" } + }, + "acknowledgements": { + "type": "array", + "items": { + "type": "object", + "required": [ + "finding_id", + "actor", + "timestamp", + "disposition", + "input_hash" + ], + "properties": { + "finding_id": { "type": "string", "minLength": 1 }, + "actor": { "type": "string", "minLength": 1 }, + "timestamp": { "type": "string", "format": "date-time" }, + "disposition": { + "enum": ["accepted", "rejected", "deferred"] + }, + "input_hash": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + } + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + }, + "lineage": { + "type": "object", + "required": ["source_type", "source_hash", "configuration_hash"], + "properties": { + "source_type": { "type": "string", "minLength": 1 }, + "source_hash": { "type": "string", "pattern": "^[0-9a-f]{64}$" }, + "configuration_hash": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "generated_at": { "type": "string", "format": "date-time" } + }, + "additionalProperties": false + } + } +} diff --git a/skills/_shared/schemas/nest-result.schema.json b/skills/_shared/schemas/nest-result.schema.json new file mode 100644 index 0000000..04e09b2 --- /dev/null +++ b/skills/_shared/schemas/nest-result.schema.json @@ -0,0 +1,90 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://structupath.ai/schemas/pi-steel/nest-result-1.0.0.json", + "title": "pi-steel canonical nest result", + "type": "object", + "required": [ + "schema_version", + "estimate_input_hash", + "configuration_hash", + "outcome", + "package_status", + "groups", + "unplaced" + ], + "properties": { + "schema_version": { "const": "1.0.0" }, + "estimate_input_hash": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "configuration_hash": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "outcome": { + "enum": [ + "ready", + "review_required", + "blocked", + "dependency_missing", + "usage_or_internal_error" + ] + }, + "package_status": { + "enum": [ + "draft", + "review_required", + "validated", + "nested_partial", + "nest_verified", + "rfq_draft_review_required", + "rfq_ready_for_review" + ] + }, + "groups": { + "type": "array", + "items": { + "type": "object", + "required": ["group_id", "material", "grade", "thickness", "placements"], + "properties": { + "group_id": { "type": "string", "minLength": 1 }, + "material": { "type": "string", "minLength": 1 }, + "grade": { "type": "string", "minLength": 1 }, + "thickness": { "type": "number", "exclusiveMinimum": 0 }, + "placements": { + "type": "array", + "items": { + "type": "object", + "required": ["placement_id", "item_id", "instance_id", "stock_id"], + "properties": { + "placement_id": { "type": "string", "minLength": 1 }, + "item_id": { "type": "string", "minLength": 1 }, + "instance_id": { "type": "string", "minLength": 1 }, + "stock_id": { "type": "string", "minLength": 1 }, + "x": { "type": "number" }, + "y": { "type": "number" }, + "rotated": { "type": "boolean" } + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + } + }, + "unplaced": { + "type": "array", + "items": { + "type": "object", + "required": ["instance_id", "reason"], + "properties": { + "instance_id": { "type": "string", "minLength": 1 }, + "reason": { "type": "string", "minLength": 1 } + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false +} diff --git a/skills/steel-nest/references/example_job.json b/skills/steel-nest/references/example_job.json index 01072dc..4b3b616 100644 --- a/skills/steel-nest/references/example_job.json +++ b/skills/steel-nest/references/example_job.json @@ -1,6 +1,9 @@ { "job_name": "SYNTHETIC-DEMO-001 Generated Training Geometry", "customer": "Example Customer", + "material": "carbon_steel", + "grade": "A36", + "unit_system": "imperial", "settings": { "kerf_in": 0.06, "part_gap_in": 0.25, @@ -18,23 +21,23 @@ } ], "parts": [ - { "name": "Training Rect A", "width": 11, "height": 7, "qty": 3, "shape": "rect", + { "source_id": "SYNTHETIC-SRC-RECT-A", "name": "Training Rect A", "width": 11, "height": 7, "qty": 3, "shape": "rect", "holes": [ { "dia": 0.75, "x": 2, "y": 2 }, { "dia": 0.75, "x": 9, "y": 5 } ] }, - { "name": "Training Rect B", "width": 13, "height": 9, "qty": 2, "shape": "rect", + { "source_id": "SYNTHETIC-SRC-RECT-B", "name": "Training Rect B", "width": 13, "height": 9, "qty": 2, "shape": "rect", "holes": [ { "dia": 0.625, "x": 3, "y": 3 }, { "dia": 0.625, "x": 10, "y": 6 } ] }, - { "name": "Training Irregular C", "width": 9, "height": 8, "qty": 4, "shape": "irregular", "area": 28 }, - { "name": "Training Rect D", "width": 17, "height": 5, "qty": 5, "shape": "rect" }, - { "name": "Training Rect E", "width": 19, "height": 11, "qty": 2, "shape": "rect", + { "source_id": "SYNTHETIC-SRC-IRREGULAR-C", "name": "Training Irregular C", "width": 9, "height": 8, "qty": 4, "shape": "irregular", "area": 28 }, + { "source_id": "SYNTHETIC-SRC-RECT-D", "name": "Training Rect D", "width": 17, "height": 5, "qty": 5, "shape": "rect" }, + { "source_id": "SYNTHETIC-SRC-RECT-E", "name": "Training Rect E", "width": 19, "height": 11, "qty": 2, "shape": "rect", "holes": [ { "w": 5, "h": 3, "x": 9.5, "y": 5.5 } ] }, - { "name": "Training Rect F", "width": 6, "height": 4, "qty": 7, "shape": "rect", + { "source_id": "SYNTHETIC-SRC-RECT-F", "name": "Training Rect F", "width": 6, "height": 4, "qty": 7, "shape": "rect", "holes": [ { "dia": 0.5, "x": 3, "y": 2 } ] } ] } diff --git a/skills/steel-nest/references/job_template.json b/skills/steel-nest/references/job_template.json index f9d2210..990aab4 100644 --- a/skills/steel-nest/references/job_template.json +++ b/skills/steel-nest/references/job_template.json @@ -1,6 +1,9 @@ { "job_name": "SYNTHETIC-DEMO-XXX Short description", "customer": "Example Customer", + "material": "carbon_steel", + "grade": "A36", + "unit_system": "imperial", "_comment_settings": "Cut/gap in inches. kerf = torch cut width (plasma ~0.06, oxy ~0.1, laser ~0.02). part_gap = clearance between parts on top of kerf. edge_margin = keep-out from plate edge (clamp/grip zone). density lb/in^3: A36 steel = 0.2836, aluminum = 0.098, stainless 304 = 0.289.", "settings": { @@ -25,13 +28,13 @@ "_comment_parts": "One entry per unique part. width/height in inches (bounding box for irregular parts). qty = pieces needed. rotatable=false to lock grain/rolling direction. shape: \"rect\" nests exactly; \"irregular\" nests by bounding box — add \"area\" (true cut area in^2) for exact weight/cost on those.", "_comment_holes": "Optional per-part \"holes\": each hole's x,y is its CENTER measured from the part's lower-left corner. Round hole = {\"dia\": D, \"x\":, \"y\":}. Rectangular cutout = {\"w\":, \"h\":, \"x\":, \"y\":}. Holes are subtracted from weight/cost, rotate with the part, and are cut as real geometry on the HOLES layer of the burn-table DXF.", "parts": [ - { "name": "Part A", "width": 12, "height": 8, "qty": 6, "shape": "rect", + { "source_id": "SYNTHETIC-SRC-PART-A", "name": "Part A", "width": 12, "height": 8, "qty": 6, "shape": "rect", "holes": [ { "dia": 0.875, "x": 2, "y": 2 }, { "dia": 0.875, "x": 10, "y": 6 } ] }, - { "name": "Part B", "width": 10, "height": 10, "qty": 4, "shape": "irregular", "area": 62 }, - { "name": "Part C", "width": 24, "height": 3, "qty": 10, "shape": "rect", "rotatable": false, + { "source_id": "SYNTHETIC-SRC-PART-B", "name": "Part B", "width": 10, "height": 10, "qty": 4, "shape": "irregular", "area": 62 }, + { "source_id": "SYNTHETIC-SRC-PART-C", "name": "Part C", "width": 24, "height": 3, "qty": 10, "shape": "rect", "rotatable": false, "holes": [ { "w": 3, "h": 1, "x": 12, "y": 1.5 } ] } ] } diff --git a/skills/steel-takeoff/assets/bom-template.csv b/skills/steel-takeoff/assets/bom-template.csv index 05d7cbf..a50233a 100644 --- a/skills/steel-takeoff/assets/bom-template.csv +++ b/skills/steel-takeoff/assets/bom-template.csv @@ -1 +1 @@ -Mark,Qty,Size,Grade,Length_ft,Unit_Wt_plf,Total_Wt_lbs,Connections,Notes +Source_ID,Mark,Qty,Size,Grade,Length_ft,Unit_Wt_plf,Total_Wt_lbs,Connections,Notes,Source_Sheet,Source_Detail,Intent diff --git a/skills/steel-takeoff/scripts/calculate-weight.sh b/skills/steel-takeoff/scripts/calculate-weight.sh index 913ed6c..925498e 100755 --- a/skills/steel-takeoff/scripts/calculate-weight.sh +++ b/skills/steel-takeoff/scripts/calculate-weight.sh @@ -4,7 +4,7 @@ # Usage: ./scripts/calculate-weight.sh path/to/bom.csv [connection_pct] [misc_pct] # # BOM CSV format: -# Mark,Qty,Size,Grade,Length_ft,Unit_Wt_plf,Total_Wt_lbs,Connections,Notes +# Source_ID,Mark,Qty,Size,Grade,Length_ft,Unit_Wt_plf,... # # Optional arguments: # connection_pct - Connection allowance percentage (default: 12) @@ -68,7 +68,7 @@ with open(bom_file) as f: qty = int(row.get("Qty", 0)) size = row.get("Size", "?").strip() - grade = row.get("Grade", "A992").strip() + grade = row.get("Grade", "").strip() or "UNSPECIFIED" length_str = row.get("Length_ft", "0").strip() unit_wt_str = row.get("Unit_Wt_plf", "0").strip() @@ -136,6 +136,8 @@ print(f" {'Misc Steel Allow ({:.0f}%):':<30} {misc_wt:>12,.0f} lb ({misc_wt/20 print(f" {'─'*60}") print(f" {'GRAND TOTAL:':<30} {grand_total:>12,.0f} lb ({grand_total/2000:>8,.1f} tons)") print(f" {'─'*60}") +print("\n No price was calculated. Pricing requires an explicit currency, unit basis,") +print(" effective date, and source supplied by the project input.") print(f"\n{'='*78}\n") PYTHON diff --git a/skills/steel-takeoff/scripts/validate-bom.py b/skills/steel-takeoff/scripts/validate-bom.py index 83b9500..5ca83be 100755 --- a/skills/steel-takeoff/scripts/validate-bom.py +++ b/skills/steel-takeoff/scripts/validate-bom.py @@ -1,218 +1,173 @@ #!/usr/bin/env python3 -""" -BOM Validator — Validates a steel bill of materials CSV. +"""Validate a legacy BOM through the canonical estimate-package contract.""" -Usage: python3 scripts/validate-bom.py path/to/bom.csv +from __future__ import annotations -Checks: - ✓ Valid AISC designations (cross-references shapes database) - ✓ Non-zero quantities and lengths - ✓ Correct grade for shape type - ✓ Reasonable weight-per-foot values - ✓ No duplicate marks - ✓ Required columns present -""" - -import csv +import argparse import json -import os import sys from pathlib import Path -def load_shapes_db(): - """Load the AISC shapes database.""" - skill_dir = Path(__file__).resolve().parent.parent - db_path = skill_dir / "assets" / "aisc-shapes-database.json" - if not db_path.exists(): - print(f"WARNING: Shapes database not found at {db_path}") - return {} - with open(db_path) as f: - shapes = json.load(f) +SHARED_ROOT = Path(__file__).resolve().parents[2] / "_shared" +if str(SHARED_ROOT) not in sys.path: + sys.path.insert(0, str(SHARED_ROOT)) +from bootstrap import bootstrap_shared # noqa: E402 - return {s["designation"]: s for s in shapes} +bootstrap_shared(__file__) +from pi_steel.parsing import adapt_legacy_bom_csv # noqa: E402 +from pi_steel.validation import validate_estimate_package # noqa: E402 -def normalize_designation(raw: str) -> str: - """Normalize a steel designation for lookup.""" - return raw.upper().replace(" ", "").strip() +def load_shapes_db() -> dict[str, dict]: + db_path = Path(__file__).resolve().parent.parent / "assets" / "aisc-shapes-database.json" + if not db_path.exists(): + return {} + shapes = json.loads(db_path.read_text(encoding="utf-8")) + return {shape["designation"]: shape for shape in shapes} -def validate_grade(shape_type: str, grade: str) -> list[str]: - """Check if the material grade is appropriate for the shape type.""" - warnings = [] - grade_upper = grade.upper().strip() +def normalize_designation(raw: str) -> str: + return raw.upper().replace(" ", "").strip() - # Standard grade assignments - standard_grades = { - "W": ["A992", "A572", "A913", "A36"], - "HSS-RECT": ["A500", "A500 GR.C", "A500 GR. C", "A500GRC", "A500C"], - "HSS-RND": ["A500", "A500 GR.B", "A500 GR. B", "A500GRB", "A500B", "A53"], - "C": ["A36", "A572"], - "L": ["A36", "A572"], - "PIPE": ["A53", "A53 GR.B", "A53B", "A500"], - } - if shape_type in standard_grades: - valid = standard_grades[shape_type] - # Check if grade matches any valid option - matches = any( - grade_upper.replace(" ", "").replace(".", "").startswith(v.replace(" ", "").replace(".", "")) - for v in valid +def grade_warnings(shape_type: str, grade: str) -> list[str]: + normalized = grade.upper().replace(" ", "").replace(".", "") + expected = { + "W": ("A992", "A572", "A913", "A36"), + "HSS-RECT": ("A500", "A500GRC"), + "HSS-RND": ("A500", "A500GRB", "A53"), + "C": ("A36", "A572"), + "L": ("A36", "A572"), + "PIPE": ("A53", "A500"), + }.get(shape_type, ()) + if expected and not any(normalized.startswith(value) for value in expected): + return [f"Grade {grade!r} is unusual for shape type {shape_type}."] + return [] + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser(description=__doc__) + result.add_argument("bom", type=Path) + result.add_argument( + "--project-id", + default="SYNTHETIC-UNSPECIFIED", + help="Stable project identity used to derive canonical item IDs.", + ) + result.add_argument( + "--revision-id", + default="SYNTHETIC-REV-UNSPECIFIED", + help="Source revision identity used to derive canonical item IDs.", + ) + result.add_argument( + "--json", + action="store_true", + help="Print the validation result as JSON.", + ) + return result + + +def main(argv: list[str] | None = None) -> int: + args = parser().parse_args(argv) + if not args.bom.is_file(): + print(f"ERROR: File not found: {args.bom}", file=sys.stderr) + return 1 + try: + package = adapt_legacy_bom_csv( + args.bom, + project_id=args.project_id, + revision_id=args.revision_id, ) - if not matches: - warnings.append( - f"Grade '{grade}' unusual for {shape_type} — " - f"expected one of: {', '.join(standard_grades[shape_type][:3])}" + except (OSError, ValueError, TypeError) as exc: + print(f"ERROR: Could not parse BOM: {exc}", file=sys.stderr) + return 1 + + result = validate_estimate_package(package) + findings = list(result.active_findings) + shapes = load_shapes_db() + for index, item in enumerate(package["items"]): + designation = normalize_designation(item.get("designation", "")) + shape = shapes.get(designation) + if designation and shapes and shape is None and not designation.startswith( + ("PL", "PLATE", "BU", "WT") + ): + findings.append( + { + "finding_id": f"legacy-shape:{index}", + "code": "unverified_designation", + "severity": "warning", + "path": f"$.items[{index}].designation", + "message": f"{item.get('designation')!r} was not found in the bundled reference data.", + "relevant_hash": result.input_hash, + } + ) + elif shape is not None: + unit_weight = item.get("unit_weight_plf") + if unit_weight and abs(unit_weight - shape["weight_per_ft"]) > 0.5: + findings.append( + { + "finding_id": f"legacy-weight:{index}", + "code": "unit_weight_mismatch", + "severity": "warning", + "path": f"$.items[{index}].unit_weight_plf", + "message": ( + f"Unit weight {unit_weight:g} plf does not match the " + f"bundled reference value {shape['weight_per_ft']:g} plf." + ), + "relevant_hash": result.input_hash, + } + ) + for message in grade_warnings(shape["type"], item.get("grade", "")): + findings.append( + { + "finding_id": f"legacy-grade:{index}", + "code": "unusual_grade", + "severity": "warning", + "path": f"$.items[{index}].grade", + "message": message, + "relevant_hash": result.input_hash, + } + ) + if item.get("length_ft", 0) > 80: + findings.append( + { + "finding_id": f"legacy-length:{index}", + "code": "unusual_member_length", + "severity": "warning", + "path": f"$.items[{index}].length_ft", + "message": "Member length exceeds 80 ft; verify the source.", + "relevant_hash": result.input_hash, + } ) - return warnings - - -def main(): - if len(sys.argv) < 2: - print("Usage: python3 validate-bom.py ") - sys.exit(1) - - bom_file = sys.argv[1] - if not os.path.exists(bom_file): - print(f"ERROR: File not found: {bom_file}") - sys.exit(1) - - shapes_db = load_shapes_db() - errors = [] - warnings = [] - marks_seen = {} - total_weight = 0 - line_num = 0 - - required_columns = {"Mark", "Qty", "Size", "Length_ft", "Unit_Wt_plf"} - - print(f"\n{'='*60}") - print(f" BOM VALIDATION: {os.path.basename(bom_file)}") - print(f"{'='*60}\n") - - with open(bom_file) as f: - reader = csv.DictReader(f) - - # Check required columns - if reader.fieldnames: - actual_cols = set(reader.fieldnames) - missing_cols = required_columns - actual_cols - if missing_cols: - errors.append(f"Missing required columns: {', '.join(missing_cols)}") - - for row in reader: - line_num += 1 - mark = row.get("Mark", "").strip() - if not mark or mark.startswith("#"): - continue - - size = row.get("Size", "").strip() - grade = row.get("Grade", "").strip() - qty_str = row.get("Qty", "0").strip() - length_str = row.get("Length_ft", "0").strip() - unit_wt_str = row.get("Unit_Wt_plf", "0").strip() - - prefix = f"Line {line_num} ({mark})" - - # Check for duplicate marks - if mark in marks_seen: - warnings.append(f"{prefix}: Duplicate mark (also on line {marks_seen[mark]})") - marks_seen[mark] = line_num - - # Validate quantity - try: - qty = int(qty_str) if qty_str else 0 - if qty <= 0: - errors.append(f"{prefix}: Quantity is {qty} — must be > 0") - except ValueError: - errors.append(f"{prefix}: Invalid quantity '{qty_str}'") - qty = 0 - - # Validate length - try: - if "'" in length_str: - parts = length_str.replace('"', '').split("'") - feet = float(parts[0].replace('-', '').strip()) - inches = float(parts[1].replace('-', '').strip()) if len(parts) > 1 and parts[1].strip() else 0 - length = feet + inches / 12.0 - else: - length = float(length_str) if length_str else 0 - if length <= 0: - errors.append(f"{prefix}: Length is {length} — must be > 0") - elif length > 80: - warnings.append(f"{prefix}: Length {length:.1f} ft exceeds typical max (80 ft) — verify") - except ValueError: - errors.append(f"{prefix}: Invalid length '{length_str}'") - length = 0 - - # Validate unit weight - try: - unit_wt = float(unit_wt_str) if unit_wt_str else 0 - if unit_wt <= 0: - errors.append(f"{prefix}: Unit weight is {unit_wt} — must be > 0") - except ValueError: - errors.append(f"{prefix}: Invalid unit weight '{unit_wt_str}'") - unit_wt = 0 - - # Validate AISC designation - normalized = normalize_designation(size) - if shapes_db: - if normalized in shapes_db: - db_shape = shapes_db[normalized] - db_wt = db_shape["weight_per_ft"] - if unit_wt > 0 and abs(unit_wt - db_wt) > 0.5: - warnings.append( - f"{prefix}: Unit weight {unit_wt} plf doesn't match " - f"AISC database ({db_wt} plf) for {size}" - ) - # Validate grade - if grade: - grade_warns = validate_grade(db_shape["type"], grade) - warnings.extend(f"{prefix}: {w}" for w in grade_warns) - else: - # Not a fatal error — could be a plate or built-up section - if not any(normalized.startswith(p) for p in ["PL", "PLATE", "BU", "WT"]): - warnings.append(f"{prefix}: '{size}' not found in AISC database — verify designation") - - # Validate grade is present - if not grade: - warnings.append(f"{prefix}: No material grade specified") - - total_weight += qty * length * unit_wt - - # Print results - if errors: - print(" ❌ ERRORS (must fix):") - for e in errors: - print(f" • {e}") - print() - - if warnings: - print(" ⚠️ WARNINGS (review):") - for w in warnings: - print(f" • {w}") - print() - - print(f" 📊 SUMMARY:") - print(f" Lines validated: {line_num}") - print(f" Unique marks: {len(marks_seen)}") - print(f" Errors: {len(errors)}") - print(f" Warnings: {len(warnings)}") - print(f" Total weight: {total_weight:,.0f} lbs ({total_weight/2000:,.1f} tons)") - - if errors: - print(f"\n ❌ VALIDATION FAILED — {len(errors)} error(s) found") - print(f"{'='*60}\n") - sys.exit(1) - elif warnings: - print(f"\n ⚠️ PASSED WITH WARNINGS — review {len(warnings)} warning(s)") - print(f"{'='*60}\n") + total_weight = sum( + item["quantity"] * item.get("length_ft", 0) * item.get("unit_weight_plf", 0) + for item in package["items"] + if item.get("intent") == "fabricated_part" + ) + blockers = [finding for finding in findings if finding["severity"] == "blocker"] + warnings = [finding for finding in findings if finding["severity"] == "warning"] + output = { + "status": "invalid" if blockers else ("review_required" if warnings else "validated"), + "input_hash": result.input_hash, + "line_count": len(package["items"]), + "total_weight_lbs": total_weight, + "findings": findings, + } + if args.json: + print(json.dumps(output, indent=2, sort_keys=True)) else: - print(f"\n ✅ VALIDATION PASSED — no issues found") - print(f"{'='*60}\n") + print(f"BOM VALIDATION: {args.bom.name}") + for finding in findings: + label = "ERROR" if finding["severity"] == "blocker" else finding["severity"].upper() + print(f"{label}: {finding['path']}: {finding['message']}") + print( + f"SUMMARY: {len(package['items'])} lines; " + f"{total_weight:,.0f} lb; {len(blockers)} blockers; {len(warnings)} warnings" + ) + print(f"STATUS: {output['status']}") + return 1 if blockers else 0 if __name__ == "__main__": - main() + raise SystemExit(main()) diff --git a/tests/fixtures/contracts/README.md b/tests/fixtures/contracts/README.md new file mode 100644 index 0000000..82151b8 --- /dev/null +++ b/tests/fixtures/contracts/README.md @@ -0,0 +1,5 @@ +# Contract fixtures + +Every file in this directory was created from scratch for automated tests. Project +identifiers are synthetic, names use public-policy placeholders, and no fixture is +derived from a production estimate, drawing, BOM, inventory record, or nest. diff --git a/tests/fixtures/contracts/legacy-bom.csv b/tests/fixtures/contracts/legacy-bom.csv new file mode 100644 index 0000000..591cc4f --- /dev/null +++ b/tests/fixtures/contracts/legacy-bom.csv @@ -0,0 +1,3 @@ +Source_ID,Mark,Qty,Size,Grade,Length_ft,Unit_Wt_plf,Total_Wt_lbs,Connections,Notes,Source_Sheet,Source_Detail,Intent +SYNTHETIC-SRC-B1,B1,2,W12X26,A992,12.5,26,650,BOLTED,Synthetic member,SYNTHETIC-S1,SYNTHETIC-D1,fabricated_part +SYNTHETIC-SRC-H1,H1,1,HSS6X6X3/8,A500 GR.C,8'-6",27.48,233.58,WELDED,Synthetic member,SYNTHETIC-S2,SYNTHETIC-D2,fabricated_part diff --git a/tests/fixtures/contracts/legacy-nest.json b/tests/fixtures/contracts/legacy-nest.json new file mode 100644 index 0000000..14a0a2f --- /dev/null +++ b/tests/fixtures/contracts/legacy-nest.json @@ -0,0 +1,32 @@ +{ + "job_name": "SYNTHETIC-CONTRACT-NEST", + "customer": "Example Customer", + "material": "carbon_steel", + "grade": "A36", + "unit_system": "imperial", + "settings": { + "thickness_in": 0.5, + "kerf_in": 0.06, + "part_gap_in": 0.25, + "edge_margin_in": 0.5 + }, + "stock": [ + { + "name": "Synthetic Plate", + "width": 48, + "height": 24, + "thickness": 0.5, + "qty": 1 + } + ], + "parts": [ + { + "source_id": "SYNTHETIC-SRC-P1", + "name": "SYNTHETIC-P1", + "width": 8, + "height": 6, + "qty": 2, + "shape": "rect" + } + ] +} diff --git a/tests/test_contracts.py b/tests/test_contracts.py new file mode 100644 index 0000000..b44ceeb --- /dev/null +++ b/tests/test_contracts.py @@ -0,0 +1,367 @@ +import copy +import json +import math +import sys +from pathlib import Path + +import jsonschema +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +SHARED = ROOT / "skills" / "_shared" +sys.path.insert(0, str(SHARED)) + +from pi_steel.contracts import ( + ESTIMATE_PACKAGE_VERSION, + NEST_RESULT_VERSION, + instance_ids, + item_id_for, + placement_ids, +) +from pi_steel.run_manifest import PACKAGE_STATUSES, RUN_OUTCOMES +from pi_steel.geometry_verify import group_plate_items +from pi_steel.parsing import adapt_legacy_bom_csv, adapt_legacy_nest +from pi_steel.validation import ( + acknowledge_finding, + eligible_on_hand_stock, + purchasable_items, + stock_requiring_purchase, + validate_estimate_package, +) + + +FIXTURES = ROOT / "tests" / "fixtures" / "contracts" + + +def valid_package(): + source_id = "SYNTHETIC-SRC-P1" + return { + "schema_version": ESTIMATE_PACKAGE_VERSION, + "project": { + "project_id": "SYNTHETIC-CONTRACT-001", + "revision": {"revision_id": "SYNTHETIC-REV-A"}, + }, + "unit_system": "imperial", + "items": [ + { + "intent": "fabricated_part", + "source_id": source_id, + "item_id": item_id_for( + "SYNTHETIC-CONTRACT-001", "SYNTHETIC-REV-A", source_id + ), + "quantity": 2, + "mark": "SYNTHETIC-P1", + "material": "carbon_steel", + "grade": "A36", + "geometry": { + "shape": "rect", + "width": 8, + "height": 6, + "thickness": 0.5, + "holes": [{"kind": "round", "diameter": 1, "x": 2, "y": 2}], + }, + "source_evidence": [ + { + "source": "SYNTHETIC-DRAWING", + "locator": "SYNTHETIC-DETAIL-1", + } + ], + } + ], + "stock": [], + "commercial_basis": {"currency": "USD", "costs": []}, + "review": {"status": "draft", "findings": [], "acknowledgements": []}, + "lineage": { + "source_type": "synthetic_test", + "source_hash": "a" * 64, + "configuration_hash": "b" * 64, + }, + } + + +def blocker_codes(result): + return {f["code"] for f in result.findings if f["severity"] == "blocker"} + + +def warning_codes(result): + return {f["code"] for f in result.findings if f["severity"] == "warning"} + + +def test_contract_schemas_are_draft_2020_12_and_accept_canonical_package(): + package = valid_package() + estimate_schema = json.loads( + (SHARED / "schemas" / "estimate-package.schema.json").read_text() + ) + nest_schema = json.loads( + (SHARED / "schemas" / "nest-result.schema.json").read_text() + ) + assert estimate_schema["$schema"].endswith("/draft/2020-12/schema") + assert nest_schema["$schema"].endswith("/draft/2020-12/schema") + jsonschema.Draft202012Validator(estimate_schema).validate(package) + assert nest_schema["properties"]["schema_version"]["const"] == NEST_RESULT_VERSION + assert set(nest_schema["properties"]["outcome"]["enum"]) == set(RUN_OUTCOMES) + assert set(nest_schema["properties"]["package_status"]["enum"]) == set( + PACKAGE_STATUSES + ) + + +def test_legacy_bom_preserves_marks_grades_lengths_weights_and_explicit_ids(): + package = adapt_legacy_bom_csv( + FIXTURES / "legacy-bom.csv", + project_id="SYNTHETIC-CONTRACT-001", + revision_id="SYNTHETIC-REV-A", + ) + rows = package["items"] + assert [row["mark"] for row in rows] == ["B1", "H1"] + assert [row["grade"] for row in rows] == ["A992", "A500 GR.C"] + assert [row["length_ft"] for row in rows] == [12.5, 8.5] + assert [row["unit_weight_plf"] for row in rows] == [26.0, 27.48] + assert [row["source_id"] for row in rows] == [ + "SYNTHETIC-SRC-B1", + "SYNTHETIC-SRC-H1", + ] + + +@pytest.mark.parametrize( + ("mutation", "expected_code"), + [ + (lambda p: p["items"][0].update(quantity=0), "invalid_quantity"), + ( + lambda p: p["items"][0]["geometry"].update(width=math.inf), + "nonfinite_dimension", + ), + ( + lambda p: p["items"][0]["geometry"].update(shape="ellipse"), + "unsupported_shape", + ), + ( + lambda p: p["items"][0]["geometry"].update( + holes=[{"kind": "round", "diameter": 4, "x": 0.5, "y": 0.5}] + ), + "hole_out_of_bounds", + ), + ( + lambda p: p["items"][0]["geometry"].update( + width=1, + height=1, + holes=[{"kind": "round", "diameter": 2, "x": 0.5, "y": 0.5}], + ), + "nonpositive_net_area", + ), + ], +) +def test_field_specific_geometry_and_quantity_blockers(mutation, expected_code): + package = valid_package() + mutation(package) + result = validate_estimate_package(package) + assert expected_code in blocker_codes(result) + assert all(f["path"] for f in result.findings) + + +def test_duplicate_source_and_item_ids_are_blockers(): + package = valid_package() + package["items"].append(copy.deepcopy(package["items"][0])) + codes = blocker_codes(validate_estimate_package(package)) + assert {"duplicate_source_id", "duplicate_item_id"} <= codes + + +def test_grouping_never_mixes_material_grade_or_thickness(): + package = valid_package() + items = [package["items"][0]] + for field, value in (("grade", "A572"), ("material", "stainless_steel")): + changed = copy.deepcopy(items[0]) + changed["item_id"] += f"-{field}" + changed[field] = value + items.append(changed) + changed = copy.deepcopy(items[0]) + changed["item_id"] += "-thickness" + changed["geometry"]["thickness"] = 0.375 + items.append(changed) + groups = group_plate_items(items) + assert len(groups) == 4 + + +def test_allowances_are_totals_only_and_never_purchased(): + package = valid_package() + package["items"].append( + { + "intent": "allowance", + "source_id": "SYNTHETIC-SRC-ALLOWANCE", + "item_id": "item:synthetic-allowance", + "quantity": 1, + "description": "Synthetic connection allowance", + "allowance_basis": { + "kind": "percent", + "value": 10, + "applies_to": "fabricated_weight", + }, + } + ) + assert [item["intent"] for item in purchasable_items(package)] == [ + "fabricated_part" + ] + + +def test_on_hand_stock_requires_traceable_confirmed_inventory(): + package = valid_package() + stock = { + "stock_kind": "on_hand", + "inventory_id": "SYNTHETIC-INV-1", + "material": "carbon_steel", + "grade": "A36", + "width": 48, + "height": 24, + "thickness": 0.5, + "quantity": 1, + "status": "available", + "measured_at": "2026-07-28", + "source": "SYNTHETIC-INVENTORY", + "reviewer_confirmation": { + "actor": "Example Reviewer", + "timestamp": "2026-07-28T12:00:00Z", + "estimate_hash": "", + }, + } + package["stock"] = [stock] + assert eligible_on_hand_stock(package) == [] + stock["reviewer_confirmation"]["estimate_hash"] = validate_estimate_package( + package + ).input_hash + assert eligible_on_hand_stock(package) == [stock] + assert stock_requiring_purchase(package) == [] + del stock["inventory_id"] + assert eligible_on_hand_stock(package) == [] + assert stock_requiring_purchase(package) == [] + + +def test_missing_source_evidence_is_visible_warning_and_ack_is_hash_bound(): + package = valid_package() + package["items"][0].pop("source_evidence") + result = validate_estimate_package(package) + assert "missing_source_evidence" in warning_codes(result) + finding = next(f for f in result.findings if f["code"] == "missing_source_evidence") + acknowledge_finding( + package, + finding, + actor="Example Reviewer", + timestamp="2026-07-28T12:00:00Z", + disposition="accepted", + ) + assert not any( + f["code"] == "missing_source_evidence" + for f in validate_estimate_package(package).active_findings + ) + package["project"]["revision"]["revision_id"] = "SYNTHETIC-REV-B" + assert "missing_source_evidence" in { + f["code"] for f in validate_estimate_package(package).active_findings + } + + +def test_legacy_nest_inherits_explicit_job_basis_but_never_invents_missing_basis(): + legacy = json.loads((FIXTURES / "legacy-nest.json").read_text()) + package = adapt_legacy_nest( + legacy, + project_id="SYNTHETIC-CONTRACT-001", + revision_id="SYNTHETIC-REV-A", + ) + part = package["items"][0] + assert (part["material"], part["grade"], part["geometry"]["thickness"]) == ( + "carbon_steel", + "A36", + 0.5, + ) + legacy.pop("grade") + missing = adapt_legacy_nest( + legacy, + project_id="SYNTHETIC-CONTRACT-001", + revision_id="SYNTHETIC-REV-A", + ) + result = validate_estimate_package(missing) + assert result.status == "review_required" + assert "missing_material_basis" in blocker_codes(result) + + +def test_unknown_version_gets_migration_diagnostic(): + package = valid_package() + package["schema_version"] = "99.0.0" + result = validate_estimate_package(package) + assert result.status == "invalid" + assert result.findings[0]["code"] == "unsupported_contract_version" + assert "migrate" in result.findings[0]["message"].lower() + + +def test_identifiers_are_reorder_stable_and_quantity_expansion_is_predictable(): + source_ids = ["SYNTHETIC-SRC-A", "SYNTHETIC-SRC-B"] + forward = [ + item_id_for("SYNTHETIC-PROJECT", "SYNTHETIC-REV-A", source) + for source in source_ids + ] + reverse = [ + item_id_for("SYNTHETIC-PROJECT", "SYNTHETIC-REV-A", source) + for source in reversed(source_ids) + ] + assert forward == list(reversed(reverse)) + assert item_id_for( + "SYNTHETIC-PROJECT", "SYNTHETIC-REV-B", source_ids[0] + ) == item_id_for( + "SYNTHETIC-PROJECT", "SYNTHETIC-REV-A", source_ids[0] + ) + assert instance_ids(forward[0], 2) == instance_ids(forward[0], 3)[:2] + assert placement_ids(forward[0], 2) == placement_ids(forward[0], 3)[:2] + + +def test_acknowledgement_cannot_waive_a_blocking_validation_error(): + package = valid_package() + package["items"][0]["quantity"] = 0 + result = validate_estimate_package(package) + finding = next(f for f in result.findings if f["code"] == "invalid_quantity") + acknowledge_finding( + package, + finding, + actor="Example Reviewer", + timestamp="2026-07-28T12:00:00Z", + disposition="accepted", + ) + + assert "invalid_quantity" in blocker_codes(validate_estimate_package(package)) + + +def test_missing_legacy_mark_uses_revision_scoped_id_and_duplicate_rows_do_not_merge( + tmp_path, +): + csv_path = tmp_path / "synthetic-unmarked.csv" + csv_path.write_text( + "Mark,Qty,Size,Grade,Length_ft,Unit_Wt_plf\n" + ",1,W8X10,A992,4,10\n" + ",1,W8X10,A992,4,10\n", + encoding="utf-8", + ) + package = adapt_legacy_bom_csv( + csv_path, + project_id="SYNTHETIC-CONTRACT-001", + revision_id="SYNTHETIC-REV-A", + ) + assert package["items"][0]["source_id"].startswith("legacy:SYNTHETIC-REV-A:") + assert "unstable_source_identity" in warning_codes( + validate_estimate_package(package) + ) + assert "duplicate_source_id" in blocker_codes(validate_estimate_package(package)) + + +def test_absent_costs_do_not_create_prices_and_explicit_cost_preserves_basis(): + package = valid_package() + assert package["commercial_basis"]["costs"] == [] + package["commercial_basis"]["costs"].append( + { + "cost_id": "SYNTHETIC-COST-1", + "amount": 1, + "currency": "USD", + "unit_basis": "per_pound", + "effective_date": "2026-07-28", + "source": "synthetic_test_input", + "synthetic": True, + } + ) + result = validate_estimate_package(package) + assert "invalid_cost_basis" not in blocker_codes(result) From b27b7bfa78e8d681c218e9ce4394d87d5fab98cf Mon Sep 17 00:00:00 2001 From: Victor Garcia Date: Tue, 28 Jul 2026 12:30:26 -0600 Subject: [PATCH 07/15] feat(nest): verify grouped placements and metrics --- skills/_shared/pi_steel/geometry_verify.py | 113 ++ .../_shared/schemas/nest-result.schema.json | 342 ++++- skills/steel-nest/SKILL.md | 34 +- skills/steel-nest/references/example_job.json | 2 +- .../steel-nest/references/job_template.json | 4 +- skills/steel-nest/scripts/nest.py | 1169 +++++++++++++---- tests/fixtures/nest/README.md | 4 + tests/fixtures/nest/grouped-engine.json | 57 + .../nest/irregular-reference-only.json | 6 +- tests/test_nest_burn_guard.py | 4 + tests/test_nest_cli_contract.py | 121 ++ tests/test_nest_engine.py | 197 +++ tests/test_nest_invariants.py | 89 ++ tests/test_nest_outputs.py | 7 +- 14 files changed, 1832 insertions(+), 317 deletions(-) create mode 100644 tests/fixtures/nest/grouped-engine.json create mode 100644 tests/test_nest_cli_contract.py create mode 100644 tests/test_nest_engine.py create mode 100644 tests/test_nest_invariants.py diff --git a/skills/_shared/pi_steel/geometry_verify.py b/skills/_shared/pi_steel/geometry_verify.py index 61312b6..72db803 100644 --- a/skills/_shared/pi_steel/geometry_verify.py +++ b/skills/_shared/pi_steel/geometry_verify.py @@ -73,3 +73,116 @@ def group_plate_items(items: list[dict[str, Any]]) -> list[dict[str, Any]]: } for key, grouped in sorted(groups.items(), key=lambda pair: repr(pair[0])) ] + + +def verify_nest_placements( + plate_reports: list[dict[str, Any]], + *, + edge_margin: float, + inter_part_clearance: float, +) -> list[dict[str, Any]]: + """Verify normalized placements without relying on a packing algorithm's state.""" + findings: list[dict[str, Any]] = [] + epsilon = 1e-9 + for plate_index, plate in enumerate(plate_reports): + usable_width = plate.get("W", 0) - 2 * edge_margin + usable_height = plate.get("H", 0) - 2 * edge_margin + placements = plate.get("placements", []) + plate_basis = ( + plate.get("material"), + plate.get("grade"), + plate.get("thickness"), + ) + for placement_index, placement in enumerate(placements): + path = f"$.plate_reports[{plate_index}].placements[{placement_index}]" + values = ( + placement.get("x"), + placement.get("y"), + placement.get("w"), + placement.get("h"), + ) + if not all( + isinstance(value, (int, float)) + and not isinstance(value, bool) + and math.isfinite(value) + for value in values + ): + findings.append( + { + "code": "nonfinite_placement", + "path": path, + "message": "Placement coordinates and dimensions must be finite.", + } + ) + continue + x, y, width, height = values + if ( + width <= 0 + or height <= 0 + or x < -epsilon + or y < -epsilon + or x + width > usable_width + epsilon + or y + height > usable_height + epsilon + ): + findings.append( + { + "code": "placement_out_of_bounds", + "path": path, + "message": "Placement must remain inside the edge-margin boundary.", + } + ) + placement_basis = ( + placement.get("material"), + placement.get("grade"), + placement.get("thickness"), + ) + if placement_basis != plate_basis: + findings.append( + { + "code": "material_mismatch", + "path": path, + "message": "Placement material, grade, and thickness must match its stock.", + } + ) + + for first_index, first in enumerate(placements): + for second_index in range(first_index + 1, len(placements)): + second = placements[second_index] + values = [ + first.get(field) + for field in ("x", "y", "w", "h") + ] + [ + second.get(field) + for field in ("x", "y", "w", "h") + ] + if not all( + isinstance(value, (int, float)) + and math.isfinite(value) + for value in values + ): + continue + separated = ( + first["x"] + first["w"] + inter_part_clearance + <= second["x"] + epsilon + or second["x"] + second["w"] + inter_part_clearance + <= first["x"] + epsilon + or first["y"] + first["h"] + inter_part_clearance + <= second["y"] + epsilon + or second["y"] + second["h"] + inter_part_clearance + <= first["y"] + epsilon + ) + if not separated: + findings.append( + { + "code": "placement_overlap", + "path": ( + f"$.plate_reports[{plate_index}].placements" + f"[{first_index},{second_index}]" + ), + "message": ( + "Placements overlap or violate the required " + "kerf-plus-gap clearance." + ), + } + ) + return findings diff --git a/skills/_shared/schemas/nest-result.schema.json b/skills/_shared/schemas/nest-result.schema.json index 04e09b2..59941d6 100644 --- a/skills/_shared/schemas/nest-result.schema.json +++ b/skills/_shared/schemas/nest-result.schema.json @@ -5,23 +5,31 @@ "type": "object", "required": [ "schema_version", + "algorithm_version", + "normalized_input_hash", "estimate_input_hash", "configuration_hash", "outcome", "package_status", + "meta", + "clearance_contract", "groups", - "unplaced" + "metrics", + "cost", + "plate_reports", + "unplaced", + "verification", + "validation_findings", + "geometry_readiness", + "burn_dxf_eligible", + "burn_dxf_warnings" ], "properties": { "schema_version": { "const": "1.0.0" }, - "estimate_input_hash": { - "type": "string", - "pattern": "^[0-9a-f]{64}$" - }, - "configuration_hash": { - "type": "string", - "pattern": "^[0-9a-f]{64}$" - }, + "algorithm_version": { "type": "string", "minLength": 1 }, + "normalized_input_hash": { "$ref": "#/$defs/hash" }, + "estimate_input_hash": { "$ref": "#/$defs/hash" }, + "configuration_hash": { "$ref": "#/$defs/hash" }, "outcome": { "enum": [ "ready", @@ -31,6 +39,15 @@ "usage_or_internal_error" ] }, + "run_outcome": { + "enum": [ + "ready", + "review_required", + "blocked", + "dependency_missing", + "usage_or_internal_error" + ] + }, "package_status": { "enum": [ "draft", @@ -42,6 +59,42 @@ "rfq_ready_for_review" ] }, + "meta": { + "type": "object", + "required": [ + "job_name", + "customer", + "kerf_in", + "part_gap_in", + "edge_margin_in", + "density_lb_in3", + "unit_system" + ], + "properties": { + "job_name": { "type": "string" }, + "customer": { "type": "string" }, + "kerf_in": { "type": "number", "minimum": 0 }, + "part_gap_in": { "type": "number", "minimum": 0 }, + "edge_margin_in": { "type": "number", "minimum": 0 }, + "density_lb_in3": { "type": "number", "minimum": 0 }, + "unit_system": { "type": "string" } + }, + "additionalProperties": false + }, + "clearance_contract": { + "type": "object", + "required": [ + "edge_margin_ownership", + "inter_part_clearance_ownership", + "trailing_clearance_required_at_plate_edge" + ], + "properties": { + "edge_margin_ownership": { "const": "plate_to_part" }, + "inter_part_clearance_ownership": { "const": "kerf_plus_gap" }, + "trailing_clearance_required_at_plate_edge": { "const": false } + }, + "additionalProperties": false + }, "groups": { "type": "array", "items": { @@ -49,42 +102,269 @@ "required": ["group_id", "material", "grade", "thickness", "placements"], "properties": { "group_id": { "type": "string", "minLength": 1 }, - "material": { "type": "string", "minLength": 1 }, - "grade": { "type": "string", "minLength": 1 }, - "thickness": { "type": "number", "exclusiveMinimum": 0 }, + "material": { "type": ["string", "null"] }, + "grade": { "type": ["string", "null"] }, + "thickness": { "type": "number" }, "placements": { "type": "array", - "items": { - "type": "object", - "required": ["placement_id", "item_id", "instance_id", "stock_id"], - "properties": { - "placement_id": { "type": "string", "minLength": 1 }, - "item_id": { "type": "string", "minLength": 1 }, - "instance_id": { "type": "string", "minLength": 1 }, - "stock_id": { "type": "string", "minLength": 1 }, - "x": { "type": "number" }, - "y": { "type": "number" }, - "rotated": { "type": "boolean" } - }, - "additionalProperties": false - } + "items": { "$ref": "#/$defs/placement" } } }, "additionalProperties": false } }, + "plates_used": { "type": "integer", "minimum": 0 }, + "metrics": { + "type": "object", + "required": ["packing_utilization_pct", "net_material_yield_pct"], + "properties": { + "packing_utilization_pct": { "$ref": "#/$defs/metric" }, + "net_material_yield_pct": { "$ref": "#/$defs/metric" } + }, + "additionalProperties": false + }, + "total_plate_weight_lb": { "type": "number" }, + "total_part_weight_lb": { "type": "number" }, + "total_scrap_weight_lb": { "type": "number" }, + "cost": { + "type": "object", + "required": ["status", "total"], + "properties": { + "status": { + "enum": ["known", "not_provided", "incomplete_unplaced"] + }, + "total": { "type": ["number", "null"] } + }, + "additionalProperties": false + }, + "total_material_cost": { "type": ["number", "null"] }, + "cost_known": { "type": "boolean" }, + "total_holes": { "type": "integer", "minimum": 0 }, + "part_net_cost": { + "type": "object", + "additionalProperties": { "type": "number" } + }, + "plate_reports": { + "type": "array", + "items": { "$ref": "#/$defs/plateReport" } + }, "unplaced": { "type": "array", "items": { "type": "object", - "required": ["instance_id", "reason"], + "required": ["item_id", "label", "quantity", "size", "reason"], "properties": { - "instance_id": { "type": "string", "minLength": 1 }, - "reason": { "type": "string", "minLength": 1 } + "item_id": { "type": "string", "minLength": 1 }, + "label": { "type": "string" }, + "quantity": { "type": "integer", "minimum": 1 }, + "size": { "type": "string" }, + "reason": { + "enum": ["no_compatible_stock_fit", "stock_exhausted"] + } }, "additionalProperties": false } - } + }, + "has_irregular": { "type": "boolean" }, + "invalid_hole_warnings": { + "type": "array", + "items": { "type": "string" } + }, + "validation_findings": { + "type": "array", + "items": { "$ref": "#/$defs/finding" } + }, + "verification": { + "type": "object", + "required": ["status", "findings"], + "properties": { + "status": { "enum": ["verified", "failed"] }, + "findings": { + "type": "array", + "items": { + "type": "object", + "required": ["code", "path", "message"], + "properties": { + "code": { "type": "string" }, + "path": { "type": "string" }, + "message": { "type": "string" } + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + }, + "geometry_readiness": { + "enum": ["geometry_verified", "reference_only", "diagnostic"] + }, + "burn_dxf_eligible": { "type": "boolean" }, + "burn_dxf_warnings": { + "type": "array", + "items": { "type": "string" } + }, + "rfq_nesting": { "$ref": "#/$defs/rfqHandoff" } }, - "additionalProperties": false + "additionalProperties": false, + "$defs": { + "hash": { + "type": "string", + "pattern": "^[0-9a-f]{64}$" + }, + "metric": { + "type": "object", + "required": ["value", "approximation"], + "properties": { + "value": { "type": "number", "minimum": 0, "maximum": 100 }, + "approximation": { + "enum": ["exact", "bounding_box", "declared_area", "bounding_box_estimate"] + } + }, + "additionalProperties": false + }, + "placement": { + "type": "object", + "required": [ + "part_id", + "source_id", + "item_id", + "instance_id", + "placement_id", + "stock_id", + "label", + "x", + "y", + "w", + "h", + "rotated", + "shape", + "ow", + "oh", + "holes", + "base_area", + "holes_area", + "material", + "grade", + "thickness" + ], + "properties": { + "part_id": { "type": "string" }, + "source_id": { "type": "string" }, + "item_id": { "type": "string" }, + "instance_id": { "type": "string" }, + "placement_id": { "type": "string" }, + "stock_id": { "type": "string" }, + "label": { "type": "string" }, + "x": { "type": "number" }, + "y": { "type": "number" }, + "w": { "type": "number", "exclusiveMinimum": 0 }, + "h": { "type": "number", "exclusiveMinimum": 0 }, + "rotated": { "type": "boolean" }, + "shape": { "enum": ["rect", "irregular"] }, + "ow": { "type": "number", "exclusiveMinimum": 0 }, + "oh": { "type": "number", "exclusiveMinimum": 0 }, + "holes": { "type": "array", "items": { "type": "object" } }, + "base_area": { "type": "number" }, + "holes_area": { "type": "number" }, + "material": { "type": "string" }, + "grade": { "type": "string" }, + "thickness": { "type": "number", "exclusiveMinimum": 0 } + }, + "additionalProperties": false + }, + "remnantCandidate": { + "type": "object", + "required": ["width", "height", "area", "status"], + "properties": { + "width": { "type": "number", "exclusiveMinimum": 0 }, + "height": { "type": "number", "exclusiveMinimum": 0 }, + "area": { "type": "number", "exclusiveMinimum": 0 }, + "status": { "const": "candidate_unverified" } + }, + "additionalProperties": false + }, + "plateReport": { + "type": "object", + "required": [ + "index", + "stock", + "stock_id", + "material", + "grade", + "size", + "W", + "H", + "thickness", + "parts", + "num_parts", + "num_holes", + "packing_utilization_pct", + "net_material_yield_pct", + "plate_weight_lb", + "part_weight_lb", + "scrap_weight_lb", + "plate_cost", + "cost_basis", + "remnant_candidates", + "placements" + ], + "properties": { + "index": { "type": "integer", "minimum": 1 }, + "stock": { "type": "string" }, + "stock_id": { "type": "string" }, + "material": { "type": "string" }, + "grade": { "type": "string" }, + "size": { "type": "string" }, + "W": { "type": "number", "exclusiveMinimum": 0 }, + "H": { "type": "number", "exclusiveMinimum": 0 }, + "thickness": { "type": "number", "exclusiveMinimum": 0 }, + "parts": { + "type": "object", + "additionalProperties": { "type": "integer", "minimum": 1 } + }, + "num_parts": { "type": "integer", "minimum": 1 }, + "num_holes": { "type": "integer", "minimum": 0 }, + "packing_utilization_pct": { "$ref": "#/$defs/metric" }, + "net_material_yield_pct": { "$ref": "#/$defs/metric" }, + "plate_weight_lb": { "type": "number" }, + "part_weight_lb": { "type": "number" }, + "scrap_weight_lb": { "type": "number" }, + "plate_cost": { "type": ["number", "null"] }, + "cost_basis": { + "type": ["string", "null"], + "enum": ["per_sheet", "per_pound", null] + }, + "remnant_candidates": { + "type": "array", + "items": { "$ref": "#/$defs/remnantCandidate" } + }, + "placements": { + "type": "array", + "items": { "$ref": "#/$defs/placement" } + } + }, + "additionalProperties": false + }, + "finding": { + "type": "object", + "required": ["code", "severity", "path", "message"], + "properties": { + "code": { "type": "string" }, + "severity": { "enum": ["error", "warning"] }, + "path": { "type": "string" }, + "message": { "type": "string" } + }, + "additionalProperties": false + }, + "rfqHandoff": { + "type": "object", + "required": ["schema_version", "source_nest_result_version", "rows"], + "properties": { + "schema_version": { "const": "1.0.0" }, + "source_nest_result_version": { "const": "1.0.0" }, + "rows": { "type": "array", "items": { "type": "object" } } + }, + "additionalProperties": false + } + } } diff --git a/skills/steel-nest/SKILL.md b/skills/steel-nest/SKILL.md index 019c471..2546800 100644 --- a/skills/steel-nest/SKILL.md +++ b/skills/steel-nest/SKILL.md @@ -1,13 +1,13 @@ --- name: steel-nest -description: "Nest steel parts onto stock plates and estimate material — the plate-layout / cutting step that CAM software does. Use this skill whenever someone mentions nesting, plate layout, plate optimization, cut list, cutting plan, yield, drop/remnant, how many sheets/plates a job needs, how much plate to buy, or laying parts out on a sheet. Also trigger when a new order/SO comes in and someone asks 'how much material', 'how many plates', 'what's the yield', or 'lay these parts out' — even if they don't say the word 'nest'. Produces a nesting layout (PDF + PNG), a cut list, yield/scrap/remnant numbers, material cost, and guarded reference/burn DXFs. Rectangular parts nest exactly; irregular parts nest by bounding box." +description: "Nest steel parts onto compatible stock plates and estimate material — the plate-layout / cutting step that CAM software does. Use this skill whenever someone mentions nesting, plate layout, plate optimization, cut list, cutting plan, yield, remnant candidates, how many sheets/plates a job needs, how much plate to buy, or laying parts out on a sheet. Produces a verified result, packing utilization, net material yield, guarded layouts, and optional cost totals only when an explicit basis exists." --- # Steel Plate Nesting & Estimate ## What This Skill Does -This is an estimating-oriented version of the plate-nesting step performed by CAM software: it takes a list of parts and the plate stock on hand, packs the parts onto as few plates as possible, and tells you the yield, the drops you can reuse, the material weight, and the cost. It also draws the layout and exports guarded reference or cut-geometry DXFs. +This is an estimating-oriented version of the plate-nesting step performed by CAM software. It validates and separates material/grade/thickness groups, packs them onto compatible stock, independently verifies placements, and reports two different metrics: bounding-footprint packing utilization and net material yield. Leftover rectangles are remnant candidates, not certified reusable stock. It exists so an estimator can get a fast, repeatable material number and a reviewable layout. It does not replace CAM setup or operator verification. @@ -17,10 +17,12 @@ Be honest with the user about the boundary — it protects the shop from over-tr **Reliable:** - Rectangular / plate-blank parts nest **exactly** (MaxRects bin-packing with rotation). -- Multiple plate sizes, kerf + gap spacing, edge margin (grip/clamp keep-out). +- Multiple compatible plate sizes with independently verified bounds, non-overlap, and material grouping. +- Explicit clearance ownership: edge margin is the plate-to-part keep-out; kerf plus part gap is the minimum edge-to-edge clearance between parts. No trailing kerf/gap is required at the usable plate boundary. - **Holes and rectangular cutouts** on verified rectangular parts — subtracted from weight/cost, rotated with the part, and emitted as cut geometry only when the complete job passes the output gate. -- Yield %, scrap weight, largest reusable **drop** per plate. -- Material weight and cost (by $/lb — which also values scrap — or by $/sheet). +- Separate packing-utilization and net-material-yield percentages with approximation labels. +- Remnant candidates per plate, explicitly not certified reusable drops. +- Material weight and optional cost by one explicit basis per stock entry (`cost_per_lb` or `cost_per_sheet`, never both). - Labeled layout (PDF + one PNG per plate). - **Reference files**: `reference_nest.dxf` plus `reference_plate_N.dxf`, with sheet outlines, bounding boxes, holes, and labels for estimating review. - **Guarded cut-geometry files**: one DXF per sheet (`burn_plate_N.dxf`) containing only closed part outlines on `PROFILE` and holes/cutouts on `HOLES`, with origin at the sheet corner. They exist only when every part is rectangular, every required part fits, and every supported hole stays inside its part. @@ -39,7 +41,7 @@ Everything drives a single job JSON (schema in `references/job_template.json`; a Gather three things: 1. **Parts** — for each unique part: name, width × height (inches; use the bounding box for odd shapes), quantity, whether it's `rect` or `irregular`, and whether rotation is allowed (`rotatable: false` locks grain/rolling direction for anisotropic material or directional finish). If a rectangular part has **holes or cutouts**, add a `holes` list — each hole's `x,y` is its center from the part's lower-left corner: round = `{"dia":, "x":, "y":}`, rectangular cutout = `{"w":, "h":, "x":, "y":}`. A supported hole must remain fully inside its part or fabrication-style DXFs are suppressed. Holes are optional; skip them if you only need the layout/estimate. -2. **Stock** — plate size(s) on hand (width × height × thickness), how many sheets are available (or `unlimited` to buy as needed), and price (`cost_per_lb` preferred; `cost_per_sheet` works too). +2. **Stock** — plate size(s), explicit material, grade, and thickness, plus finite quantity or `unlimited`. A price is optional; if provided, use exactly one approved basis (`cost_per_lb` or `cost_per_sheet`) and retain its source outside this legacy JSON boundary. 3. **Cut settings** — kerf, part gap, edge margin, material density. Sensible defaults are in the template; only ask if the user hasn't implied them. Common kerf: plasma ~0.06", oxy-fuel ~0.10", laser ~0.02", waterjet ~0.03". If a spreadsheet is provided, read it with pandas first, map columns to the part fields, and confirm your interpretation before nesting. Do not silently guess quantities or dimensions. @@ -57,7 +59,9 @@ Write the job JSON, then run the engine: python3 scripts/nest.py --job --out ``` -The legacy job JSON and command arguments remain accepted. `` is now a +The legacy job JSON shape and command arguments remain accepted. Material, grade, +thickness, and imperial unit basis must now be explicit at job level or on each +part/stock entry; the engine never infers those facts from a display name. `` is a publication root rather than a flat artifact directory. Every invocation creates `/runs//` and atomically updates `/latest-run.json`; follow that pointer to find the current run. This prevents an older burn file from appearing @@ -73,9 +77,9 @@ Each run contains: - `plate_1.png`, `plate_2.png`, … — one image per plate - `burn_plate_1.dxf`, `burn_plate_2.dxf`, … — geometry-verified cut entities only; absent for review-required or blocked runs - `reference_nest.dxf` and `reference_plate_N.dxf` — explicitly reference-only layouts -- `rfq_nesting.json` — the Material / Nesting Plan / Drop Notes block for the `steel-rfq` hand-off (see below) +- `rfq_nesting.json` — versioned `1.0.0` Material / Nesting Plan / Remnant Candidate rows for the `steel-rfq` hand-off; absent on blocked runs - `report.txt` — the text report -- `result.json` — structured result (plates, placements, holes, yield, cost) for downstream use +- `result.json` — schema-versioned result with normalized input/configuration hashes, algorithm version, placements, verifier findings, both metrics, and cost status Exit meanings: @@ -91,20 +95,20 @@ capability reported by `python3 scripts/doctor.py`. ## What to Deliver -Always deliver the **PDF layout** and give the headline numbers in the message: plates used, overall yield %, total material cost, and any parts that **did not fit** (the report flags these — never hide them; it means they need more or bigger stock). Offer the reference DXF and per-plate PNGs. Offer burn DXFs only when they were emitted by the guard, and still state that CAM/operator verification is required. If burn DXFs were suppressed, report every reason. If the parts were irregular, restate the bounding-box caveat so the quote isn't over-trusted. +Always deliver the **PDF layout** and give the headline numbers in the message: plates used, packing utilization, net material yield, cost status, and aggregated quantities that did not fit. Offer the reference DXF and per-plate PNGs. Offer burn DXFs only when they were emitted by the guard, and still state that CAM/operator verification is required. -Verify before presenting: the engine already checks that no parts overlap and all fit in-bounds, but sanity-check the yield and cost against the plate count (e.g., cost = plates × sheet cost, or plate weight × $/lb). If a plate shows very low yield, mention it — it's usually the tail plate and may be worth holding parts for the next order. +Verify before presenting: require `verification.status = verified`, reconcile known cost to its recorded per-sheet or per-pound basis, and never turn a missing or incomplete cost into `$0`. ## Integration with steel-rfq -The `steel-rfq` skill has a "Nesting / Drop Reference" table. This engine writes exactly that data to `rfq_nesting.json` inside each isolated run — one row per plate material with `material`, `nesting_plan`, `drop_notes`, plus `sheets_needed` (for cross-checking the estimate's assumed sheet count) and `total_cost`. Resolve the current run through `latest-run.json` before reading it. Keep this JSON shape stable — the RFQ skill depends on those field names. +The `steel-rfq` skill has a "Nesting / Drop Reference" table. The engine writes the versioned object `{schema_version, source_nest_result_version, rows}`. Rows remain separate by stock identity, material, grade, thickness, and sheet size, even when display names match. Resolve the current run through `latest-run.json` and reject unknown handoff versions. ## Common Variations -**Mixed thickness / grade in one order** — nest each thickness as its own job (parts of different thickness can't share a plate). Run the engine once per thickness and combine the numbers in the summary. +**Mixed thickness / grade in one order** — provide the basis on each part and stock entry. The engine creates independent compatible groups and never opens an incompatible plate. -**"Just tell me how many sheets"** — still run it; the plate count is the answer, and you get the yield and cost for free. Don't estimate sheet count by dividing areas — packing loss makes that wrong. +**"Just tell me how many sheets"** — still run it; the plate count is the answer. Cost remains absent unless an explicit basis is supplied. -**Remnant/drop reuse** — add the leftover drop from a previous job as another `stock` entry (its size, `qty: 1`) so the engine tries to consume it first-ish. Note: with multiple stock sizes the engine fills them in the order listed, so list drops/offcuts first if you want them used up. +**Remnant reuse** — output rectangles are candidates only. Measure, identify, status, and approve a candidate as inventory before supplying it as a distinct stock entry in a later estimate. **Grain / directional material** — set `rotatable: false` on those parts so the nester won't spin them 90°. diff --git a/skills/steel-nest/references/example_job.json b/skills/steel-nest/references/example_job.json index 4b3b616..e272193 100644 --- a/skills/steel-nest/references/example_job.json +++ b/skills/steel-nest/references/example_job.json @@ -16,7 +16,7 @@ "name": "Synthetic Plate", "width": 60, "height": 30, - "thickness": 0.375, + "thickness": 0.5, "qty": 3 } ], diff --git a/skills/steel-nest/references/job_template.json b/skills/steel-nest/references/job_template.json index 990aab4..a3a0d13 100644 --- a/skills/steel-nest/references/job_template.json +++ b/skills/steel-nest/references/job_template.json @@ -5,7 +5,7 @@ "grade": "A36", "unit_system": "imperial", - "_comment_settings": "Cut/gap in inches. kerf = torch cut width (plasma ~0.06, oxy ~0.1, laser ~0.02). part_gap = clearance between parts on top of kerf. edge_margin = keep-out from plate edge (clamp/grip zone). density lb/in^3: A36 steel = 0.2836, aluminum = 0.098, stainless 304 = 0.289.", + "_comment_settings": "Cut/gap in inches. kerf + part_gap is the minimum edge-to-edge clearance between parts. edge_margin alone is the plate-edge keep-out; no trailing kerf/gap is required at that boundary. density lb/in^3 must match the explicit material basis.", "settings": { "kerf_in": 0.06, "part_gap_in": 0.25, @@ -14,7 +14,7 @@ "density_lb_in3": 0.2836 }, - "_comment_stock": "One entry per plate size you have on hand. qty = how many sheets available (or set \"unlimited\": true to buy as many as needed). Add either cost_per_lb or cost_per_sheet only from an approved project input; public templates intentionally contain no pricing.", + "_comment_stock": "One entry per distinct plate size/material/grade/thickness. qty = finite sheets available or set unlimited=true. Add either cost_per_lb or cost_per_sheet from an approved project input, never both; public templates intentionally contain no pricing.", "stock": [ { "name": "A36 Plate 1/2\"", diff --git a/skills/steel-nest/scripts/nest.py b/skills/steel-nest/scripts/nest.py index 4f5103d..2436c79 100644 --- a/skills/steel-nest/scripts/nest.py +++ b/skills/steel-nest/scripts/nest.py @@ -11,7 +11,8 @@ * Rectangular parts can carry HOLES (round) and rectangular CUTOUTS -- subtracted from weight/cost, rotated with the part, drawn in the layout, and emitted as cut geometry only after the complete job passes its gate. - * Yield / scrap / largest reusable drop, part weight, material cost. + * Separate packing utilization and net material yield, part/plate weight, + optional reconciled cost, and unverified remnant candidates. * Labeled layout (PNG per plate + combined PDF). * DXF outputs: - reference_nest.dxf all plates side-by-side (reference only) @@ -51,14 +52,24 @@ bootstrap_shared(__file__) from pi_steel import ( # noqa: E402 + NEST_RESULT_VERSION, RunPublisher, canonical_json_bytes, + item_id_for, outcome_exit_code, + placement_ids, sha256_bytes, ) +from pi_steel.contracts import content_hash, fallback_source_id, instance_ids # noqa: E402 +from pi_steel.geometry_verify import ( # noqa: E402 + SUPPORTED_SHAPES, + finite_positive, + hole_within_bounds, + verify_nest_placements, +) STEEL_DENSITY = 0.2836 # lb/in^3, A36 mild steel -NEST_ALGORITHM_VERSION = "maxrects-bssf-u1" +NEST_ALGORITHM_VERSION = "maxrects-bssf-u3" class StageArgumentParser(argparse.ArgumentParser): @@ -83,6 +94,11 @@ class FreeRect: @dataclass class Placement: part_id: str + source_id: str + item_id: str + instance_id: str + placement_id: str + stock_id: str label: str x: float # placed lower-left, usable (post-margin) coords y: float @@ -95,6 +111,9 @@ class Placement: holes: list = field(default_factory=list) # in original part coords base_area: float = 0.0 # gross area (bbox for rect, declared area for irregular) holes_area: float = 0.0 # total area removed by holes/cutouts + material: str = "" + grade: str = "" + thickness: float = 0.0 class MaxRectsBin: @@ -188,10 +207,6 @@ def _contains(cls, outer, inner): inner.x + inner.w <= outer.x + outer.w + cls.EPS and inner.y + inner.h <= outer.y + outer.h + cls.EPS) - def largest_free(self): - return max(self.free, key=lambda r: r.w * r.h) if self.free else None - - # -------------------------------------------------------------------------- # Holes # -------------------------------------------------------------------------- @@ -216,272 +231,824 @@ def hole_local(pc, hole): return hx, hy -def hole_containment_warnings(job): - """Return supported-hole geometry failures for the current legacy job.""" - warnings = [] - eps = 1e-9 - for part in job["parts"]: - width = float(part["width"]) - height = float(part["height"]) - for index, hole in enumerate(part.get("holes", []) or [], 1): - try: - x = float(hole["x"]) - y = float(hole["y"]) - if hole.get("dia") is not None: - radius = float(hole["dia"]) / 2.0 - contained = ( - radius > 0 - and x - radius >= -eps - and y - radius >= -eps - and x + radius <= width + eps - and y + radius <= height + eps - ) - elif hole.get("w") is not None and hole.get("h") is not None: - hole_width = float(hole["w"]) - hole_height = float(hole["h"]) - contained = ( - hole_width > 0 - and hole_height > 0 - and x - hole_width / 2.0 >= -eps - and y - hole_height / 2.0 >= -eps - and x + hole_width / 2.0 <= width + eps - and y + hole_height / 2.0 <= height + eps - ) - else: - contained = False - except (KeyError, TypeError, ValueError): - contained = False - - if not contained: - warnings.append( - f"Part '{part['name']}' hole {index} is unsupported or extends " - "outside the part; burn DXFs are suppressed." - ) +# -------------------------------------------------------------------------- +# Job runner +# -------------------------------------------------------------------------- +def _legacy_hole_to_canonical(hole): + if hole.get("dia") is not None: + return { + "kind": "round", + "diameter": hole.get("dia"), + "x": hole.get("x"), + "y": hole.get("y"), + } + return { + "kind": "rect", + "width": hole.get("w"), + "height": hole.get("h"), + "x": hole.get("x"), + "y": hole.get("y"), + } - return warnings +def _validation_finding(code, path, message, severity="error"): + return { + "code": code, + "severity": severity, + "path": path, + "message": message, + } -def burn_dxf_warnings(job, unplaced): - """Return reasons the current job is unsafe for fabrication-style DXF output.""" - warnings = [] - if any(part.get("shape", "rect") == "irregular" for part in job["parts"]): - warnings.append( - "Irregular parts use approximate bounding boxes; burn DXFs are suppressed." +def normalize_job(job): + """Normalize and validate the legacy direct-use JSON before any placement.""" + findings = [] + settings = job.get("settings", {}) + + def number(value, path, *, positive=False, nonnegative=False): + try: + parsed = float(value) + except (TypeError, ValueError): + parsed = math.nan + valid = math.isfinite(parsed) + if positive: + valid = valid and parsed > 0 + if nonnegative: + valid = valid and parsed >= 0 + if not valid: + findings.append( + _validation_finding( + "invalid_numeric_input", + path, + "Value must be finite" + + (" and greater than zero." if positive else " and non-negative."), + ) + ) + return 0.0 + return parsed + + kerf = number(settings.get("kerf_in", 0.06), "$.settings.kerf_in", nonnegative=True) + gap = number(settings.get("part_gap_in", 0.25), "$.settings.part_gap_in", nonnegative=True) + margin = number( + settings.get("edge_margin_in", 0.5), + "$.settings.edge_margin_in", + nonnegative=True, + ) + density = number( + settings.get("density_lb_in3", STEEL_DENSITY), + "$.settings.density_lb_in3", + positive=True, + ) + unit_system = job.get("unit_system") + if unit_system is None: + findings.append( + _validation_finding( + "missing_unit_basis", + "$.unit_system", + "The direct nesting engine requires an explicit imperial unit basis.", + ) + ) + elif unit_system != "imperial": + findings.append( + _validation_finding( + "unsupported_unit_system", + "$.unit_system", + "The direct nesting engine currently requires imperial inches.", + ) ) - if unplaced: - warnings.append( - f"{len(unplaced)} required part(s) did not fit; burn DXFs are suppressed." + project_id = job.get("project_id") or job.get("job_name") or "LEGACY-NEST" + revision_id = job.get("revision_id", "LEGACY-REVISION") + default_material = job.get("material") + default_grade = job.get("grade") + default_thickness = settings.get("thickness_in") + parts = [] + for index, part in enumerate(job.get("parts", [])): + path = f"$.parts[{index}]" + width = number(part.get("width"), f"{path}.width", positive=True) + height = number(part.get("height"), f"{path}.height", positive=True) + thickness = number( + part.get("thickness", default_thickness), + f"{path}.thickness", + positive=True, + ) + try: + quantity = int(part.get("qty", 1)) + quantity_valid = quantity > 0 and quantity == float(part.get("qty", 1)) + except (TypeError, ValueError): + quantity, quantity_valid = 0, False + if not quantity_valid: + findings.append( + _validation_finding( + "invalid_quantity", f"{path}.qty", "Quantity must be a positive integer." + ) + ) + shape = part.get("shape", "rect") + if shape not in SUPPORTED_SHAPES: + findings.append( + _validation_finding( + "unsupported_shape", + f"{path}.shape", + "Supported shapes are rect and irregular.", + ) + ) + material = part.get("material", default_material) + grade = part.get("grade", default_grade) + if not material or not grade or not finite_positive(thickness): + findings.append( + _validation_finding( + "missing_material_basis", + path, + "Material, grade, and thickness must be explicit before placement.", + ) + ) + holes = part.get("holes", []) or [] + holes_area = 0.0 + if finite_positive(width) and finite_positive(height): + for hole_index, hole in enumerate(holes): + if not hole_within_bounds( + _legacy_hole_to_canonical(hole), width, height + ): + findings.append( + _validation_finding( + "invalid_hole_geometry", + f"{path}.holes[{hole_index}]", + "Hole is unsupported or extends outside the part.", + ) + ) + try: + holes_area += hole_area(hole) + except (TypeError, ValueError): + pass + base_area = width * height if finite_positive(width) and finite_positive(height) else 0 + approximation = "exact" + if shape == "irregular": + if part.get("area") is None: + approximation = "bounding_box_estimate" + findings.append( + _validation_finding( + "missing_irregular_area", + f"{path}.area", + "Irregular net area is approximated by its bounding box.", + severity="warning", + ) + ) + else: + declared_area = number(part.get("area"), f"{path}.area", positive=True) + if math.isfinite(declared_area) and declared_area > base_area + 1e-9: + findings.append( + _validation_finding( + "invalid_irregular_area", + f"{path}.area", + "Declared irregular area cannot exceed its bounding box.", + ) + ) + base_area = declared_area + approximation = "declared_area" + if base_area - holes_area <= 0: + findings.append( + _validation_finding( + "nonpositive_net_area", + path, + "Part net area after holes must be greater than zero.", + ) + ) + explicit_source = part.get("source_id") + source_id = explicit_source or fallback_source_id( + revision_id, + { + key: part.get(key) + for key in ( + "name", + "material", + "grade", + "thickness", + "width", + "height", + "shape", + ) + }, + ) + item_id = part.get("item_id") or item_id_for( + project_id, revision_id, source_id + ) + parts.append( + { + "source_id": source_id, + "item_id": item_id, + "label": part.get("name", source_id), + "w": width, + "h": height, + "quantity": quantity, + "rotatable": bool(part.get("rotatable", True)), + "shape": shape, + "holes": holes, + "base_area": base_area, + "holes_area": holes_area, + "net_area_approximation": approximation, + "material": material, + "grade": grade, + "thickness": thickness, + } ) - warnings.extend(hole_containment_warnings(job)) - return warnings + stock_types = [] + for index, stock in enumerate(job.get("stock", [])): + path = f"$.stock[{index}]" + width = number(stock.get("width"), f"{path}.width", positive=True) + height = number(stock.get("height"), f"{path}.height", positive=True) + thickness = number( + stock.get("thickness", default_thickness), + f"{path}.thickness", + positive=True, + ) + material = stock.get("material", default_material) + grade = stock.get("grade", default_grade) + if not material or not grade or not finite_positive(thickness): + findings.append( + _validation_finding( + "missing_material_basis", + path, + "Stock material, grade, and thickness must be explicit.", + ) + ) + unlimited = bool(stock.get("unlimited", False)) + try: + quantity = math.inf if unlimited else int(stock.get("qty", 1)) + quantity_valid = unlimited or ( + quantity >= 0 and quantity == float(stock.get("qty", 1)) + ) + except (TypeError, ValueError): + quantity, quantity_valid = 0, False + if not quantity_valid: + findings.append( + _validation_finding( + "invalid_stock_quantity", + f"{path}.qty", + "Stock quantity must be a non-negative integer or unlimited.", + ) + ) + per_pound = stock.get("cost_per_lb") + per_sheet = stock.get("cost_per_sheet") + if per_pound is not None and per_sheet is not None: + findings.append( + _validation_finding( + "conflicting_cost_basis", + path, + "Use either cost_per_lb or cost_per_sheet for one stock entry, not both.", + ) + ) + for field, value in ( + ("cost_per_lb", per_pound), + ("cost_per_sheet", per_sheet), + ): + if value is not None: + parsed_cost = number(value, f"{path}.{field}", nonnegative=True) + if field == "cost_per_lb": + per_pound = parsed_cost + else: + per_sheet = parsed_cost + stock_id = stock.get("stock_id") or ( + "stock:" + + content_hash( + { + "name": stock.get("name", "Plate"), + "material": material, + "grade": grade, + "thickness": thickness, + "width": width, + "height": height, + } + )[:24] + ) + stock_types.append( + { + "stock_id": stock_id, + "name": stock.get("name", "Plate"), + "material": material, + "grade": grade, + "W": width, + "H": height, + "thickness": thickness, + "qty": quantity, + "cost_per_lb": per_pound, + "cost_per_sheet": per_sheet, + "used": 0, + } + ) + for collection_name, values, identity_field in ( + ("parts", parts, "item_id"), + ("stock", stock_types, "stock_id"), + ): + seen = {} + for index, value in enumerate(values): + identity = value[identity_field] + if identity in seen: + findings.append( + _validation_finding( + f"duplicate_{identity_field}", + f"$.{collection_name}[{index}].{identity_field}", + ( + f"{identity_field} duplicates row {seen[identity]}; " + "indistinguishable rows are not merged." + ), + ) + ) + else: + seen[identity] = index + parts.sort(key=lambda part: part["item_id"]) + stock_types.sort(key=lambda stock: stock["stock_id"]) + if not parts: + findings.append( + _validation_finding("missing_parts", "$.parts", "At least one part is required.") + ) + if not stock_types: + findings.append( + _validation_finding("missing_stock", "$.stock", "At least one stock entry is required.") + ) + normalized = { + "job_name": job.get("job_name", "Nesting job"), + "customer": job.get("customer", ""), + "unit_system": unit_system or "unspecified", + "settings": { + "kerf_in": kerf, + "part_gap_in": gap, + "edge_margin_in": margin, + "density_lb_in3": density, + }, + "parts": parts, + "stock": [ + { + key: ("unlimited" if key == "qty" and math.isinf(value) else value) + for key, value in stock.items() + if key != "used" + } + for stock in stock_types + ], + } + return normalized, stock_types, findings + + +def _material_key(value): + return value.get("material"), value.get("grade"), value.get("thickness") + + +def _aggregate_unplaced(units): + grouped = {} + for unit in units: + key = (unit["item_id"], unit["reason"]) + row = grouped.setdefault( + key, + { + "item_id": unit["item_id"], + "label": unit["label"], + "quantity": 0, + "size": f'{_fmt(unit["w"])} x {_fmt(unit["h"])}', + "reason": unit["reason"], + }, + ) + row["quantity"] += 1 + return sorted(grouped.values(), key=lambda row: (row["item_id"], row["reason"])) -# -------------------------------------------------------------------------- -# Job runner -# -------------------------------------------------------------------------- def run_job(job): - s = job.get("settings", {}) - kerf = float(s.get("kerf_in", 0.06)) - gap = float(s.get("part_gap_in", 0.25)) - margin = float(s.get("edge_margin_in", 0.5)) - density = float(s.get("density_lb_in3", STEEL_DENSITY)) + normalized, stock_types, validation_findings = normalize_job(job) + settings = normalized["settings"] + kerf = settings["kerf_in"] + gap = settings["part_gap_in"] + margin = settings["edge_margin_in"] + density = settings["density_lb_in3"] spacing = kerf + gap + normalized_hash = sha256_bytes(canonical_json_bytes(normalized)) + blockers = [ + finding + for finding in validation_findings + if finding["severity"] == "error" + ] + if blockers: + return _summarize( + normalized, + [], + [], + density, + margin, + kerf, + gap, + validation_findings, + normalized_hash, + ) - # expand parts, largest first units = [] - for p in job["parts"]: - holes = p.get("holes", []) or [] - h_area = sum(hole_area(h) for h in holes) - base = float(p["width"]) * float(p["height"]) - if p.get("shape") == "irregular" and p.get("area"): - base = float(p["area"]) - for _ in range(int(p.get("qty", 1))): - units.append({ - "part_id": p["name"], "label": p["name"], - "w": float(p["width"]), "h": float(p["height"]), - "rotatable": bool(p.get("rotatable", True)), - "shape": p.get("shape", "rect"), - "holes": holes, "base_area": base, "holes_area": h_area, - }) - units.sort(key=lambda u: u["w"] * u["h"], reverse=True) - - stock_types = [] - for st in job["stock"]: - stock_types.append({ - "name": st.get("name", "Plate"), - "W": float(st["width"]), "H": float(st["height"]), - "thickness": float(st.get("thickness", s.get("thickness_in", 0.5))), - "qty": math.inf if st.get("unlimited") else int(st.get("qty", 1)), - "cost_per_lb": st.get("cost_per_lb"), - "cost_per_sheet": st.get("cost_per_sheet"), - "used": 0, - }) - + for part in normalized["parts"]: + item_instances = instance_ids(part["item_id"], part["quantity"]) + item_placements = placement_ids(part["item_id"], part["quantity"]) + for index in range(part["quantity"]): + units.append( + { + **part, + "part_id": part["item_id"], + "instance_id": item_instances[index], + "placement_id": item_placements[index], + } + ) + units.sort(key=lambda unit: (-unit["w"] * unit["h"], unit["instance_id"])) plates = [] - def open_plate(fit=None): - for stype in stock_types: - if stype["used"] >= stype["qty"]: + def compatible(stock, unit): + return _material_key(stock) == _material_key(unit) + + def can_fit(stock, unit): + usable_width = stock["W"] - 2 * margin + usable_height = stock["H"] - 2 * margin + direct = unit["w"] <= usable_width + 1e-9 and unit["h"] <= usable_height + 1e-9 + rotated = ( + unit["rotatable"] + and unit["h"] <= usable_width + 1e-9 + and unit["w"] <= usable_height + 1e-9 + ) + return compatible(stock, unit) and (direct or rotated) + + def open_plate(unit): + for stock in stock_types: + if stock["used"] >= stock["qty"] or not can_fit(stock, unit): continue - uw, uh = stype["W"] - 2 * margin, stype["H"] - 2 * margin - if fit is not None: - fw, fh = fit["w"] + spacing, fit["h"] + spacing - ok = (fw <= uw + 1e-9 and fh <= uh + 1e-9) - if fit["rotatable"]: - ok = ok or (fh <= uw + 1e-9 and fw <= uh + 1e-9) - if not ok: - continue - stype["used"] += 1 - plates.append({"stock": stype, "bin": MaxRectsBin(uw, uh), "placements": []}) - return plates[-1] + usable_width = stock["W"] - 2 * margin + usable_height = stock["H"] - 2 * margin + stock["used"] += 1 + plate = { + "stock": stock, + "bin": MaxRectsBin(usable_width + spacing, usable_height + spacing), + "placements": [], + } + plates.append(plate) + return plate return None - def fits_any(u): - fw, fh = u["w"] + spacing, u["h"] + spacing - for stype in stock_types: - uw, uh = stype["W"] - 2 * margin, stype["H"] - 2 * margin - if (fw <= uw + 1e-9 and fh <= uh + 1e-9) or \ - (u["rotatable"] and fh <= uw + 1e-9 and fw <= uh + 1e-9): - return True - return False - - def commit(pl, u, res): - x, y, rw, rh, rot = res - pl["placements"].append(Placement( - u["part_id"], u["label"], x, y, rw - spacing, rh - spacing, rot, - u["shape"], u["w"], u["h"], u["holes"], u["base_area"], u["holes_area"])) - - unplaced = [] - for u in units: - if not fits_any(u): - unplaced.append(u) + def commit(plate, unit, placement): + x, y, packed_width, packed_height, rotated = placement + plate["placements"].append( + Placement( + part_id=unit["part_id"], + source_id=unit["source_id"], + item_id=unit["item_id"], + instance_id=unit["instance_id"], + placement_id=unit["placement_id"], + stock_id=plate["stock"]["stock_id"], + label=unit["label"], + x=x, + y=y, + w=packed_width - spacing, + h=packed_height - spacing, + rotated=rotated, + shape=unit["shape"], + ow=unit["w"], + oh=unit["h"], + holes=unit["holes"], + base_area=unit["base_area"], + holes_area=unit["holes_area"], + material=unit["material"], + grade=unit["grade"], + thickness=unit["thickness"], + ) + ) + + unplaced_units = [] + for unit in units: + compatible_stock = [ + stock for stock in stock_types if can_fit(stock, unit) + ] + if not compatible_stock: + unplaced_units.append({**unit, "reason": "no_compatible_stock_fit"}) continue - fw, fh = u["w"] + spacing, u["h"] + spacing + packed_width, packed_height = unit["w"] + spacing, unit["h"] + spacing placed = False - for pl in plates: - res = pl["bin"].insert(fw, fh, u["rotatable"]) - if res: - commit(pl, u, res) + for plate in plates: + if not compatible(plate["stock"], unit): + continue + placement = plate["bin"].insert( + packed_width, packed_height, unit["rotatable"] + ) + if placement: + commit(plate, unit, placement) placed = True break if not placed: - newpl = open_plate(fit=u) - if newpl is not None: - res = newpl["bin"].insert(fw, fh, u["rotatable"]) - if res: - commit(newpl, u, res) + plate = open_plate(unit) + if plate is not None: + placement = plate["bin"].insert( + packed_width, packed_height, unit["rotatable"] + ) + if placement: + commit(plate, unit, placement) placed = True if not placed: - unplaced.append(u) - - used_plates = [pl for pl in plates if pl["placements"]] - for i, pl in enumerate(used_plates, 1): - pl["index"] = i - - return _summarize(job, used_plates, unplaced, density, margin, kerf, gap) + unplaced_units.append({**unit, "reason": "stock_exhausted"}) + + used_plates = [plate for plate in plates if plate["placements"]] + for index, plate in enumerate(used_plates, 1): + plate["index"] = index + return _summarize( + normalized, + used_plates, + _aggregate_unplaced(unplaced_units), + density, + margin, + kerf, + gap, + validation_findings, + normalized_hash, + ) -def _summarize(job, used_plates, unplaced, density, margin, kerf, gap): +def _metric(value, approximation): + return {"value": round(value, 1), "approximation": approximation} + + +def _remnant_candidates(plate, margin, spacing): + stock = plate["stock"] + usable_width = stock["W"] - 2 * margin + usable_height = stock["H"] - 2 * margin + candidates = [] + for free in plate["bin"].free: + width = max(0.0, min(free.w, usable_width - free.x)) + height = max(0.0, min(free.h, usable_height - free.y)) + if width > spacing and height > spacing: + candidates.append( + { + "width": round(width, 2), + "height": round(height, 2), + "area": round(width * height, 2), + "status": "candidate_unverified", + } + ) + return sorted( + candidates, key=lambda candidate: candidate["area"], reverse=True + )[:3] + + +def _summarize( + normalized, + used_plates, + unplaced, + density, + margin, + kerf, + gap, + validation_findings, + normalized_hash, +): plate_reports = [] - tot_plate_area = tot_part_area_bbox = 0.0 - tot_plate_wt = tot_part_wt = 0.0 - tot_cost = 0.0 - cost_known = True - part_net = {} - - for pl in used_plates: - stype = pl["stock"] - W, H, t = stype["W"], stype["H"], stype["thickness"] - plate_area = W * H - plate_wt = plate_area * t * density - - p_area_bbox = p_wt = 0.0 - n_holes = 0 + total_plate_area = total_packing_area = total_net_area = 0.0 + total_plate_weight = total_part_weight = 0.0 + total_cost = 0.0 + all_used_costs_known = bool(used_plates) + part_net_cost = {} + has_irregular = any(part["shape"] == "irregular" for part in normalized["parts"]) + net_approximations = { + part["net_area_approximation"] for part in normalized["parts"] + } + + for plate in used_plates: + stock = plate["stock"] + width, height, thickness = stock["W"], stock["H"], stock["thickness"] + plate_area = width * height + plate_weight = plate_area * thickness * density + packing_area = net_area_total = part_weight = 0.0 + holes = 0 parts_on = {} - for pc in pl["placements"]: - bbox = pc.w * pc.h - net_area = max(0.0, pc.base_area - pc.holes_area) - p_area_bbox += bbox - wt = net_area * t * density - p_wt += wt - n_holes += len(pc.holes) - parts_on[pc.label] = parts_on.get(pc.label, 0) + 1 - if stype.get("cost_per_lb") is not None: - part_net[pc.label] = part_net.get(pc.label, 0.0) + wt * float(stype["cost_per_lb"]) - - if stype.get("cost_per_sheet") is not None: - plate_cost = float(stype["cost_per_sheet"]) - elif stype.get("cost_per_lb") is not None: - plate_cost = plate_wt * float(stype["cost_per_lb"]) + for placement in plate["placements"]: + packing_area += placement.w * placement.h + net_area = max(0.0, placement.base_area - placement.holes_area) + net_area_total += net_area + weight = net_area * thickness * density + part_weight += weight + holes += len(placement.holes) + parts_on[placement.label] = parts_on.get(placement.label, 0) + 1 + if stock["cost_per_lb"] is not None: + part_net_cost[placement.label] = ( + part_net_cost.get(placement.label, 0.0) + + weight * stock["cost_per_lb"] + ) + + if stock["cost_per_sheet"] is not None: + plate_cost = stock["cost_per_sheet"] + cost_basis = "per_sheet" + elif stock["cost_per_lb"] is not None: + plate_cost = plate_weight * stock["cost_per_lb"] + cost_basis = "per_pound" else: plate_cost = None - cost_known = False - - lf = pl["bin"].largest_free() - remnant = (round(lf.w, 2), round(lf.h, 2)) if lf else None - - plate_reports.append({ - "index": pl["index"], "stock": stype["name"], - "size": f"{_fmt(W)} x {_fmt(H)} x {_fmt(t)}", - "W": W, "H": H, "thickness": t, - "parts": parts_on, "num_parts": len(pl["placements"]), "num_holes": n_holes, - "yield_pct": round(100 * p_area_bbox / plate_area, 1), - "plate_weight_lb": round(plate_wt, 1), - "part_weight_lb": round(p_wt, 1), - "scrap_weight_lb": round(plate_wt - p_wt, 1), - "plate_cost": None if plate_cost is None else round(plate_cost, 2), - "largest_remnant": remnant, - "placements": [vars(pc) for pc in pl["placements"]], - }) - - tot_plate_area += plate_area - tot_part_area_bbox += p_area_bbox - tot_plate_wt += plate_wt - tot_part_wt += p_wt + cost_basis = None + all_used_costs_known = False if plate_cost is not None: - tot_cost += plate_cost - - overall_yield = round(100 * tot_part_area_bbox / tot_plate_area, 1) if tot_plate_area else 0.0 + total_cost += plate_cost + packing_approximation = "bounding_box" if any( + placement.shape == "irregular" for placement in plate["placements"] + ) else "exact" + plate_net_statuses = { + next( + part["net_area_approximation"] + for part in normalized["parts"] + if part["item_id"] == placement.item_id + ) + for placement in plate["placements"] + } + net_approximation = ( + "exact" + if plate_net_statuses == {"exact"} + else ( + "bounding_box_estimate" + if "bounding_box_estimate" in plate_net_statuses + else "declared_area" + ) + ) + report = { + "index": plate["index"], + "stock": stock["name"], + "stock_id": stock["stock_id"], + "material": stock["material"], + "grade": stock["grade"], + "size": f"{_fmt(width)} x {_fmt(height)} x {_fmt(thickness)}", + "W": width, + "H": height, + "thickness": thickness, + "parts": parts_on, + "num_parts": len(plate["placements"]), + "num_holes": holes, + "packing_utilization_pct": _metric( + 100 * packing_area / plate_area, packing_approximation + ), + "net_material_yield_pct": _metric( + 100 * net_area_total / plate_area, net_approximation + ), + "plate_weight_lb": round(plate_weight, 1), + "part_weight_lb": round(part_weight, 1), + "scrap_weight_lb": round(plate_weight - part_weight, 1), + "plate_cost": None if plate_cost is None else round(plate_cost, 2), + "cost_basis": cost_basis, + "remnant_candidates": _remnant_candidates(plate, margin, kerf + gap), + "placements": [vars(placement) for placement in plate["placements"]], + } + plate_reports.append(report) + total_plate_area += plate_area + total_packing_area += packing_area + total_net_area += net_area_total + total_plate_weight += plate_weight + total_part_weight += part_weight - invalid_holes = hole_containment_warnings(job) - burn_warnings = burn_dxf_warnings(job, unplaced) - has_irregular = any(p.get("shape") == "irregular" for p in job["parts"]) - if unplaced or invalid_holes: + if unplaced: + cost_status, cost_total = "incomplete_unplaced", None + elif all_used_costs_known: + cost_status, cost_total = "known", round(total_cost, 2) + else: + cost_status, cost_total = "not_provided", None + packing_status = "bounding_box" if has_irregular else "exact" + net_status = ( + "bounding_box_estimate" + if "bounding_box_estimate" in net_approximations + else ("declared_area" if "declared_area" in net_approximations else "exact") + ) + metrics = { + "packing_utilization_pct": _metric( + 100 * total_packing_area / total_plate_area if total_plate_area else 0, + packing_status, + ), + "net_material_yield_pct": _metric( + 100 * total_net_area / total_plate_area if total_plate_area else 0, + net_status, + ), + } + verification_findings = verify_nest_placements( + plate_reports, + edge_margin=margin, + inter_part_clearance=kerf + gap, + ) + invalid_holes = [ + finding["message"] + for finding in validation_findings + if finding["code"] == "invalid_hole_geometry" + ] + blockers = [ + finding + for finding in validation_findings + if finding["severity"] == "error" + ] + burn_warnings = [] + if has_irregular: + burn_warnings.append( + "Irregular parts use approximate bounding boxes; burn DXFs are suppressed." + ) + if unplaced: + burn_warnings.append( + f"{sum(row['quantity'] for row in unplaced)} required part(s) did not fit; " + "burn DXFs are suppressed." + ) + burn_warnings.extend(invalid_holes) + burn_warnings.extend(finding["message"] for finding in verification_findings) + burn_warnings.extend( + finding["message"] for finding in blockers if finding["code"] != "invalid_hole_geometry" + ) + if blockers or unplaced or verification_findings: geometry_readiness = "diagnostic" elif has_irregular: geometry_readiness = "reference_only" else: geometry_readiness = "geometry_verified" - res = { + + groups = [] + group_keys = sorted( + {_material_key(part) for part in normalized["parts"]}, key=repr + ) + for material, grade, thickness in group_keys: + placements = [ + placement + for plate in plate_reports + for placement in plate["placements"] + if ( + placement["material"], + placement["grade"], + placement["thickness"], + ) + == (material, grade, thickness) + ] + groups.append( + { + "group_id": "nest-group:" + + content_hash( + { + "material": material, + "grade": grade, + "thickness": thickness, + } + )[:20], + "material": material, + "grade": grade, + "thickness": thickness, + "placements": placements, + } + ) + configuration_hash = sha256_bytes( + canonical_json_bytes( + { + "algorithm_version": NEST_ALGORITHM_VERSION, + "settings": normalized["settings"], + "clearance_contract": "edge-margin-and-inter-part-v1", + } + ) + ) + result = { + "schema_version": NEST_RESULT_VERSION, + "algorithm_version": NEST_ALGORITHM_VERSION, + "normalized_input_hash": normalized_hash, + "estimate_input_hash": normalized_hash, + "configuration_hash": configuration_hash, + "outcome": "blocked", + "package_status": "draft", "meta": { - "job_name": job.get("job_name", "Nesting job"), - "customer": job.get("customer", ""), - "kerf_in": kerf, "part_gap_in": gap, "edge_margin_in": margin, + "job_name": normalized["job_name"], + "customer": normalized["customer"], + "kerf_in": kerf, + "part_gap_in": gap, + "edge_margin_in": margin, "density_lb_in3": density, + "unit_system": normalized["unit_system"], + }, + "clearance_contract": { + "edge_margin_ownership": "plate_to_part", + "inter_part_clearance_ownership": "kerf_plus_gap", + "trailing_clearance_required_at_plate_edge": False, + }, + "groups": groups, + "plates_used": len(plate_reports), + "metrics": metrics, + "total_plate_weight_lb": round(total_plate_weight, 1), + "total_part_weight_lb": round(total_part_weight, 1), + "total_scrap_weight_lb": round(total_plate_weight - total_part_weight, 1), + "cost": {"status": cost_status, "total": cost_total}, + "total_material_cost": cost_total, + "cost_known": cost_status == "known", + "total_holes": sum(report["num_holes"] for report in plate_reports), + "part_net_cost": { + key: round(value, 2) for key, value in part_net_cost.items() }, - "plates_used": len(used_plates), - "overall_yield_pct": overall_yield, - "total_plate_weight_lb": round(tot_plate_wt, 1), - "total_part_weight_lb": round(tot_part_wt, 1), - "total_scrap_weight_lb": round(tot_plate_wt - tot_part_wt, 1), - "total_material_cost": None if not cost_known else round(tot_cost, 2), - "cost_known": cost_known, - "total_holes": sum(pr["num_holes"] for pr in plate_reports), - "part_net_cost": {k: round(v, 2) for k, v in part_net.items()}, "plate_reports": plate_reports, - "unplaced": [{"label": u["label"], "size": f'{_fmt(u["w"])} x {_fmt(u["h"])}'} - for u in unplaced], + "unplaced": unplaced, "has_irregular": has_irregular, "invalid_hole_warnings": invalid_holes, + "validation_findings": validation_findings, + "verification": { + "status": "verified" if not verification_findings else "failed", + "findings": verification_findings, + }, "geometry_readiness": geometry_readiness, "burn_dxf_eligible": not burn_warnings, "burn_dxf_warnings": burn_warnings, } - res["rfq_nesting"] = rfq_nesting_block(res) - return res + result["rfq_nesting"] = rfq_nesting_block(result) + outcome, package_status, _ = stage_decision(result) + result["outcome"] = outcome + result["package_status"] = package_status + return result def _fmt(v): @@ -492,34 +1059,74 @@ def _fmt(v): # RFQ hand-off block (feeds steel-rfq "Nesting / Drop Reference" table) # -------------------------------------------------------------------------- def rfq_nesting_block(res): - """Group plates by stock material -> Material | Nesting Plan | Drop Notes rows.""" + """Build the versioned nest-to-RFQ handoff without merging stock variants.""" from collections import defaultdict groups = defaultdict(list) for pr in res["plate_reports"]: - groups[pr["stock"]].append(pr) + groups[ + ( + pr["stock_id"], + pr["material"], + pr["grade"], + pr["thickness"], + pr["W"], + pr["H"], + ) + ].append(pr) blocks = [] - for name, prs in groups.items(): + for key, prs in sorted(groups.items(), key=lambda item: repr(item[0])): + stock_id, material, grade, thickness, width, height = key sheets = len(prs) - W, H = prs[0]["W"], prs[0]["H"] - pa = sum(pr["yield_pct"] / 100 * pr["W"] * pr["H"] for pr in prs) - ta = sum(pr["W"] * pr["H"] for pr in prs) - yld = round(100 * pa / ta, 1) if ta else 0.0 + packing_area = sum( + pr["packing_utilization_pct"]["value"] / 100 * pr["W"] * pr["H"] + for pr in prs + ) + total_area = sum(pr["W"] * pr["H"] for pr in prs) + utilization = round(100 * packing_area / total_area, 1) if total_area else 0.0 parts = sum(pr["num_parts"] for pr in prs) - drops = sorted([pr["largest_remnant"] for pr in prs if pr["largest_remnant"]], - key=lambda d: d[0] * d[1], reverse=True)[:3] - drop_txt = "; ".join(f"{_fmt(d[0])}x{_fmt(d[1])}" for d in drops) or "minimal" - cost = sum((pr["plate_cost"] or 0) for pr in prs) + candidates = sorted( + [ + candidate + for pr in prs + for candidate in pr["remnant_candidates"] + ], + key=lambda candidate: candidate["area"], + reverse=True, + )[:3] + candidate_text = ( + "; ".join( + f"{_fmt(candidate['width'])}x{_fmt(candidate['height'])}" + for candidate in candidates + ) + or "none" + ) + costs_known = all(pr["plate_cost"] is not None for pr in prs) + cost = sum(pr["plate_cost"] for pr in prs if pr["plate_cost"] is not None) blocks.append({ - "material": name, + "stock_id": stock_id, + "stock_name": prs[0]["stock"], + "material": material, + "grade": grade, + "thickness": thickness, "sheets_needed": sheets, - "sheet_size": f"{_fmt(W)}x{_fmt(H)}", - "yield_pct": yld, - "nesting_plan": f"{sheets} x {_fmt(W)}x{_fmt(H)} sheet(s) - {yld}% yield, {parts} parts", - "drop_notes": f"Largest reusable drops: {drop_txt} in", - "total_cost": round(cost, 2) if res["cost_known"] else None, + "sheet_size": f"{_fmt(width)}x{_fmt(height)}", + "packing_utilization_pct": utilization, + "nesting_plan": ( + f"{sheets} x {_fmt(width)}x{_fmt(height)} sheet(s) - " + f"{utilization}% packing utilization, {parts} parts" + ), + "drop_notes": ( + f"Remnant candidates (not certified reusable): {candidate_text} in" + ), + "remnant_candidates": candidates, + "total_cost": round(cost, 2) if costs_known and not res["unplaced"] else None, }) - return blocks + return { + "schema_version": "1.0.0", + "source_nest_result_version": NEST_RESULT_VERSION, + "rows": blocks, + } # -------------------------------------------------------------------------- @@ -535,7 +1142,16 @@ def render_text(res): f"Edge margin {m['edge_margin_in']}\" | {m['density_lb_in3']} lb/in^3") L.append("") L.append(f" Plates used ............ {res['plates_used']}") - L.append(f" Overall yield .......... {res['overall_yield_pct']}%") + packing = res["metrics"]["packing_utilization_pct"] + net_yield = res["metrics"]["net_material_yield_pct"] + L.append( + f" Packing utilization ... {packing['value']}% " + f"({packing['approximation']})" + ) + L.append( + f" Net material yield .... {net_yield['value']}% " + f"({net_yield['approximation']})" + ) L.append(f" Holes / cutouts ........ {res['total_holes']}") L.append(f" Total plate weight ..... {res['total_plate_weight_lb']} lb") L.append(f" Net part weight ........ {res['total_part_weight_lb']} lb (holes removed)") @@ -551,14 +1167,20 @@ def render_text(res): L.append(f" PLATE {pr['index']} — {pr['stock']} ({pr['size']} in)") L.append(f" Parts: {parts_str}") L.append(f" Holes: {pr['num_holes']}") - L.append(f" Yield: {pr['yield_pct']}% " - f"Part wt {pr['part_weight_lb']} lb / plate {pr['plate_weight_lb']} lb " - f"Scrap {pr['scrap_weight_lb']} lb") + L.append( + f" Packing: {pr['packing_utilization_pct']['value']}% " + f"Net yield {pr['net_material_yield_pct']['value']}% " + f"Part wt {pr['part_weight_lb']} lb / plate " + f"{pr['plate_weight_lb']} lb Scrap {pr['scrap_weight_lb']} lb" + ) if pr["plate_cost"] is not None: L.append(f" Cost: ${pr['plate_cost']:,.2f}") - if pr["largest_remnant"]: - rw, rh = pr["largest_remnant"] - L.append(f" Biggest usable drop: {_fmt(rw)} x {_fmt(rh)} in") + if pr["remnant_candidates"]: + candidate = pr["remnant_candidates"][0] + L.append( + " Largest remnant candidate (not certified): " + f"{_fmt(candidate['width'])} x {_fmt(candidate['height'])} in" + ) L.append("") if res["part_net_cost"]: L.append(" Net material cost per part (metal in part, before markup):") @@ -569,7 +1191,10 @@ def render_text(res): L.append(" " + "!" * 60) L.append(" DID NOT FIT (need more/larger stock):") for u in res["unplaced"]: - L.append(f" - {u['label']} ({u['size']} in)") + L.append( + f" - {u['label']} x{u['quantity']} " + f"({u['size']} in; {u['reason']})" + ) L.append("") if res["has_irregular"]: L.append(" NOTE: irregular parts are nested by BOUNDING BOX. Supply a") @@ -632,7 +1257,8 @@ def render_layout(res, outdir): ax.set_ylim(-1, H + 1) ax.set_aspect("equal") ax.set_title(f"PLATE {pr['index']} — {pr['stock']} ({pr['size']} in) " - f"Yield {pr['yield_pct']}% Holes {pr['num_holes']}", + f"Packing {pr['packing_utilization_pct']['value']}% " + f"Holes {pr['num_holes']}", fontsize=12, fontweight="bold") ax.set_xlabel("inches") ax.grid(True, lw=0.3, color="#eee") @@ -774,22 +1400,24 @@ def render_burn_dxfs(res, outdir): def stage_decision(res, geometry_verified_only=False): - """Map the temporary legacy nest result onto the shared stage contract.""" - findings = [] + """Map the independently verified result onto the shared stage contract.""" + findings = list(res.get("validation_findings", [])) + findings.extend( + {**finding, "severity": "error"} + for finding in res.get("verification", {}).get("findings", []) + ) if res["unplaced"]: findings.append({ "code": "UNPLACED_PARTS", "severity": "error", - "message": f"{len(res['unplaced'])} required part(s) remain unplaced.", - }) - for warning in res["invalid_hole_warnings"]: - findings.append({ - "code": "INVALID_HOLE_GEOMETRY", - "severity": "error", - "message": warning, + "path": "$.unplaced", + "message": ( + f"{sum(row['quantity'] for row in res['unplaced'])} " + "required part(s) remain unplaced." + ), }) - if findings: + if any(finding["severity"] == "error" for finding in findings): outcome = "blocked" package_status = "nested_partial" if res["unplaced"] else "draft" elif res["has_irregular"]: @@ -798,6 +1426,7 @@ def stage_decision(res, geometry_verified_only=False): findings.append({ "code": "APPROXIMATE_PROFILE_GEOMETRY", "severity": "warning", + "path": "$.groups", "message": ( "Irregular profiles are represented by bounding boxes in " "reference-only artifacts." @@ -811,6 +1440,7 @@ def stage_decision(res, geometry_verified_only=False): findings.append({ "code": "GEOMETRY_VERIFIED_REQUIRED", "severity": "error", + "path": "$.geometry_readiness", "message": "The requested geometry-verified output is unavailable.", }) @@ -850,15 +1480,20 @@ def publish_nest_run(job, args): outcome, package_status, findings = stage_decision( result, args.geometry_verified_only ) + result["outcome"] = outcome result["run_outcome"] = outcome result["package_status"] = package_status report = render_text(result) configuration = { "algorithm_version": NEST_ALGORITHM_VERSION, + "engine_configuration_hash": result["configuration_hash"], "geometry_verified_only": args.geometry_verified_only, "render": not args.no_render, } + publication_configuration_hash = sha256_bytes( + canonical_json_bytes(configuration) + ) approximations = [] if result["has_irregular"]: approximations.append({ @@ -880,11 +1515,12 @@ def publish_nest_run(job, args): stage="steel-nest", run_outcome=outcome, package_status=package_status, - input_hash=sha256_bytes(canonical_json_bytes(job)), - configuration_hash=sha256_bytes(canonical_json_bytes(configuration)), + input_hash=result["normalized_input_hash"], + configuration_hash=publication_configuration_hash, schema_versions={ "run_manifest": "1.0.0", - "nest_result": "legacy-u1", + "nest_result": NEST_RESULT_VERSION, + "rfq_nesting": "1.0.0", }, tool_versions={ "pi_steel": package_version(), @@ -904,9 +1540,10 @@ def publish_nest_run(job, args): media_type="text/plain", ) publisher.write_json("result.json", result, readiness="diagnostic") - publisher.write_json( - "rfq_nesting.json", result["rfq_nesting"], readiness="diagnostic" - ) + if outcome in {"ready", "review_required"}: + publisher.write_json( + "rfq_nesting.json", result["rfq_nesting"], readiness="diagnostic" + ) if not args.no_render and not missing_dependencies: publisher.register_artifact( diff --git a/tests/fixtures/nest/README.md b/tests/fixtures/nest/README.md index 78a9bc0..42f7b24 100644 --- a/tests/fixtures/nest/README.md +++ b/tests/fixtures/nest/README.md @@ -7,3 +7,7 @@ customer, vendor, drawing, takeoff, inventory, or commercial data. `irregular-reference-only.json` uses invented geometry to prove that an unresolved irregular outline remains reference-only and cannot produce a burn DXF. It establishes no compatibility claim for any CAM product or version. + +`grouped-engine.json` uses invented plate and part dimensions to exercise +material/grade/thickness segregation, finite and unlimited stock, and +same-display-name stock identities. It contains no pricing. diff --git a/tests/fixtures/nest/grouped-engine.json b/tests/fixtures/nest/grouped-engine.json new file mode 100644 index 0000000..80b4d5e --- /dev/null +++ b/tests/fixtures/nest/grouped-engine.json @@ -0,0 +1,57 @@ +{ + "job_name": "SYNTHETIC-GROUPED-NEST", + "customer": "Example Customer", + "unit_system": "imperial", + "settings": { + "kerf_in": 0.05, + "part_gap_in": 0.2, + "edge_margin_in": 0.5, + "density_lb_in3": 0.2836 + }, + "stock": [ + { + "stock_id": "SYNTHETIC-STOCK-A36-0500", + "name": "Synthetic Plate", + "material": "carbon_steel", + "grade": "A36", + "width": 24, + "height": 12, + "thickness": 0.5, + "qty": 1 + }, + { + "stock_id": "SYNTHETIC-STOCK-A572-0375", + "name": "Synthetic Plate", + "material": "carbon_steel", + "grade": "A572", + "width": 24, + "height": 12, + "thickness": 0.375, + "unlimited": true + } + ], + "parts": [ + { + "source_id": "SYNTHETIC-SRC-A36", + "name": "SYNTHETIC-A36-PART", + "material": "carbon_steel", + "grade": "A36", + "thickness": 0.5, + "width": 8, + "height": 4, + "qty": 2, + "shape": "rect" + }, + { + "source_id": "SYNTHETIC-SRC-A572", + "name": "SYNTHETIC-A572-PART", + "material": "carbon_steel", + "grade": "A572", + "thickness": 0.375, + "width": 7, + "height": 5, + "qty": 3, + "shape": "rect" + } + ] +} diff --git a/tests/fixtures/nest/irregular-reference-only.json b/tests/fixtures/nest/irregular-reference-only.json index e632b06..697d0c0 100644 --- a/tests/fixtures/nest/irregular-reference-only.json +++ b/tests/fixtures/nest/irregular-reference-only.json @@ -1,11 +1,15 @@ { "job_name": "SYNTHETIC-IRREGULAR-REFERENCE", "customer": "Example Customer", + "material": "carbon_steel", + "grade": "A36", + "unit_system": "imperial", "settings": { "kerf_in": 0.06, "part_gap_in": 0.25, "edge_margin_in": 0.5, - "density_lb_in3": 0.2836 + "density_lb_in3": 0.2836, + "thickness_in": 0.5 }, "stock": [ { diff --git a/tests/test_nest_burn_guard.py b/tests/test_nest_burn_guard.py index 2730c0d..5ebd5b1 100644 --- a/tests/test_nest_burn_guard.py +++ b/tests/test_nest_burn_guard.py @@ -18,11 +18,15 @@ def job_with(part, *, stock_qty=1): return { "job_name": "SYNTHETIC-BURN-GUARD", + "material": "carbon_steel", + "grade": "A36", + "unit_system": "imperial", "settings": { "kerf_in": 0.06, "part_gap_in": 0.25, "edge_margin_in": 0.5, "density_lb_in3": 0.2836, + "thickness_in": 0.5, }, "stock": [ { diff --git a/tests/test_nest_cli_contract.py b/tests/test_nest_cli_contract.py new file mode 100644 index 0000000..6459d3e --- /dev/null +++ b/tests/test_nest_cli_contract.py @@ -0,0 +1,121 @@ +import json +import os +import subprocess +import sys +from pathlib import Path + +import jsonschema + + +ROOT = Path(__file__).resolve().parents[1] +NEST_SCRIPT = ROOT / "skills" / "steel-nest" / "scripts" / "nest.py" +NEST_SCHEMA = ROOT / "skills" / "_shared" / "schemas" / "nest-result.schema.json" + + +def run_cli(tmp_path, job, run_id): + job_path = tmp_path / f"{run_id}.json" + job_path.write_text(json.dumps(job), encoding="utf-8") + output = tmp_path / "published" + environment = os.environ.copy() + environment.pop("PYTHONPATH", None) + completed = subprocess.run( + [ + sys.executable, + NEST_SCRIPT, + "--job", + job_path, + "--out", + output, + "--run-id", + run_id, + "--no-render", + ], + cwd=tmp_path, + env=environment, + capture_output=True, + text=True, + ) + pointer = json.loads((output / "latest-run.json").read_text()) + run_path = output / pointer["run_directory"] + return completed, run_path + + +def valid_job(): + return { + "job_name": "SYNTHETIC-CLI-CONTRACT", + "material": "carbon_steel", + "grade": "A36", + "unit_system": "imperial", + "settings": { + "kerf_in": 0.05, + "part_gap_in": 0.2, + "edge_margin_in": 0.5, + "thickness_in": 0.5, + }, + "stock": [ + { + "stock_id": "SYNTHETIC-STOCK-CLI", + "name": "Synthetic Plate", + "width": 20, + "height": 10, + "thickness": 0.5, + "qty": 1, + } + ], + "parts": [ + { + "source_id": "SYNTHETIC-SRC-CLI", + "name": "SYNTHETIC-CLI-PART", + "width": 6, + "height": 4, + "qty": 1, + "shape": "rect", + } + ], + } + + +def test_ready_cli_publishes_versioned_result_handoff_and_manifest(tmp_path): + completed, run_path = run_cli( + tmp_path, valid_job(), "SYNTHETIC-CLI-READY" + ) + assert completed.returncode == 0, completed.stdout + completed.stderr + result = json.loads((run_path / "result.json").read_text()) + schema = json.loads(NEST_SCHEMA.read_text()) + jsonschema.Draft202012Validator(schema).validate(result) + assert result["algorithm_version"] == nest_algorithm_version() + assert len(result["normalized_input_hash"]) == 64 + handoff = json.loads((run_path / "rfq_nesting.json").read_text()) + assert handoff["schema_version"] == "1.0.0" + assert handoff["rows"][0]["grade"] == "A36" + manifest = json.loads((run_path / "run-manifest.json").read_text()) + assert manifest["schema_versions"]["nest_result"] == "1.0.0" + + +def nest_algorithm_version(): + text = NEST_SCRIPT.read_text() + marker = 'NEST_ALGORITHM_VERSION = "' + return text.split(marker, 1)[1].split('"', 1)[0] + + +def test_review_required_and_blocked_outcomes_obey_exit_contract(tmp_path): + review = valid_job() + review["parts"][0].update(shape="irregular", area=12) + review_completed, review_path = run_cli( + tmp_path, review, "SYNTHETIC-CLI-REVIEW" + ) + assert review_completed.returncode == 2 + assert json.loads((review_path / "run-manifest.json").read_text())[ + "run_outcome" + ] == "review_required" + + blocked = valid_job() + blocked["parts"][0]["width"] = -1 + blocked_completed, blocked_path = run_cli( + tmp_path, blocked, "SYNTHETIC-CLI-BLOCKED" + ) + assert blocked_completed.returncode == 3 + assert json.loads((blocked_path / "run-manifest.json").read_text())[ + "run_outcome" + ] == "blocked" + assert not (blocked_path / "rfq_nesting.json").exists() diff --git a/tests/test_nest_engine.py b/tests/test_nest_engine.py new file mode 100644 index 0000000..daf46be --- /dev/null +++ b/tests/test_nest_engine.py @@ -0,0 +1,197 @@ +import importlib.util +import json +import sys +from copy import deepcopy +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +NEST_SCRIPT = ROOT / "skills" / "steel-nest" / "scripts" / "nest.py" +SPEC = importlib.util.spec_from_file_location("pi_steel_nest_engine", NEST_SCRIPT) +nest = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = nest +SPEC.loader.exec_module(nest) + + +def base_job(): + return { + "job_name": "SYNTHETIC-ENGINE", + "material": "carbon_steel", + "grade": "A36", + "unit_system": "imperial", + "settings": { + "kerf_in": 0.05, + "part_gap_in": 0.2, + "edge_margin_in": 0.5, + "thickness_in": 0.5, + "density_lb_in3": 0.2836, + }, + "stock": [ + { + "stock_id": "SYNTHETIC-STOCK-1", + "name": "Synthetic Plate", + "width": 11, + "height": 7, + "thickness": 0.5, + "qty": 1, + } + ], + "parts": [ + { + "source_id": "SYNTHETIC-SRC-1", + "name": "SYNTHETIC-PART-1", + "width": 10, + "height": 6, + "qty": 1, + "shape": "rect", + } + ], + } + + +def test_characterizes_exact_fit_without_trailing_clearance_rejection(): + result = nest.run_job(base_job()) + placement = result["plate_reports"][0]["placements"][0] + assert result["unplaced"] == [] + assert (placement["x"], placement["y"], placement["w"], placement["h"]) == ( + 0, + 0, + 10, + 6, + ) + assert result["clearance_contract"] == { + "edge_margin_ownership": "plate_to_part", + "inter_part_clearance_ownership": "kerf_plus_gap", + "trailing_clearance_required_at_plate_edge": False, + } + + +def test_rotated_fit_and_nonrotatable_oversize_quantity_aggregation(): + job = base_job() + job["stock"][0].update(width=7, height=11) + rotated = nest.run_job(job) + assert rotated["plate_reports"][0]["placements"][0]["rotated"] is True + + job["parts"][0]["rotatable"] = False + job["parts"][0]["qty"] = 3 + blocked = nest.run_job(job) + assert blocked["plate_reports"] == [] + assert blocked["unplaced"] == [ + { + "item_id": blocked["unplaced"][0]["item_id"], + "label": "SYNTHETIC-PART-1", + "quantity": 3, + "size": "10 x 6", + "reason": "no_compatible_stock_fit", + } + ] + + +def test_material_groups_and_same_name_stock_variants_remain_distinct(): + job = json.loads( + ( + ROOT / "tests" / "fixtures" / "nest" / "grouped-engine.json" + ).read_text() + ) + result = nest.run_job(job) + assert len(result["groups"]) == 2 + assert { + (group["material"], group["grade"], group["thickness"]) + for group in result["groups"] + } == { + ("carbon_steel", "A36", 0.5), + ("carbon_steel", "A572", 0.375), + } + assert len({plate["stock_id"] for plate in result["plate_reports"]}) == 2 + assert len(result["rfq_nesting"]["rows"]) == 2 + + reordered = deepcopy(job) + reordered["parts"].reverse() + reordered["stock"].reverse() + reordered_result = nest.run_job(reordered) + assert reordered_result["normalized_input_hash"] == result["normalized_input_hash"] + assert [ + placement["placement_id"] + for plate in reordered_result["plate_reports"] + for placement in plate["placements"] + ] == [ + placement["placement_id"] + for plate in result["plate_reports"] + for placement in plate["placements"] + ] + + +def test_finite_exhaustion_blocks_and_unlimited_stock_opens_only_when_compatible(): + job = base_job() + job["parts"][0]["qty"] = 2 + job["stock"].append( + { + "stock_id": "SYNTHETIC-STOCK-INCOMPATIBLE", + "name": "Synthetic Plate", + "material": "carbon_steel", + "grade": "A572", + "width": 11, + "height": 7, + "thickness": 0.5, + "unlimited": True, + } + ) + exhausted = nest.run_job(job) + assert exhausted["unplaced"][0]["reason"] == "stock_exhausted" + assert exhausted["unplaced"][0]["quantity"] == 1 + + job["stock"][1]["grade"] = "A36" + complete = nest.run_job(job) + assert complete["unplaced"] == [] + assert complete["plates_used"] == 2 + + +def test_cost_bases_reconcile_and_conflicts_block_before_placement(): + per_sheet = base_job() + per_sheet["stock"][0].update(cost_per_sheet=100, synthetic_cost=True) + sheet_result = nest.run_job(per_sheet) + assert sheet_result["cost"]["status"] == "known" + assert sheet_result["cost"]["total"] == 100 + + per_pound = base_job() + per_pound["stock"][0].update(cost_per_lb=1, synthetic_cost=True) + pound_result = nest.run_job(per_pound) + expected = 11 * 7 * 0.5 * 0.2836 + assert pound_result["cost"]["total"] == pytest.approx(expected, abs=0.01) + + conflict = deepcopy(per_sheet) + conflict["stock"][0]["cost_per_lb"] = 1 + blocked = nest.run_job(conflict) + assert blocked["plate_reports"] == [] + assert any( + finding["code"] == "conflicting_cost_basis" + for finding in blocked["validation_findings"] + ) + + +def test_unplaced_without_open_plate_never_reports_known_zero_cost(): + job = base_job() + job["stock"][0]["qty"] = 0 + result = nest.run_job(job) + assert result["unplaced"][0]["quantity"] == 1 + assert result["cost"]["status"] == "incomplete_unplaced" + assert result["cost"]["total"] is None + + +def test_packing_and_net_yield_are_separate_and_labeled(): + job = base_job() + job["stock"][0].update(width=20, height=20) + job["parts"][0].update( + shape="irregular", + area=24, + holes=[{"dia": 1, "x": 5, "y": 3}], + ) + result = nest.run_job(job) + metrics = result["metrics"] + assert metrics["packing_utilization_pct"]["value"] == 15.0 + assert metrics["net_material_yield_pct"]["value"] < 6.0 + assert metrics["packing_utilization_pct"]["approximation"] == "bounding_box" + assert metrics["net_material_yield_pct"]["approximation"] == "declared_area" + assert "overall_yield_pct" not in result diff --git a/tests/test_nest_invariants.py b/tests/test_nest_invariants.py new file mode 100644 index 0000000..f9ebad0 --- /dev/null +++ b/tests/test_nest_invariants.py @@ -0,0 +1,89 @@ +import importlib.util +import sys +from copy import deepcopy +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SHARED = ROOT / "skills" / "_shared" +sys.path.insert(0, str(SHARED)) +NEST_SCRIPT = ROOT / "skills" / "steel-nest" / "scripts" / "nest.py" +SPEC = importlib.util.spec_from_file_location("pi_steel_nest_invariants", NEST_SCRIPT) +nest = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = nest +SPEC.loader.exec_module(nest) + +from pi_steel.geometry_verify import verify_nest_placements + + +def two_part_job(): + return { + "job_name": "SYNTHETIC-INVARIANTS", + "material": "carbon_steel", + "grade": "A36", + "unit_system": "imperial", + "settings": { + "kerf_in": 0.05, + "part_gap_in": 0.2, + "edge_margin_in": 0.5, + "thickness_in": 0.5, + }, + "stock": [ + { + "stock_id": "SYNTHETIC-STOCK-INVARIANT", + "name": "Synthetic Plate", + "width": 24, + "height": 12, + "thickness": 0.5, + "qty": 1, + } + ], + "parts": [ + { + "source_id": "SYNTHETIC-SRC-A", + "name": "SYNTHETIC-A", + "width": 8, + "height": 4, + "qty": 1, + "shape": "rect", + }, + { + "source_id": "SYNTHETIC-SRC-B", + "name": "SYNTHETIC-B", + "width": 7, + "height": 3, + "qty": 1, + "shape": "rect", + }, + ], + } + + +def test_generated_placements_pass_independent_pure_verifier(): + result = nest.run_job(two_part_job()) + assert result["verification"] == {"status": "verified", "findings": []} + assert verify_nest_placements( + result["plate_reports"], + edge_margin=result["meta"]["edge_margin_in"], + inter_part_clearance=( + result["meta"]["kerf_in"] + result["meta"]["part_gap_in"] + ), + ) == [] + + +def test_verifier_rejects_overlap_bounds_clearance_and_material_mismatch(): + result = nest.run_job(two_part_job()) + plates = deepcopy(result["plate_reports"]) + first, second = plates[0]["placements"] + second["x"], second["y"] = first["x"], first["y"] + second["grade"] = "A572" + first["x"] = -1 + codes = { + finding["code"] + for finding in verify_nest_placements( + plates, + edge_margin=result["meta"]["edge_margin_in"], + inter_part_clearance=0.25, + ) + } + assert {"placement_out_of_bounds", "placement_overlap", "material_mismatch"} <= codes diff --git a/tests/test_nest_outputs.py b/tests/test_nest_outputs.py index b300ee8..1ea4a32 100644 --- a/tests/test_nest_outputs.py +++ b/tests/test_nest_outputs.py @@ -21,11 +21,15 @@ def rectangular_job(): return { "job_name": "SYNTHETIC-DXF-CHARACTERIZATION", + "material": "carbon_steel", + "grade": "A36", + "unit_system": "imperial", "settings": { "kerf_in": 0.06, "part_gap_in": 0.25, "edge_margin_in": 0.5, "density_lb_in3": 0.2836, + "thickness_in": 0.5, }, "stock": [ { @@ -240,7 +244,8 @@ def test_invalid_hole_is_blocked_with_only_safe_reference_and_diagnostics(tmp_pa assert completed.returncode == 3, completed.stdout + completed.stderr run_path = latest_run(output_root) assert not list(run_path.glob("burn_plate_*.dxf")) - assert (run_path / "reference_nest.dxf").exists() + assert not (run_path / "reference_nest.dxf").exists() + assert (run_path / "qa-report.json").exists() manifest = load_json(run_path / "run-manifest.json") assert manifest["run_outcome"] == "blocked" assert "geometry_verified" not in { From 164758c037452dd1453fafbfed30677e18a83c4c Mon Sep 17 00:00:00 2001 From: Victor Garcia Date: Tue, 28 Jul 2026 12:41:17 -0600 Subject: [PATCH 08/15] feat(rfq): compile deterministic draft workbooks --- .../_shared/schemas/nest-result.schema.json | 10 +- skills/steel-nest/scripts/nest.py | 2 + skills/steel-rfq/SKILL.md | 264 ++-- .../assets/company-profile.example.json | 12 +- skills/steel-rfq/references/rfq-input.md | 57 + skills/steel-rfq/scripts/generate-rfq.py | 1060 +++++++++++++++++ skills/steel-rfq/scripts/recalc.py | 13 +- tests/fixtures/rfq/README.md | 9 + tests/fixtures/rfq/estimate-package.json | 134 +++ tests/fixtures/rfq/nest-handoff.json | 51 + tests/fixtures/rfq/synthetic-profile.json | 13 + tests/golden/rfq/semantic-workbook.json | 20 + tests/test_installed_scripts.py | 2 + tests/test_nest_engine.py | 2 + tests/test_rfq_generator.py | 228 ++++ tests/test_rfq_workbook_contract.py | 108 ++ 16 files changed, 1807 insertions(+), 178 deletions(-) create mode 100644 skills/steel-rfq/references/rfq-input.md create mode 100755 skills/steel-rfq/scripts/generate-rfq.py create mode 100644 tests/fixtures/rfq/README.md create mode 100644 tests/fixtures/rfq/estimate-package.json create mode 100644 tests/fixtures/rfq/nest-handoff.json create mode 100644 tests/fixtures/rfq/synthetic-profile.json create mode 100644 tests/golden/rfq/semantic-workbook.json create mode 100644 tests/test_rfq_generator.py create mode 100644 tests/test_rfq_workbook_contract.py diff --git a/skills/_shared/schemas/nest-result.schema.json b/skills/_shared/schemas/nest-result.schema.json index 59941d6..b1b10b0 100644 --- a/skills/_shared/schemas/nest-result.schema.json +++ b/skills/_shared/schemas/nest-result.schema.json @@ -358,10 +358,18 @@ }, "rfqHandoff": { "type": "object", - "required": ["schema_version", "source_nest_result_version", "rows"], + "required": [ + "schema_version", + "source_nest_result_version", + "geometry_readiness", + "rows" + ], "properties": { "schema_version": { "const": "1.0.0" }, "source_nest_result_version": { "const": "1.0.0" }, + "geometry_readiness": { + "enum": ["geometry_verified", "reference_only", "diagnostic"] + }, "rows": { "type": "array", "items": { "type": "object" } } }, "additionalProperties": false diff --git a/skills/steel-nest/scripts/nest.py b/skills/steel-nest/scripts/nest.py index 2436c79..1731b20 100644 --- a/skills/steel-nest/scripts/nest.py +++ b/skills/steel-nest/scripts/nest.py @@ -1121,10 +1121,12 @@ def rfq_nesting_block(res): ), "remnant_candidates": candidates, "total_cost": round(cost, 2) if costs_known and not res["unplaced"] else None, + "geometry_readiness": res["geometry_readiness"], }) return { "schema_version": "1.0.0", "source_nest_result_version": NEST_RESULT_VERSION, + "geometry_readiness": res["geometry_readiness"], "rows": blocks, } diff --git a/skills/steel-rfq/SKILL.md b/skills/steel-rfq/SKILL.md index c26e028..b8d78af 100644 --- a/skills/steel-rfq/SKILL.md +++ b/skills/steel-rfq/SKILL.md @@ -1,175 +1,97 @@ --- name: steel-rfq -description: "Generate standardized Request for Quotation (RFQ) spreadsheets from steel estimate takeoff files. Use this skill whenever someone mentions RFQ, request for quote, vendor quote, material quote, sending a material list to vendors, quoting steel, or getting pricing from a supplier. Also trigger when someone uploads a steel estimate or takeoff spreadsheet and wants to send it out for pricing. Even if they just say 'I need to send this to my vendor' or 'get me prices on this material' — that's an RFQ." +description: "Compile a deterministic draft steel RFQ workbook from a validated canonical estimate package or an exact-header legacy workbook. Use for material quote lists, RFQ spreadsheets, or versioned nesting references. Produces artifacts only; it never sends, awards, or authorizes purchasing." --- -# Steel RFQ Generator - -## What This Skill Does - -This skill takes a steel estimate/takeoff spreadsheet (.xlsx) and produces a clean, standardized RFQ spreadsheet that can be sent directly to steel vendors. The output is always an Excel file with a consistent layout so every vendor gets the same professional format regardless of who creates it. - -The RFQ format was designed around how steel vendors actually work — materials grouped by type (W-shapes, plates, flat bar) so vendors can quickly identify what they have in stock, quote pricing, and flag what they don't carry. - -## Company Profile (required setup) - -The RFQ carries the requesting company's identity. Load it from an explicit -`PI_STEEL_CONFIG` path, the ignored project-local -`.pi-steel/company-profile.json`, or the platform user-config directory. Never save -runtime company data inside this installed skill or the public repository. If no -profile is available, ask the user for their company name, city/state, and approved -terms before generating. - -Profile fields: -- `company_name` — appears in the header and terms & conditions -- `city_state` — appears in the header (e.g., "Example City, ST") -- `payment_terms` — required approved text; no shipped default -- `quote_validity_days` — optional approved value; no shipped default -- `logo` — optional filename in `assets/` to embed top-left - -Never invent company details. If the profile is incomplete, ask. - -## Input - -Always a steel estimate spreadsheet (.xlsx). These files typically have: -- A "Steel Takeoff" sheet (or similar) with line items for structural members, plates, and connection hardware -- Item numbers (A-01, B-01, C-01, etc.), categories, descriptions, sizes, quantities, stock lengths, and weights -- Items marked "BY OTHERS" (typically purlins) that are NOT part of the fabricator's scope - -The first step is always to read and understand the takeoff data before generating the RFQ. - -## Reading the Estimate - -1. Use pandas to read all sheets: `pd.ExcelFile(path)` then inspect sheet names -2. Find the takeoff sheet (look for "Takeoff", "Steel Takeoff", "Material List", "BOM", or the sheet with item-level steel data) -3. Read the full sheet with `header=None` to capture all rows including section headers -4. Identify the data structure: - - Section headers (e.g., "A. MAIN BUILDING — W-SHAPES") - - Column headers (Item, Category, Description, Size, Qty, Length, Weight, Stock Purchase, Purchase Wt, Notes) - - Line items with actual material data - - Items with "BY OTHERS" in the description — these get **excluded** - - Subtotal and total rows - -## RFQ Output Format - -The output is always a single .xlsx file with this exact structure: - -### Header Block (Rows 1–3) -- **Row 1**: "REQUEST FOR QUOTATION — Structural Steel" — dark blue background (#1F3864), white bold Arial 14pt, merged across all columns -- **Row 2**: "[Project Name] | [Company Name] — [City, ST]" — same dark blue, white bold Arial 11pt -- **Row 3**: "Date Issued: [today] | Response Requested By: _______________ | Project Location: [location]" — same dark blue, white Arial 10pt - -### Vendor Info Block (Rows 4–6) -Fillable fields for the vendor: -- Row 5: Company Name (merged A–B, input in C–E yellow) | Contact Name (merged F–G, input in H–J yellow) -- Row 6: Phone/Email (merged A–B, input in C–E yellow) | Quote Valid Until (merged F–G, input in H–J yellow) - -### Instructions Row (Row 7) -Single merged row with italic gray text explaining how to fill out the yellow columns. Text: -> "Instructions: Please fill in the YELLOW columns (Unit Price, Total Price, Availability, Lead Time, Alternate Size, Notes). Mark Availability as 'In Stock', 'Lead Time', or 'Unavailable'. If suggesting an alternate size, list it and adjust pricing accordingly." - -### Column Headers (Row 8) -14 columns with medium blue background (#2E75B6), white bold Arial 10pt: - -| Column | Header | Width | Purpose | -|--------|--------|-------|---------| -| A | Item | 7 | Item number from estimate (A-01, C-02, E-01, etc.) | -| B | Category | 13 | Columns, Girders, Plate, Flat Bar, etc. | -| C | Description | 38 | Full description including which structure it belongs to | -| D | Size / Designation | 22 | AISC shape or plate dimensions | -| E | Grade | 12 | A992 Gr.50, A572 Gr.50, A36, etc. | -| F | Qty | 6 | Number of pieces or sticks needed | -| G | Stock Length / Size | 20 | How it's being purchased (e.g., "3 sticks × 30′") | -| H | Est. Purchase Wt (lbs) | 20 | Weight from the estimate | -| I | Unit Price ($) | 14 | **VENDOR FILLS** — yellow background | -| J | Total Price ($) | 14 | **VENDOR FILLS** — yellow background | -| K | Availability | 14 | **VENDOR FILLS** — yellow background | -| L | Lead Time (days) | 14 | **VENDOR FILLS** — yellow background | -| M | Alternate Size | 18 | **VENDOR FILLS** — yellow background | -| N | Notes | 30 | **VENDOR FILLS** — yellow background | - -Vendor columns (I–N) get a darker gold header (#BF8F00) to distinguish them from the material data columns. - -### Material Data Rows - -**Grouping**: Always group by material type, not by structure. The three groups are: -1. **W-SHAPES — [Grade]** (columns and girders from all structures combined) -2. **PLATE STOCK — [Grade]** (all plate material) -3. **FLAT BAR STOCK — [Grade]** (all flat bar material) - -Each group gets a section header row: merged across all columns, medium blue background, white bold text. - -**Data rows**: -- Alternate row shading: light blue (#D6E4F0) and white -- Vendor columns (I–N) always have yellow background (#FFF2CC) regardless of row -- Weight column (H) formatted as `#,##0` -- Price columns (I–J) formatted as `$#,##0.00` -- All cells have thin borders - -**Filtering rules**: -- EXCLUDE any item where the description contains "BY OTHERS" (purlins, etc.) -- EXCLUDE any item where Qty = 0 -- If the estimate has a "Stock Purchase" section (Section E or similar) with consolidated purchase items for plates/flat bar, use those instead of the individual connection plate items. The stock purchase items represent what's actually being ordered (full sheets, full bars), which is what the vendor needs to quote. -- Keep individual W-shape items because each shape/length combination matters for vendor stock - -### Totals Row -- "TOTAL PURCHASE WEIGHT / PRICE" label merged A–G, right-aligned bold -- Column H: `=SUM(H[first]:H[last])` formula for total weight, green background (#E2EFDA) -- Column J: `=SUM(J[first]:J[last])` formula for total price, green background - -### Nesting / Drop Reference Table -Below the totals (skip a row), add a reference section: -- Header: "NESTING / DROP REFERENCE (For Fabricator Use)" — dark blue bold text -- Three columns: Material | Nesting Plan | Drop Notes -- Column headers with medium blue background -- One row per material showing the nesting layout and expected drop from the estimate -- This helps cross-check vendor stock lengths against the cutting plan - -### Terms & Conditions -Below the nesting table (skip a row), add: -- Header: "TERMS & CONDITIONS" — dark blue bold text -- Load terms only from the user's approved company profile or an explicitly supplied - project template. -- Do not ship, infer, or invent default commercial terms. Missing approved terms keep - the workbook in draft/review-required status. -- Public examples must use obvious placeholders and must not contain a real company's - payment, delivery, cancellation, inspection, substitution, or purchasing policy. - -### Branding / Logo -If the company profile names a logo file and it exists in the skill's `assets/` directory, insert it in cell A1 area (top-left) and adjust the header text to not overlap. Otherwise use the text header as described above. - -## Print Setup -- Orientation: Landscape -- Fit to width: 1 page -- Fit to height: 0 (auto) -- Print title rows: Row 8 (column headers repeat on each page) - -## File Naming -Output file: `[ProjectName]_RFQ_Material_List.xlsx` -- Extract project name from the estimate file (look in "Project Info" sheet or the first rows of the takeoff) -- Replace spaces with underscores -- Example: `Synthetic_Demo_RFQ_Material_List.xlsx` - -## Step-by-Step Workflow - -1. Load the company profile (or collect it from the user — see Company Profile above) -2. Read the uploaded estimate file with pandas to understand its structure -3. Identify the takeoff sheet and parse all line items -4. Filter out "BY OTHERS" items and zero-quantity items -5. Group remaining items by material type (W-shapes → Plates → Flat Bar) -6. If there's a stock purchase section, use those for plates/flat bar instead of individual pieces -7. Build the RFQ spreadsheet using openpyxl following the format above -8. Add formulas for totals and verify the SUM ranges cover exactly the data rows -9. Add nesting/drop reference from the estimate notes — or, if the `steel-nest` skill has been run for this job, read its `rfq_nesting.json` output straight into the table -10. Add only user-approved terms from the selected profile or project template -11. Insert logo if configured -12. Save the file, then run `python3 scripts/recalc.py ` so formula values are computed (openpyxl writes formulas but never calculates them) -13. Verify no formula errors and present the file to the user - -## Common Variations - -**Multiple structures in one project**: Combine all materials into one RFQ, but note which structure each item belongs to in the Description column (e.g., "W12×65 Columns (Bldg A)"). - -**Plate stock with nesting**: When plates are purchased as full sheets and then cut, show the full sheet as the line item (that's what the vendor ships) and put the cutting plan in the nesting reference table. - -**Mixed grades**: Group by material type first, then note the grade in each row. If a project has A992, A572, and A36, they all appear in the appropriate material type section with grade clearly marked. +# Steel RFQ Compiler + +## Boundary + +This skill creates a reviewable draft workbook. It does not contact vendors, +send RFQs, select a quote, approve substitutions, award work, or authorize a +purchase. Every workbook and manifest records `DRAFT — NOT SENT OR AWARDED`. + +## Required inputs + +- Canonical estimate-package JSON `1.0.0`, or the conservative exact-header + legacy XLSX contract in `references/rfq-input.md`. +- Explicit issue date. +- A runtime company profile with company name, city/state, and an approved, + hash-bound terms template. +- Optional versioned nesting handoff `1.0.0`. + +Resolve the company profile from `PI_STEEL_CONFIG`, project-local ignored +`.pi-steel/company-profile.json`, then the platform user-config directory. +Never write runtime identity, terms, or logos into this installed skill. + +The shipped `assets/company-profile.example.json` is deliberately unapproved. +Copy it to a runtime config location, supply reviewed terms and approval +metadata, recompute the exact UTF-8 SHA-256, then change status to `approved`. + +## Deterministic scope and purchasing rules + +- Canonical `intent` controls scope. Description text cannot exclude an item. +- `allowance`, `exclusion`, and `by_others` never become vendor quantities. +- Zero quantity in the legacy adapter maps to excluded scope. +- Consolidated purchased stock replaces fabricated pieces only through explicit + `dimensions.replaces_item_ids`. +- Lines remain traceable by source and item ID. +- Groups and nesting references remain separate by material, grade, thickness, + size, and stock identity. +- Missing identity, approved terms, supported versions, or valid input blocks + workbook generation. + +## Workbook contract + +The compiler owns: + +- `RFQ Draft` and hidden `RFQ Metadata` sheets. +- Dark-blue title block, fillable vendor block, fixed A:N headers, grouped + purchase lines, yellow vendor-response cells, formulas, borders, widths, and + alternating row fills. +- Exact total formulas covering the deterministic material range. +- Versioned nesting/remnant reference rows with visible reference-only labels. +- Approved terms content and approval lineage. +- Landscape print setup, one-page width, repeated row-8 headers, and stable + project-derived filename. + +A missing optional logo falls back to the text header and does not invent +branding. + +## Run + +```bash +python3 scripts/generate-rfq.py \ + --input \ + --nest \ + --issued-date \ + --project-location "Example City, ST" \ + --out +``` + +Each invocation publishes an isolated run with `run-manifest.json`, +`qa-report.json`, the draft `.xlsx` when gates pass, and +`workbook-semantic.json`. + +Exit codes: + +- `0`: draft RFQ ready for human review. +- `2`: draft generated with review-required warnings or reference-only nesting. +- `3`: blocked; diagnostics only, no workbook. +- `1`: usage or internal input error. + +## Formula calculation status + +The compiler always requests full recalculation on open. If LibreOffice is +available and succeeds, QA says `baked_via_libreoffice`. Otherwise QA says +`deferred_recalculate_on_open`; it never claims cached formula values were +computed. + +## Verification before delivery + +- Confirm the manifest outcome and artifact allow-list. +- Confirm workbook metadata remains draft-only. +- Confirm every purchase line is traceable and typed in scope. +- Confirm totals formulas span the intended material rows. +- Confirm nesting rows preserve material/grade/thickness/size boundaries. +- Report deferred formula caching and reference-only nesting visibly. diff --git a/skills/steel-rfq/assets/company-profile.example.json b/skills/steel-rfq/assets/company-profile.example.json index 55403dd..134ee31 100644 --- a/skills/steel-rfq/assets/company-profile.example.json +++ b/skills/steel-rfq/assets/company-profile.example.json @@ -1,7 +1,13 @@ { "company_name": "Example Fabricator", "city_state": "Example City, ST", - "payment_terms": "ENTER APPROVED PAYMENT TERMS", - "quote_validity_days": null, - "logo": null + "logo": null, + "terms_template": { + "template_id": "EXAMPLE-TERMS-UNAPPROVED", + "content": "ENTER SYNTHETICALLY APPROVED TERMS TEMPLATE", + "content_hash": "607418ebf6466ee2f5f45b7ea63603baba276b5f95860cd8a079dfa1d652fe6b", + "approver": "ENTER APPROVER", + "approval_date": null, + "status": "draft" + } } diff --git a/skills/steel-rfq/references/rfq-input.md b/skills/steel-rfq/references/rfq-input.md new file mode 100644 index 0000000..161a0b2 --- /dev/null +++ b/skills/steel-rfq/references/rfq-input.md @@ -0,0 +1,57 @@ +# RFQ compiler input contract + +The deterministic compiler accepts either: + +1. A canonical estimate package at schema version `1.0.0`. +2. A legacy `.xlsx` workbook with a sheet named exactly `Steel Takeoff` and + these exact row-1 headers: + + `Source_ID`, `Item_ID`, `Scope`, `Description`, `Material`, `Grade`, + `Thickness`, `Size`, `Qty`, `Purchase Weight`. + +The legacy adapter is intentionally narrow. `Scope` must be exactly `IN SCOPE`, +`BY OTHERS`, or `EXCLUDED`; descriptions never control scope. Missing stable +IDs, renamed columns, inferred section headers, and ambiguous quantities are +rejected for explicit mapping outside the compiler. + +Canonical filtering uses `intent`. Allowances, exclusions, and by-others items +never become vendor lines. A purchased-stock or hardware item replaces +fabricated items only when its `dimensions.replaces_item_ids` explicitly names +those canonical item IDs. + +The optional nesting input is the versioned `rfq_nesting.json` object: + +```json +{ + "schema_version": "1.0.0", + "source_nest_result_version": "1.0.0", + "rows": [] +} +``` + +Unknown versions are blocked. Rows stay separate by stock identity, material, +grade, thickness, and sheet size. + +The company profile is resolved in this order: + +1. File named by `PI_STEEL_CONFIG`. +2. Ignored project file `.pi-steel/company-profile.json` beside the input. +3. Platform user configuration at `pi-steel/company-profile.json`. + +Runtime profiles must not be saved under the installed package. The profile +requires company identity and an approved `terms_template` whose SHA-256 +matches its exact UTF-8 content. Editing terms invalidates approval. A missing +optional logo uses the text header. + +The command requires an explicit issue date: + +```bash +python3 scripts/generate-rfq.py \ + --input estimate-package.json \ + --nest rfq_nesting.json \ + --issued-date 2026-07-28 \ + --out published/ +``` + +Every workbook is marked `DRAFT — NOT SENT OR AWARDED`. The compiler contains +no send, award, vendor-selection, or purchase-authorization action. diff --git a/skills/steel-rfq/scripts/generate-rfq.py b/skills/steel-rfq/scripts/generate-rfq.py new file mode 100755 index 0000000..2f1c69d --- /dev/null +++ b/skills/steel-rfq/scripts/generate-rfq.py @@ -0,0 +1,1060 @@ +#!/usr/bin/env python3 +"""Compile a deterministic, draft-only steel RFQ workbook.""" + +from __future__ import annotations + +import argparse +import hashlib +import importlib.util +import json +import os +import re +import sys +from datetime import date +from pathlib import Path +from typing import Any + +import openpyxl +from openpyxl import Workbook +from openpyxl.drawing.image import Image +from openpyxl.styles import Alignment, Border, Font, PatternFill, Side + + +SHARED_ROOT = Path(__file__).resolve().parents[2] / "_shared" +if str(SHARED_ROOT) not in sys.path: + sys.path.insert(0, str(SHARED_ROOT)) +from bootstrap import bootstrap_shared # noqa: E402 + +bootstrap_shared(__file__) +from pi_steel import RunPublisher, canonical_json_bytes, outcome_exit_code, sha256_bytes # noqa: E402 +from pi_steel.contracts import ESTIMATE_PACKAGE_VERSION, estimate_input_hash # noqa: E402 +from pi_steel.validation import validate_estimate_package # noqa: E402 + +RECALC_PATH = Path(__file__).with_name("recalc.py") +RECALC_SPEC = importlib.util.spec_from_file_location("pi_steel_rfq_recalc", RECALC_PATH) +recalc = importlib.util.module_from_spec(RECALC_SPEC) +RECALC_SPEC.loader.exec_module(recalc) + + +RFQ_COMPILER_VERSION = "1.0.0" +NEST_HANDOFF_VERSION = "1.0.0" +HEADERS = [ + "Item", + "Category", + "Description", + "Size / Designation", + "Grade", + "Qty", + "Stock Length / Size", + "Est. Purchase Wt (lbs)", + "Unit Price ($)", + "Total Price ($)", + "Availability", + "Lead Time (days)", + "Alternate Size", + "Notes", +] +COLUMN_WIDTHS = { + "A": 7, + "B": 13, + "C": 38, + "D": 22, + "E": 12, + "F": 6, + "G": 20, + "H": 20, + "I": 14, + "J": 14, + "K": 14, + "L": 14, + "M": 18, + "N": 30, +} +CATEGORY_ORDER = { + "W-SHAPES": 10, + "LONG PRODUCTS": 20, + "PLATE STOCK": 30, + "PLATE PARTS": 40, + "FLAT BAR STOCK": 50, + "HARDWARE": 60, + "OTHER": 90, +} + + +class RfqInputError(ValueError): + """Raised when an input cannot be mapped without guessing.""" + + +class StageArgumentParser(argparse.ArgumentParser): + def error(self, message): + self.print_usage(sys.stderr) + self.exit(1, f"{self.prog}: error: {message}\n") + + +def _valid_date(value: Any) -> bool: + try: + return date.fromisoformat(str(value)).isoformat() == str(value) + except ValueError: + return False + + +def _fmt(value: Any) -> str: + if value is None: + return "" + if isinstance(value, float): + return f"{value:g}" + return str(value) + + +def _category_for(item: dict[str, Any]) -> str: + dimensions = item.get("dimensions") or {} + if dimensions.get("category"): + return str(dimensions["category"]).upper() + designation = str(item.get("designation") or item.get("specification") or "") + normalized = designation.upper().replace(" ", "") + if normalized.startswith("W"): + return "W-SHAPES" + if normalized.startswith(("PL", "PLATE")) or item.get("geometry"): + return "PLATE PARTS" + if normalized.startswith(("FB", "FLATBAR")): + return "FLAT BAR STOCK" + if item.get("intent") == "hardware": + return "HARDWARE" + return "OTHER" + + +def _size_for(item: dict[str, Any]) -> str: + dimensions = item.get("dimensions") or {} + if dimensions.get("size"): + return str(dimensions["size"]) + if item.get("designation"): + return str(item["designation"]) + if item.get("specification"): + return str(item["specification"]) + geometry = item.get("geometry") or {} + if geometry: + return " x ".join( + _fmt(geometry.get(field)) for field in ("width", "height", "thickness") + ) + return "" + + +def _normalized_item(item: dict[str, Any]) -> dict[str, Any]: + dimensions = item.get("dimensions") or {} + geometry = item.get("geometry") or {} + quantity = item["quantity"] + weight = dimensions.get("purchase_weight_lbs", item.get("total_weight_lbs")) + if weight is None and item.get("length_ft") is not None and item.get("unit_weight_plf") is not None: + weight = quantity * item["length_ft"] * item["unit_weight_plf"] + stock_length = dimensions.get("stock_length") + if stock_length is None and item.get("length_ft") is not None: + stock_length = f"{quantity} pcs x {_fmt(item['length_ft'])} ft" + return { + "source_id": item["source_id"], + "item_id": item["item_id"], + "intent": item["intent"], + "item": item.get("mark") or item["source_id"], + "category": _category_for(item), + "description": item.get("description") or item.get("mark") or item["source_id"], + "size": _size_for(item), + "material": item.get("material", ""), + "grade": item.get("grade", ""), + "thickness": dimensions.get("thickness", geometry.get("thickness")), + "quantity": quantity, + "stock_length": stock_length or "", + "purchase_weight_lbs": weight, + "replaces_item_ids": tuple(dimensions.get("replaces_item_ids", [])), + } + + +def normalize_canonical_package(package: dict[str, Any]) -> dict[str, Any]: + """Map typed canonical scope to deterministic purchase lines.""" + result = validate_estimate_package(package) + if result.blockers: + raise RfqInputError( + "canonical package is blocked: " + + "; ".join(finding["message"] for finding in result.blockers) + ) + source_ids = {item["item_id"] for item in package["items"]} + items_by_id = {item["item_id"]: item for item in package["items"]} + replacements: set[str] = set() + for item in package["items"]: + if item.get("intent") not in {"purchased_stock", "hardware"}: + continue + for replaced_id in (item.get("dimensions") or {}).get( + "replaces_item_ids", [] + ): + if replaced_id not in source_ids: + raise RfqInputError( + f"purchase relationship references unknown item {replaced_id!r}" + ) + if items_by_id[replaced_id]["intent"] != "fabricated_part": + raise RfqInputError( + "purchase relationships may replace fabricated_part items only" + ) + replacements.add(replaced_id) + included = [] + for item in package["items"]: + if item["intent"] in {"allowance", "exclusion", "by_others"}: + continue + if item["item_id"] in replacements: + continue + included.append(_normalized_item(item)) + included.sort( + key=lambda item: ( + CATEGORY_ORDER.get(item["category"], 80), + item["material"], + item["grade"], + item["thickness"] if item["thickness"] is not None else -1, + item["size"], + item["item_id"], + ) + ) + return { + "input_kind": "canonical_estimate_package", + "input_version": ESTIMATE_PACKAGE_VERSION, + "input_hash": result.input_hash, + "project_id": package["project"]["project_id"], + "project_name": package["project"].get("name") + or package["project"]["project_id"], + "revision_id": package["project"]["revision"]["revision_id"], + "currency": package["commercial_basis"]["currency"], + "items": included, + "warnings": [finding["message"] for finding in result.warnings], + } + + +LEGACY_HEADERS = [ + "Source_ID", + "Item_ID", + "Scope", + "Description", + "Material", + "Grade", + "Thickness", + "Size", + "Qty", + "Currency", + "Purchase Weight", +] + + +def normalize_legacy_xlsx(path: str | Path) -> dict[str, Any]: + """Conservatively map one exact legacy sheet/header contract.""" + workbook = openpyxl.load_workbook(path, data_only=False, read_only=True) + if "Steel Takeoff" not in workbook.sheetnames: + raise RfqInputError("legacy workbook requires a 'Steel Takeoff' sheet") + sheet = workbook["Steel Takeoff"] + headers = [sheet.cell(1, column).value for column in range(1, len(LEGACY_HEADERS) + 1)] + if headers != LEGACY_HEADERS: + raise RfqInputError( + "legacy adapter requires exact headers: " + ", ".join(LEGACY_HEADERS) + ) + items = [] + currencies = set() + for row_number, values in enumerate( + sheet.iter_rows(min_row=2, max_col=len(LEGACY_HEADERS), values_only=True), + start=2, + ): + row = dict(zip(LEGACY_HEADERS, values)) + if not any(value is not None for value in values): + continue + if not row["Source_ID"] or not row["Item_ID"]: + raise RfqInputError( + f"legacy row {row_number} requires Source_ID and Item_ID" + ) + if row["Scope"] not in {"IN SCOPE", "BY OTHERS", "EXCLUDED"}: + raise RfqInputError( + f"legacy row {row_number} has ambiguous Scope {row['Scope']!r}" + ) + currency = str(row["Currency"] or "").strip().upper() + if not re.fullmatch(r"[A-Z]{3}", currency): + raise RfqInputError( + f"legacy row {row_number} requires an explicit three-letter Currency" + ) + currencies.add(currency) + try: + quantity = int(row["Qty"]) + except (TypeError, ValueError): + raise RfqInputError( + f"legacy row {row_number} has invalid Qty" + ) from None + if quantity != row["Qty"] or quantity < 0: + raise RfqInputError( + f"legacy row {row_number} has invalid Qty" + ) + intent = ( + "by_others" + if row["Scope"] == "BY OTHERS" + else ("exclusion" if row["Scope"] == "EXCLUDED" or quantity == 0 else "purchased_stock") + ) + if intent in {"by_others", "exclusion"}: + continue + category = _category_for( + { + "intent": intent, + "specification": row["Size"], + "dimensions": {}, + } + ) + if category == "PLATE PARTS": + category = "PLATE STOCK" + item = { + "source_id": str(row["Source_ID"]), + "item_id": str(row["Item_ID"]), + "intent": intent, + "item": str(row["Source_ID"]), + "category": category, + "description": str(row["Description"] or ""), + "size": str(row["Size"] or ""), + "material": str(row["Material"] or ""), + "grade": str(row["Grade"] or ""), + "thickness": row["Thickness"], + "quantity": quantity, + "stock_length": "", + "purchase_weight_lbs": row["Purchase Weight"], + "replaces_item_ids": (), + } + items.append(item) + if len(currencies) != 1: + raise RfqInputError("legacy workbook must use one explicit currency") + items.sort( + key=lambda item: ( + CATEGORY_ORDER.get(item["category"], 80), + item["material"], + item["grade"], + item["thickness"] if item["thickness"] is not None else -1, + item["size"], + item["item_id"], + ) + ) + input_path = Path(path) + input_hash = hashlib.sha256(input_path.read_bytes()).hexdigest() + return { + "input_kind": "legacy_xlsx", + "input_version": "exact-headers-1.0.0", + "input_hash": input_hash, + "project_id": f"legacy-workbook:{input_hash[:20]}", + "project_name": input_path.stem, + "revision_id": "LEGACY-REVISION", + "currency": next(iter(currencies)), + "items": items, + "warnings": [ + "Legacy XLSX mapping used exact headers; confirm typed scope before issue." + ], + } + + +def _user_config_path() -> Path: + xdg = os.environ.get("XDG_CONFIG_HOME") + if xdg: + return Path(xdg) / "pi-steel" / "company-profile.json" + if sys.platform == "darwin": + return ( + Path.home() + / "Library" + / "Application Support" + / "pi-steel" + / "company-profile.json" + ) + return Path.home() / ".config" / "pi-steel" / "company-profile.json" + + +def resolve_company_profile(input_path: str | Path) -> tuple[dict[str, Any] | None, str]: + candidates = [] + explicit = os.environ.get("PI_STEEL_CONFIG") + if explicit: + explicit_path = Path(explicit) + if not explicit_path.is_file(): + raise RfqInputError( + "PI_STEEL_CONFIG names a profile file that does not exist" + ) + candidates.append((explicit_path, "environment")) + candidates.append( + (Path(input_path).resolve().parent / ".pi-steel" / "company-profile.json", "project") + ) + candidates.append((_user_config_path(), "user")) + for path, source in candidates: + if not path.is_file(): + continue + try: + profile = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise RfqInputError(f"company profile could not be read: {exc}") from exc + profile["_profile_path"] = str(path) + return profile, source + return None, "missing" + + +def validate_company_profile( + profile: dict[str, Any] | None, profile_path: str | Path | None = None +) -> list[dict[str, str]]: + findings = [] + + def add(code, message): + findings.append({"code": code, "severity": "error", "message": message}) + + if profile is None: + add("company_profile_missing", "A company profile is required.") + return findings + for field in ("company_name", "city_state"): + if not profile.get(field): + add("company_identity_incomplete", f"Profile field {field} is required.") + terms = profile.get("terms_template") + if not isinstance(terms, dict): + add("approved_terms_missing", "An approved terms_template is required.") + return findings + for field in ( + "template_id", + "content", + "content_hash", + "approver", + "approval_date", + "status", + ): + if not terms.get(field): + add("approved_terms_incomplete", f"Terms field {field} is required.") + content = terms.get("content", "") + actual_hash = hashlib.sha256(content.encode("utf-8")).hexdigest() + if terms.get("content_hash") != actual_hash: + add( + "terms_content_hash_mismatch", + "Terms content changed after approval; approval is invalid.", + ) + if terms.get("status") != "approved": + add("terms_not_approved", "Terms template status must be approved.") + if not _valid_date(terms.get("approval_date")): + add( + "terms_approval_date_invalid", + "Terms approval_date must be an explicit YYYY-MM-DD date.", + ) + return findings + + +def validate_nest_handoff(value: dict[str, Any] | None) -> list[dict[str, str]]: + if value is None: + return [] + findings = [] + if value.get("schema_version") != NEST_HANDOFF_VERSION: + findings.append( + { + "code": "unsupported_nest_handoff_version", + "severity": "error", + "message": "Migrate nesting handoff to version 1.0.0.", + } + ) + if value.get("source_nest_result_version") != "1.0.0": + findings.append( + { + "code": "unsupported_nest_result_version", + "severity": "error", + "message": "Nesting result version must be 1.0.0.", + } + ) + if value.get("geometry_readiness") not in { + "geometry_verified", + "reference_only", + "diagnostic", + }: + findings.append( + { + "code": "invalid_geometry_readiness", + "severity": "error", + "message": "Nesting handoff requires explicit geometry_readiness.", + } + ) + if not isinstance(value.get("rows"), list): + findings.append( + { + "code": "invalid_nest_rows", + "severity": "error", + "message": "Nesting handoff rows must be an array.", + } + ) + return findings + + +def _color(value): + if value is None: + return None + def primitive(attribute): + result = getattr(value, attribute, None) + return result if isinstance(result, (str, int, float, bool)) else None + return { + "type": primitive("type"), + "rgb": primitive("rgb"), + "indexed": primitive("indexed"), + "theme": primitive("theme"), + "tint": primitive("tint"), + } + + +def workbook_semantic_projection(path: str | Path) -> dict[str, Any]: + workbook = openpyxl.load_workbook(path, data_only=False) + sheets = [] + for sheet in workbook.worksheets: + cells = [] + styles = [] + style_ids = {} + for row in sheet.iter_rows(): + for cell in row: + if cell.value is None and not cell.has_style: + continue + style = { + "number_format": cell.number_format, + "font": { + "name": cell.font.name, + "size": cell.font.sz, + "bold": cell.font.b, + "italic": cell.font.i, + "color": _color(cell.font.color), + }, + "fill": { + "type": cell.fill.fill_type, + "fg": _color(cell.fill.fgColor), + }, + "alignment": { + "horizontal": cell.alignment.horizontal, + "vertical": cell.alignment.vertical, + "wrap_text": cell.alignment.wrap_text, + }, + "border": { + side: getattr(cell.border, side).style + for side in ("left", "right", "top", "bottom") + }, + } + style_key = json.dumps(style, sort_keys=True, separators=(",", ":")) + if style_key not in style_ids: + style_ids[style_key] = len(styles) + styles.append(style) + cells.append([cell.coordinate, cell.value, style_ids[style_key]]) + sheets.append( + { + "title": sheet.title, + "state": sheet.sheet_state, + "max_row": sheet.max_row, + "max_column": sheet.max_column, + "merges": sorted(str(value) for value in sheet.merged_cells.ranges), + "freeze_panes": str(sheet.freeze_panes) if sheet.freeze_panes else None, + "column_widths": { + key: dimension.width + for key, dimension in sorted(sheet.column_dimensions.items()) + if dimension.width is not None + }, + "print_title_rows": sheet.print_title_rows, + "page_setup": { + "orientation": sheet.page_setup.orientation, + "fitToWidth": sheet.page_setup.fitToWidth, + "fitToHeight": sheet.page_setup.fitToHeight, + }, + "styles": styles, + "cells": cells, + } + ) + return {"format_version": "1.0.0", "sheets": sheets} + + +def _safe_filename(project_name: str) -> str: + stem = re.sub(r"[^A-Za-z0-9._-]+", "_", project_name).strip("._") + return f"{stem or 'RFQ'}_RFQ_Material_List.xlsx" + + +def compile_workbook( + normalized: dict[str, Any], + profile: dict[str, Any], + *, + nest_handoff: dict[str, Any] | None, + issued_date: str, + project_location: str, + output_directory: str | Path, + profile_source: str, + bake: bool, +) -> dict[str, Any]: + profile_findings = validate_company_profile( + profile, profile.get("_profile_path") + ) + if profile_findings: + raise RfqInputError( + "company profile blocked: " + + "; ".join(finding["message"] for finding in profile_findings) + ) + nest_findings = validate_nest_handoff(nest_handoff) + if nest_findings: + raise RfqInputError( + "nest handoff blocked: " + + "; ".join(finding["message"] for finding in nest_findings) + ) + + workbook = Workbook() + sheet = workbook.active + sheet.title = "RFQ Draft" + metadata = workbook.create_sheet("RFQ Metadata") + metadata.sheet_state = "hidden" + dark_blue = "1F3864" + medium_blue = "2E75B6" + gold = "BF8F00" + yellow = "FFF2CC" + light_blue = "D6E4F0" + green = "E2EFDA" + white = "FFFFFF" + thin = Side(style="thin", color="A6A6A6") + border = Border(left=thin, right=thin, top=thin, bottom=thin) + + for column, width in COLUMN_WIDTHS.items(): + sheet.column_dimensions[column].width = width + for row in (1, 2, 3, 7): + sheet.merge_cells(start_row=row, start_column=1, end_row=row, end_column=14) + cell = sheet.cell(row, 1) + cell.fill = PatternFill("solid", fgColor=dark_blue) + cell.font = Font(name="Arial", color=white, bold=row != 3, size={1: 14, 2: 11}.get(row, 10)) + cell.alignment = Alignment(vertical="center") + for column in range(1, 15): + sheet.cell(row, column).fill = PatternFill("solid", fgColor=dark_blue) + sheet["A1"] = "REQUEST FOR QUOTATION — Structural Steel" + sheet["A2"] = ( + f"{normalized['project_name']} | {profile['company_name']} — " + f"{profile['city_state']}" + ) + sheet["A3"] = ( + f"Date Issued: {issued_date} | Response Requested By: _______________ | " + f"Project Location: {project_location}" + ) + sheet.merge_cells("A5:B5") + sheet.merge_cells("C5:E5") + sheet.merge_cells("F5:G5") + sheet.merge_cells("H5:J5") + sheet.merge_cells("A6:B6") + sheet.merge_cells("C6:E6") + sheet.merge_cells("F6:G6") + sheet.merge_cells("H6:J6") + sheet["A5"], sheet["C5"] = "Company Name", "" + sheet["F5"], sheet["H5"] = "Contact Name", "" + sheet["A6"], sheet["C6"] = "Phone / Email", "" + sheet["F6"], sheet["H6"] = "Quote Valid Until", "" + for coordinate in ("C5", "H5", "C6", "H6"): + sheet[coordinate].fill = PatternFill("solid", fgColor=yellow) + sheet["A7"] = ( + "Instructions: Complete the yellow vendor-response columns. " + "This workbook is a draft request only and is not a purchase authorization." + ) + sheet["A7"].font = Font(name="Arial", size=9, italic=True, color="666666") + + for column, header in enumerate(HEADERS, start=1): + cell = sheet.cell(8, column, header) + cell.fill = PatternFill( + "solid", fgColor=gold if column >= 9 else medium_blue + ) + cell.font = Font(name="Arial", size=10, bold=True, color=white) + cell.alignment = Alignment(horizontal="center", vertical="center", wrap_text=True) + cell.border = border + sheet.freeze_panes = "A9" + sheet.print_title_rows = "8:8" + sheet.page_setup.orientation = "landscape" + sheet.page_setup.fitToWidth = 1 + sheet.page_setup.fitToHeight = 0 + sheet.sheet_properties.pageSetUpPr.fitToPage = True + + row = 9 + material_rows = [] + grouped: dict[tuple[Any, ...], list[dict[str, Any]]] = {} + for item in normalized["items"]: + key = ( + item["category"], + item["material"], + item["grade"], + item["thickness"], + item["size"], + ) + grouped.setdefault(key, []).append(item) + for key in sorted( + grouped, + key=lambda value: ( + CATEGORY_ORDER.get(value[0], 80), + value[1], + value[2], + value[3] if value[3] is not None else -1, + value[4], + ), + ): + category, material, grade, thickness, size = key + sheet.merge_cells(start_row=row, start_column=1, end_row=row, end_column=14) + heading = sheet.cell( + row, + 1, + f"{category} — {material} — {grade}" + + (f" — {_fmt(thickness)}" if thickness is not None else ""), + ) + heading.fill = PatternFill("solid", fgColor=medium_blue) + heading.font = Font(name="Arial", bold=True, color=white) + for column in range(1, 15): + sheet.cell(row, column).fill = PatternFill("solid", fgColor=medium_blue) + row += 1 + for item in grouped[key]: + material_rows.append(row) + values = [ + item["item"], + item["category"], + item["description"], + item["size"], + item["grade"], + item["quantity"], + item["stock_length"], + item["purchase_weight_lbs"], + ] + for column, value in enumerate(values, start=1): + sheet.cell(row, column, value) + sheet.cell(row, 10, f'=IF(I{row}="","",F{row}*I{row})') + for column in range(1, 15): + cell = sheet.cell(row, column) + cell.border = border + cell.font = Font(name="Arial", size=10) + cell.alignment = Alignment(vertical="top", wrap_text=column in {3, 7, 14}) + cell.fill = PatternFill( + "solid", + fgColor=yellow + if column >= 9 + else (light_blue if len(material_rows) % 2 else white), + ) + sheet.cell(row, 8).number_format = "#,##0" + sheet.cell(row, 9).number_format = f'"{normalized["currency"]}" #,##0.00' + sheet.cell(row, 10).number_format = f'"{normalized["currency"]}" #,##0.00' + row += 1 + + if not material_rows: + raise RfqInputError("no in-scope purchase items remain after typed filtering") + first_material_row = material_rows[0] + last_material_row = material_rows[-1] + total_row = row + sheet.merge_cells(start_row=total_row, start_column=1, end_row=total_row, end_column=7) + sheet.cell(total_row, 1, "TOTAL PURCHASE WEIGHT / PRICE") + sheet.cell(total_row, 1).font = Font(name="Arial", bold=True) + sheet.cell(total_row, 1).alignment = Alignment(horizontal="right") + sheet.cell(total_row, 8, f"=SUM(H{first_material_row}:H{last_material_row})") + sheet.cell(total_row, 10, f"=SUM(J{first_material_row}:J{last_material_row})") + for column in (8, 10): + sheet.cell(total_row, column).fill = PatternFill("solid", fgColor=green) + sheet.cell(total_row, column).font = Font(name="Arial", bold=True) + sheet.cell(total_row, column).border = border + sheet.cell(total_row, 8).number_format = "#,##0" + sheet.cell(total_row, 10).number_format = f'"{normalized["currency"]}" #,##0.00' + + nest_header_row = total_row + 2 + sheet.merge_cells( + start_row=nest_header_row, start_column=1, end_row=nest_header_row, end_column=14 + ) + sheet.cell( + nest_header_row, 1, "NESTING / REMNANT REFERENCE (For Fabricator Review)" + ) + sheet.cell(nest_header_row, 1).font = Font( + name="Arial", bold=True, color=dark_blue + ) + nest_columns_row = nest_header_row + 1 + nest_headers = [ + "Material", + "Grade", + "Thickness", + "Sheet Size", + "Nesting Plan", + "Remnant Notes", + ] + for column, header in enumerate(nest_headers, start=1): + cell = sheet.cell(nest_columns_row, column, header) + cell.fill = PatternFill("solid", fgColor=medium_blue) + cell.font = Font(name="Arial", bold=True, color=white) + cell.border = border + nest_row = nest_columns_row + 1 + reference_warning = ( + (nest_handoff or {}).get("geometry_readiness") == "reference_only" + ) + for entry in (nest_handoff or {}).get("rows", []): + readiness = entry.get( + "geometry_readiness", + (nest_handoff or {}).get("geometry_readiness"), + ) + reference_warning = reference_warning or readiness == "reference_only" + values = [ + entry.get("material"), + entry.get("grade"), + entry.get("thickness"), + entry.get("sheet_size"), + entry.get("nesting_plan"), + entry.get("drop_notes") + + (" — REFERENCE ONLY" if readiness == "reference_only" else ""), + ] + for column, value in enumerate(values, start=1): + cell = sheet.cell(nest_row, column, value) + cell.border = border + cell.alignment = Alignment(vertical="top", wrap_text=True) + nest_row += 1 + + terms_header_row = nest_row + 1 + sheet.merge_cells( + start_row=terms_header_row, start_column=1, end_row=terms_header_row, end_column=14 + ) + sheet.cell(terms_header_row, 1, "TERMS & CONDITIONS — APPROVED TEMPLATE") + sheet.cell(terms_header_row, 1).font = Font( + name="Arial", bold=True, color=dark_blue + ) + terms_row = terms_header_row + 1 + sheet.merge_cells( + start_row=terms_row, start_column=1, end_row=terms_row, end_column=14 + ) + sheet.cell(terms_row, 1, profile["terms_template"]["content"]) + sheet.cell(terms_row, 1).alignment = Alignment(wrap_text=True, vertical="top") + + metadata["A1"], metadata["B1"] = "Field", "Value" + metadata["A2"], metadata["B2"] = "Document Status", "DRAFT — NOT SENT OR AWARDED" + metadata["A3"], metadata["B3"] = "Compiler Version", RFQ_COMPILER_VERSION + metadata["A4"], metadata["B4"] = "Input Hash", normalized["input_hash"] + metadata["A5"], metadata["B5"] = "Profile Source", profile_source + metadata["A6"], metadata["B6"] = "Terms Template ID", profile["terms_template"]["template_id"] + metadata["A7"], metadata["B7"] = "Terms Content Hash", profile["terms_template"]["content_hash"] + metadata["A8"], metadata["B8"] = "Terms Approver", profile["terms_template"]["approver"] + metadata["A9"], metadata["B9"] = "Terms Approval Date", profile["terms_template"]["approval_date"] + metadata["A10"], metadata["B10"] = "Issued Date", issued_date + metadata["A11"], metadata["B11"] = ( + "Formula Cache", + "Recalculate on open requested; QA report records cache status", + ) + workbook.calculation.fullCalcOnLoad = True + workbook.calculation.forceFullCalc = True + workbook.calculation.calcMode = "auto" + + logo_status = "not_configured" + logo = profile.get("logo") + if logo: + profile_path = Path(profile.get("_profile_path", ".")) + logo_path = Path(logo) + if not logo_path.is_absolute(): + logo_path = profile_path.parent / logo_path + if logo_path.is_file(): + try: + image = Image(logo_path) + image.width, image.height = 90, 36 + sheet.add_image(image, "A1") + logo_status = "embedded" + except Exception: + logo_status = "text_fallback" + else: + logo_status = "text_fallback" + + output_directory = Path(output_directory) + output_directory.mkdir(parents=True, exist_ok=True) + workbook_path = output_directory / _safe_filename(normalized["project_name"]) + workbook.save(workbook_path) + recalculation_status = recalc.recalculate(workbook_path, bake=bake) + return { + "workbook_path": workbook_path, + "recalculation_status": recalculation_status, + "logo_status": logo_status, + "reference_warning": reference_warning, + "contract": { + "first_material_row": first_material_row, + "last_material_row": last_material_row, + "total_row": total_row, + "nest_header_row": nest_header_row, + "terms_header_row": terms_header_row, + }, + } + + +def _package_version() -> str: + path = Path(__file__).resolve().parents[3] / "package.json" + try: + return json.loads(path.read_text(encoding="utf-8"))["version"] + except (OSError, KeyError, json.JSONDecodeError): + return "unknown" + + +def _load_normalized(input_path: Path) -> dict[str, Any]: + if input_path.suffix.lower() == ".json": + value = json.loads(input_path.read_text(encoding="utf-8")) + return normalize_canonical_package(value) + if input_path.suffix.lower() == ".xlsx": + return normalize_legacy_xlsx(input_path) + raise RfqInputError("input must be a canonical .json package or exact legacy .xlsx") + + +def publish_rfq_run(args) -> tuple[dict[str, Any], Path]: + input_path = Path(args.input) + input_findings = [] + if not _valid_date(args.issued_date): + input_findings.append( + { + "code": "invalid_issued_date", + "severity": "error", + "message": "issued-date must be an explicit YYYY-MM-DD date", + } + ) + try: + normalized = _load_normalized(input_path) + except (RfqInputError, json.JSONDecodeError) as exc: + normalized = { + "input_kind": "invalid", + "input_version": "unknown", + "input_hash": hashlib.sha256(input_path.read_bytes()).hexdigest(), + "project_id": input_path.stem, + "project_name": input_path.stem, + "revision_id": "unknown", + "currency": "USD", + "items": [], + "warnings": [], + } + input_findings.append( + { + "code": "invalid_rfq_input", + "severity": "error", + "message": str(exc), + } + ) + try: + profile, profile_source = resolve_company_profile(input_path) + except RfqInputError as exc: + profile, profile_source = None, "invalid" + input_findings.append( + { + "code": "invalid_company_profile", + "severity": "error", + "message": str(exc), + } + ) + profile_findings = validate_company_profile( + profile, profile.get("_profile_path") if profile else None + ) + try: + nest_handoff = ( + json.loads(Path(args.nest).read_text(encoding="utf-8")) + if args.nest + else None + ) + nest_findings = validate_nest_handoff(nest_handoff) + except (OSError, json.JSONDecodeError) as exc: + nest_handoff = None + nest_findings = [ + { + "code": "invalid_nest_handoff", + "severity": "error", + "message": str(exc), + } + ] + findings = input_findings + profile_findings + nest_findings + blockers = [finding for finding in findings if finding["severity"] == "error"] + review_reasons = list(normalized["warnings"]) + if any( + readiness == "reference_only" + for readiness in [ + (nest_handoff or {}).get("geometry_readiness"), + *[ + row.get("geometry_readiness") + for row in (nest_handoff or {}).get("rows", []) + ], + ] + ): + review_reasons.append("Nesting handoff contains reference-only geometry.") + if blockers: + outcome, package_status = "blocked", "draft" + elif review_reasons: + outcome, package_status = "review_required", "rfq_draft_review_required" + else: + outcome, package_status = "ready", "rfq_ready_for_review" + profile_hash = ( + sha256_bytes(canonical_json_bytes({k: v for k, v in profile.items() if not k.startswith("_")})) + if profile + else sha256_bytes(b"missing-profile") + ) + configuration_hash = sha256_bytes( + canonical_json_bytes( + { + "compiler_version": RFQ_COMPILER_VERSION, + "issued_date": args.issued_date, + "project_location": args.project_location, + "profile_hash": profile_hash, + "nest_handoff": nest_handoff, + "bake_requested": not args.no_bake, + } + ) + ) + qa_report = { + "schema_version": "1.0.0", + "stage": "steel-rfq", + "run_outcome": outcome, + "package_status": package_status, + "document_status": "DRAFT — NOT SENT OR AWARDED", + "profile_source": profile_source, + "findings": findings, + "warnings": review_reasons, + } + with RunPublisher( + args.out, + stage="steel-rfq", + run_outcome=outcome, + package_status=package_status, + input_hash=normalized["input_hash"], + configuration_hash=configuration_hash, + schema_versions={ + "run_manifest": "1.0.0", + "estimate_package": normalized["input_version"], + "rfq_workbook": RFQ_COMPILER_VERSION, + "rfq_nesting": NEST_HANDOFF_VERSION, + }, + tool_versions={ + "pi_steel": _package_version(), + "rfq_compiler": RFQ_COMPILER_VERSION, + }, + explicit_dates={"issued_date": args.issued_date}, + warnings=review_reasons + + [finding["message"] for finding in findings], + approximations=[], + run_id=args.run_id, + ) as publisher: + compile_result = None + if not blockers: + compile_result = compile_workbook( + normalized, + profile, + nest_handoff=nest_handoff, + issued_date=args.issued_date, + project_location=args.project_location, + output_directory=publisher.staging_path, + profile_source=profile_source, + bake=not args.no_bake, + ) + qa_report["recalculation_status"] = compile_result["recalculation_status"] + qa_report["logo_status"] = compile_result["logo_status"] + workbook_name = compile_result["workbook_path"].name + publisher.register_artifact( + workbook_name, + readiness="draft", + media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ) + publisher.write_json( + "workbook-semantic.json", + workbook_semantic_projection(compile_result["workbook_path"]), + readiness="diagnostic", + ) + publisher.write_qa_report(qa_report) + final_path = publisher.publish() + return qa_report, final_path + + +def main(argv=None) -> int: + parser = StageArgumentParser(description=__doc__) + parser.add_argument("--input", required=True) + parser.add_argument("--nest") + parser.add_argument("--out", default="out") + parser.add_argument("--issued-date", required=True) + parser.add_argument("--project-location", default="") + parser.add_argument("--no-bake", action="store_true") + parser.add_argument("--run-id", help=argparse.SUPPRESS) + args = parser.parse_args(argv) + try: + qa_report, final_path = publish_rfq_run(args) + except (OSError, json.JSONDecodeError, RfqInputError) as exc: + print(f"RFQ generation failed: {exc}", file=sys.stderr) + return 1 + print(f"Published {qa_report['run_outcome']} draft RFQ run: {final_path}") + return outcome_exit_code(qa_report["run_outcome"]) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/steel-rfq/scripts/recalc.py b/skills/steel-rfq/scripts/recalc.py index d0866a7..5c635d6 100644 --- a/skills/steel-rfq/scripts/recalc.py +++ b/skills/steel-rfq/scripts/recalc.py @@ -52,6 +52,14 @@ def libreoffice_bake(path): return False +def recalculate(path, *, bake=True): + """Set recalc-on-open and return an honest cache status.""" + flag_recalc_on_load(path) + if bake and libreoffice_bake(path): + return "baked_via_libreoffice" + return "deferred_recalculate_on_open" + + def main(): if len(sys.argv) < 2: sys.exit("usage: python3 recalc.py ") @@ -61,9 +69,8 @@ def main(): # Set recalc-on-open FIRST so LibreOffice honors it, then bake. Do NOT # reload with openpyxl afterward — that would strip the cached values # LibreOffice just computed. - flag_recalc_on_load(path) - baked = libreoffice_bake(path) - if baked: + status = recalculate(path) + if status == "baked_via_libreoffice": print(f"recalc: values computed and baked via LibreOffice — {path}") else: print(f"recalc: recalc-on-open set (open in Excel to compute values) — {path}") diff --git a/tests/fixtures/rfq/README.md b/tests/fixtures/rfq/README.md new file mode 100644 index 0000000..d1112ca --- /dev/null +++ b/tests/fixtures/rfq/README.md @@ -0,0 +1,9 @@ +# Synthetic RFQ fixtures + +Every fixture in this directory was authored from scratch for pi-steel tests. +Names use the public-policy placeholders, identifiers are prefixed with +`SYNTHETIC-`, and no file is derived from an operating company, vendor quote, +customer estimate, purchasing terms, or production workbook. + +The profile's approved terms are explicitly labeled synthetic test text. The +fixture contains weights but no vendor prices or commercial rates. diff --git a/tests/fixtures/rfq/estimate-package.json b/tests/fixtures/rfq/estimate-package.json new file mode 100644 index 0000000..9303804 --- /dev/null +++ b/tests/fixtures/rfq/estimate-package.json @@ -0,0 +1,134 @@ +{ + "schema_version": "1.0.0", + "project": { + "project_id": "SYNTHETIC-RFQ-001", + "name": "Synthetic RFQ Project", + "revision": { + "revision_id": "SYNTHETIC-REV-A", + "source_document": "SYNTHETIC-ESTIMATE" + } + }, + "unit_system": "imperial", + "items": [ + { + "intent": "fabricated_part", + "source_id": "SYNTHETIC-SRC-W1", + "item_id": "item:synthetic-w1", + "quantity": 2, + "mark": "W1", + "description": "Member whose description says BY OTHERS but remains typed fabricated scope", + "material": "carbon_steel", + "grade": "A992", + "designation": "W12X26", + "length_ft": 12, + "unit_weight_plf": 26, + "total_weight_lbs": 624, + "source_evidence": [ + { + "source": "SYNTHETIC-DRAWING", + "locator": "SYNTHETIC-DETAIL-W1" + } + ] + }, + { + "intent": "fabricated_part", + "source_id": "SYNTHETIC-SRC-P1", + "item_id": "item:synthetic-p1", + "quantity": 4, + "mark": "P1", + "description": "Synthetic connection plate", + "material": "carbon_steel", + "grade": "A36", + "geometry": { + "shape": "rect", + "width": 8, + "height": 6, + "thickness": 0.5, + "holes": [] + }, + "source_evidence": [ + { + "source": "SYNTHETIC-DRAWING", + "locator": "SYNTHETIC-DETAIL-P1" + } + ] + }, + { + "intent": "purchased_stock", + "source_id": "SYNTHETIC-SRC-SHEET1", + "item_id": "item:synthetic-sheet1", + "quantity": 1, + "mark": "S1", + "description": "Synthetic full sheet purchase", + "material": "carbon_steel", + "grade": "A36", + "specification": "PLATE", + "dimensions": { + "category": "PLATE STOCK", + "width": 48, + "height": 24, + "thickness": 0.5, + "size": "48 x 24 x 0.5", + "purchase_weight_lbs": 163, + "replaces_item_ids": ["item:synthetic-p1"] + }, + "source_evidence": [ + { + "source": "SYNTHETIC-ESTIMATE", + "locator": "SYNTHETIC-STOCK-ROW-1" + } + ] + }, + { + "intent": "hardware", + "source_id": "SYNTHETIC-SRC-H1", + "item_id": "item:synthetic-h1", + "quantity": 8, + "mark": "H1", + "description": "Synthetic bolt assembly", + "material": "steel", + "grade": "A325", + "specification": "3/4 in bolt", + "dimensions": { + "category": "HARDWARE", + "size": "3/4 in", + "purchase_weight_lbs": 12 + }, + "source_evidence": [ + { + "source": "SYNTHETIC-ESTIMATE", + "locator": "SYNTHETIC-HARDWARE-ROW-1" + } + ] + }, + { + "intent": "by_others", + "source_id": "SYNTHETIC-SRC-BO1", + "item_id": "item:synthetic-bo1", + "quantity": 1, + "description": "Synthetic excluded scope", + "reason": "Explicitly typed by others", + "source_evidence": [ + { + "source": "SYNTHETIC-ESTIMATE", + "locator": "SYNTHETIC-SCOPE-ROW-1" + } + ] + } + ], + "stock": [], + "commercial_basis": { + "currency": "USD", + "costs": [] + }, + "review": { + "status": "validated", + "findings": [], + "acknowledgements": [] + }, + "lineage": { + "source_type": "synthetic_test", + "source_hash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "configuration_hash": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } +} diff --git a/tests/fixtures/rfq/nest-handoff.json b/tests/fixtures/rfq/nest-handoff.json new file mode 100644 index 0000000..4c69f66 --- /dev/null +++ b/tests/fixtures/rfq/nest-handoff.json @@ -0,0 +1,51 @@ +{ + "schema_version": "1.0.0", + "source_nest_result_version": "1.0.0", + "geometry_readiness": "reference_only", + "rows": [ + { + "stock_id": "SYNTHETIC-STOCK-A36-0500", + "stock_name": "Synthetic Plate", + "material": "carbon_steel", + "grade": "A36", + "thickness": 0.5, + "sheets_needed": 1, + "sheet_size": "48x24", + "packing_utilization_pct": 42.5, + "nesting_plan": "1 x 48x24 sheet(s) - 42.5% packing utilization, 4 parts", + "drop_notes": "Remnant candidates (not certified reusable): 20x10 in", + "remnant_candidates": [ + { + "width": 20, + "height": 10, + "area": 200, + "status": "candidate_unverified" + } + ], + "total_cost": null, + "geometry_readiness": "reference_only" + }, + { + "stock_id": "SYNTHETIC-STOCK-A572-0375", + "stock_name": "Synthetic Plate", + "material": "carbon_steel", + "grade": "A572", + "thickness": 0.375, + "sheets_needed": 1, + "sheet_size": "60x30", + "packing_utilization_pct": 55, + "nesting_plan": "1 x 60x30 sheet(s) - 55% packing utilization, 3 parts", + "drop_notes": "Remnant candidates (not certified reusable): 18x8 in", + "remnant_candidates": [ + { + "width": 18, + "height": 8, + "area": 144, + "status": "candidate_unverified" + } + ], + "total_cost": null, + "geometry_readiness": "geometry_verified" + } + ] +} diff --git a/tests/fixtures/rfq/synthetic-profile.json b/tests/fixtures/rfq/synthetic-profile.json new file mode 100644 index 0000000..9c817a7 --- /dev/null +++ b/tests/fixtures/rfq/synthetic-profile.json @@ -0,0 +1,13 @@ +{ + "company_name": "Example Fabricator", + "city_state": "Example City, ST", + "logo": "missing-optional-logo.png", + "terms_template": { + "template_id": "SYNTHETIC-TERMS-001", + "content": "SYNTHETIC TEST TERMS — Quote subject to written review.", + "content_hash": "abb47a7a3863dc00db1a93df990d5f1626afbb89ce14489f6bf5d7ccd9ca463c", + "approver": "Example Approver", + "approval_date": "2026-07-28", + "status": "approved" + } +} diff --git a/tests/golden/rfq/semantic-workbook.json b/tests/golden/rfq/semantic-workbook.json new file mode 100644 index 0000000..100fbeb --- /dev/null +++ b/tests/golden/rfq/semantic-workbook.json @@ -0,0 +1,20 @@ +{ + "projection_format": "1.0.0", + "semantic_sha256": "2414a162bfd1b2efe4c270232353fed71f3ca8f578a9848262127a1aca7b2a25", + "sheets": [ + { + "title": "RFQ Draft", + "max_row": 23, + "max_column": 14, + "cell_count": 95, + "style_count": 25 + }, + { + "title": "RFQ Metadata", + "max_row": 11, + "max_column": 2, + "cell_count": 22, + "style_count": 1 + } + ] +} diff --git a/tests/test_installed_scripts.py b/tests/test_installed_scripts.py index 14729f1..3e1e652 100644 --- a/tests/test_installed_scripts.py +++ b/tests/test_installed_scripts.py @@ -30,6 +30,8 @@ def test_packed_npm_artifact_contains_runtime_and_runs_doctor(tmp_path): "package/skills/_shared/pi_steel/__init__.py", "package/skills/_shared/pi_steel/run_manifest.py", "package/skills/_shared/schemas/run-manifest.schema.json", + "package/skills/steel-rfq/scripts/generate-rfq.py", + "package/skills/steel-rfq/references/rfq-input.md", "package/pyproject.toml", "package/requirements.txt", "package/requirements-tested.txt", diff --git a/tests/test_nest_engine.py b/tests/test_nest_engine.py index daf46be..b233730 100644 --- a/tests/test_nest_engine.py +++ b/tests/test_nest_engine.py @@ -106,6 +106,7 @@ def test_material_groups_and_same_name_stock_variants_remain_distinct(): } assert len({plate["stock_id"] for plate in result["plate_reports"]}) == 2 assert len(result["rfq_nesting"]["rows"]) == 2 + assert result["rfq_nesting"]["geometry_readiness"] == "geometry_verified" reordered = deepcopy(job) reordered["parts"].reverse() @@ -194,4 +195,5 @@ def test_packing_and_net_yield_are_separate_and_labeled(): assert metrics["net_material_yield_pct"]["value"] < 6.0 assert metrics["packing_utilization_pct"]["approximation"] == "bounding_box" assert metrics["net_material_yield_pct"]["approximation"] == "declared_area" + assert result["rfq_nesting"]["geometry_readiness"] == "reference_only" assert "overall_yield_pct" not in result diff --git a/tests/test_rfq_generator.py b/tests/test_rfq_generator.py new file mode 100644 index 0000000..e49b41d --- /dev/null +++ b/tests/test_rfq_generator.py @@ -0,0 +1,228 @@ +import importlib.util +import json +import os +import subprocess +import sys +from copy import deepcopy +from pathlib import Path + +import openpyxl +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "skills" / "steel-rfq" / "scripts" / "generate-rfq.py" +SPEC = importlib.util.spec_from_file_location("pi_steel_generate_rfq", SCRIPT) +rfq = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = rfq +SPEC.loader.exec_module(rfq) +FIXTURES = ROOT / "tests" / "fixtures" / "rfq" + + +def load(name): + return json.loads((FIXTURES / name).read_text()) + + +def test_canonical_normalization_uses_typed_scope_and_explicit_replacements_only(): + package = load("estimate-package.json") + normalized = rfq.normalize_canonical_package(package) + ids = [item["item_id"] for item in normalized["items"]] + assert "item:synthetic-p1" not in ids + assert "item:synthetic-sheet1" in ids + assert "item:synthetic-bo1" not in ids + assert "item:synthetic-w1" in ids + assert next( + item for item in normalized["items"] if item["item_id"] == "item:synthetic-w1" + )["description"].find("BY OTHERS") >= 0 + + package["items"][2]["dimensions"].pop("replaces_item_ids") + ids_without_relationship = [ + item["item_id"] + for item in rfq.normalize_canonical_package(package)["items"] + ] + assert "item:synthetic-p1" in ids_without_relationship + + +def test_legacy_adapter_requires_exact_headers_and_maps_explicit_scope(tmp_path): + path = tmp_path / "SYNTHETIC-legacy.xlsx" + workbook = openpyxl.Workbook() + sheet = workbook.active + sheet.title = "Steel Takeoff" + sheet.append( + [ + "Source_ID", + "Item_ID", + "Scope", + "Description", + "Material", + "Grade", + "Thickness", + "Size", + "Qty", + "Currency", + "Purchase Weight", + ] + ) + sheet.append( + [ + "SYNTHETIC-SRC-1", + "item:synthetic-legacy-1", + "IN SCOPE", + "Synthetic member", + "carbon_steel", + "A992", + None, + "W10X22", + 2, + "USD", + 440, + ] + ) + sheet.append( + [ + "SYNTHETIC-SRC-2", + "item:synthetic-legacy-2", + "BY OTHERS", + "Synthetic excluded item", + "carbon_steel", + "A36", + 0.5, + "PL 8x6", + 1, + "USD", + 20, + ] + ) + sheet.append( + [ + "SYNTHETIC-SRC-3", + "item:synthetic-legacy-3", + "IN SCOPE", + "Synthetic zero quantity", + "carbon_steel", + "A572", + 0.375, + "PL 6x4", + 0, + "USD", + 0, + ] + ) + workbook.save(path) + + normalized = rfq.normalize_legacy_xlsx(path) + assert [item["item_id"] for item in normalized["items"]] == [ + "item:synthetic-legacy-1" + ] + + sheet.delete_cols(1) + workbook.save(path) + with pytest.raises(rfq.RfqInputError, match="exact headers"): + rfq.normalize_legacy_xlsx(path) + + +def test_profile_resolution_precedence_and_term_hash_invalidation( + tmp_path, monkeypatch +): + explicit = tmp_path / "explicit.json" + explicit.write_text((FIXTURES / "synthetic-profile.json").read_text()) + project = tmp_path / ".pi-steel" + project.mkdir() + (project / "company-profile.json").write_text( + (FIXTURES / "synthetic-profile.json").read_text() + ) + monkeypatch.setenv("PI_STEEL_CONFIG", str(explicit)) + profile, source = rfq.resolve_company_profile(tmp_path / "estimate.json") + assert source == "environment" + assert profile["company_name"] == "Example Fabricator" + + changed = deepcopy(profile) + changed["terms_template"]["content"] += " changed" + findings = rfq.validate_company_profile(changed, explicit) + assert "terms_content_hash_mismatch" in {finding["code"] for finding in findings} + + +def test_cli_blocks_missing_profile_and_publishes_only_diagnostics( + tmp_path, monkeypatch +): + monkeypatch.delenv("PI_STEEL_CONFIG", raising=False) + environment = os.environ.copy() + environment.pop("PI_STEEL_CONFIG", None) + environment["XDG_CONFIG_HOME"] = str(tmp_path / "empty-config") + output = tmp_path / "published" + completed = subprocess.run( + [ + sys.executable, + SCRIPT, + "--input", + FIXTURES / "estimate-package.json", + "--out", + output, + "--issued-date", + "2026-07-28", + "--run-id", + "SYNTHETIC-RFQ-BLOCKED", + "--no-bake", + ], + cwd=tmp_path, + env=environment, + text=True, + capture_output=True, + ) + assert completed.returncode == 3, completed.stdout + completed.stderr + pointer = json.loads((output / "latest-run.json").read_text()) + run_path = output / pointer["run_directory"] + assert (run_path / "qa-report.json").exists() + assert not list(run_path.glob("*.xlsx")) + assert json.loads((run_path / "run-manifest.json").read_text())[ + "run_outcome" + ] == "blocked" + + +@pytest.mark.parametrize( + ("nest_name", "expected_exit", "expected_outcome"), + [ + (None, 0, "ready"), + ("nest-handoff.json", 2, "review_required"), + ], +) +def test_cli_ready_and_review_runs_publish_draft_workbooks( + tmp_path, nest_name, expected_exit, expected_outcome +): + environment = os.environ.copy() + environment["PI_STEEL_CONFIG"] = str(FIXTURES / "synthetic-profile.json") + output = tmp_path / f"published-{expected_outcome}" + command = [ + sys.executable, + SCRIPT, + "--input", + FIXTURES / "estimate-package.json", + "--out", + output, + "--issued-date", + "2026-07-28", + "--run-id", + f"SYNTHETIC-RFQ-{expected_outcome.upper()}", + "--no-bake", + ] + if nest_name: + command.extend(["--nest", FIXTURES / nest_name]) + completed = subprocess.run( + command, + cwd=tmp_path, + env=environment, + text=True, + capture_output=True, + ) + assert completed.returncode == expected_exit, completed.stdout + completed.stderr + pointer = json.loads((output / "latest-run.json").read_text()) + run_path = output / pointer["run_directory"] + manifest = json.loads((run_path / "run-manifest.json").read_text()) + assert manifest["run_outcome"] == expected_outcome + assert any( + artifact["readiness"] == "draft" + and artifact["path"].endswith(".xlsx") + for artifact in manifest["artifacts"] + ) + workbook = openpyxl.load_workbook(next(run_path.glob("*.xlsx"))) + assert workbook["RFQ Metadata"]["B2"].value == "DRAFT — NOT SENT OR AWARDED" diff --git a/tests/test_rfq_workbook_contract.py b/tests/test_rfq_workbook_contract.py new file mode 100644 index 0000000..e2cb805 --- /dev/null +++ b/tests/test_rfq_workbook_contract.py @@ -0,0 +1,108 @@ +import importlib.util +import hashlib +import json +import sys +from pathlib import Path + +import openpyxl + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "skills" / "steel-rfq" / "scripts" / "generate-rfq.py" +SPEC = importlib.util.spec_from_file_location("pi_steel_rfq_workbook", SCRIPT) +rfq = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = rfq +SPEC.loader.exec_module(rfq) +FIXTURES = ROOT / "tests" / "fixtures" / "rfq" +GOLDEN = ROOT / "tests" / "golden" / "rfq" / "semantic-workbook.json" + + +def load(name): + return json.loads((FIXTURES / name).read_text()) + + +def build(tmp_path, monkeypatch): + package = load("estimate-package.json") + normalized = rfq.normalize_canonical_package(package) + profile = load("synthetic-profile.json") + nest = load("nest-handoff.json") + monkeypatch.setattr(rfq.recalc, "libreoffice_bake", lambda path: False) + return rfq.compile_workbook( + normalized, + profile, + nest_handoff=nest, + issued_date="2026-07-28", + project_location="Example City, ST", + output_directory=tmp_path, + profile_source="environment", + bake=True, + ) + + +def test_workbook_structure_formulas_styles_and_draft_metadata(tmp_path, monkeypatch): + result = build(tmp_path, monkeypatch) + workbook = openpyxl.load_workbook(result["workbook_path"], data_only=False) + sheet = workbook["RFQ Draft"] + assert sheet.merged_cells.ranges + assert {"A1:N1", "A2:N2", "A3:N3", "A7:N7"} <= { + str(value) for value in sheet.merged_cells.ranges + } + assert sheet.freeze_panes == "A9" + assert sheet.print_title_rows == "$8:$8" + assert sheet.page_setup.orientation == "landscape" + assert sheet.page_setup.fitToWidth == 1 + assert sheet.column_dimensions["C"].width == 38 + assert sheet["I8"].fill.fgColor.rgb.endswith("BF8F00") + assert sheet["I10"].fill.fgColor.rgb.endswith("FFF2CC") + total_row = result["contract"]["total_row"] + first_row = result["contract"]["first_material_row"] + last_row = result["contract"]["last_material_row"] + assert sheet[f"H{total_row}"].value == f"=SUM(H{first_row}:H{last_row})" + assert sheet[f"J{total_row}"].value == f"=SUM(J{first_row}:J{last_row})" + assert workbook["RFQ Metadata"]["B2"].value == "DRAFT — NOT SENT OR AWARDED" + assert result["recalculation_status"] == "deferred_recalculate_on_open" + assert result["logo_status"] == "text_fallback" + + +def test_nesting_rows_stay_separate_and_reference_warning_is_visible( + tmp_path, monkeypatch +): + result = build(tmp_path, monkeypatch) + workbook = openpyxl.load_workbook(result["workbook_path"], data_only=False) + sheet = workbook["RFQ Draft"] + values = [ + tuple(sheet.cell(row=row, column=column).value for column in range(1, 7)) + for row in range(1, sheet.max_row + 1) + ] + nest_rows = [row for row in values if row[0] in {"carbon_steel"}] + assert len(nest_rows) == 2 + assert {(row[1], row[2], row[3]) for row in nest_rows} == { + ("A36", 0.5, "48x24"), + ("A572", 0.375, "60x30"), + } + assert any( + "REFERENCE ONLY" in str(cell).upper() + for row in values + for cell in row + if cell + ) + + +def test_semantic_projection_matches_golden(tmp_path, monkeypatch): + result = build(tmp_path, monkeypatch) + actual = rfq.workbook_semantic_projection(result["workbook_path"]) + expected = json.loads(GOLDEN.read_text()) + canonical = json.dumps( + actual, sort_keys=True, separators=(",", ":"), ensure_ascii=False + ).encode() + assert hashlib.sha256(canonical).hexdigest() == expected["semantic_sha256"] + assert [ + { + "title": sheet["title"], + "max_row": sheet["max_row"], + "max_column": sheet["max_column"], + "cell_count": len(sheet["cells"]), + "style_count": len(sheet["styles"]), + } + for sheet in actual["sheets"] + ] == expected["sheets"] From 2c17a2123d17da7b59412e165e41e60d739e9f23 Mon Sep 17 00:00:00 2001 From: Victor Garcia Date: Tue, 28 Jul 2026 12:56:32 -0600 Subject: [PATCH 09/15] feat(estimate): orchestrate trustworthy package pipeline --- skills/_shared/pi_steel/validation.py | 13 + skills/steel-estimate/SKILL.md | 80 ++ .../references/estimate-package-example.json | 91 ++ .../references/output-contract.md | 47 + .../scripts/build-estimate-package.py | 818 ++++++++++++++++++ skills/steel-rfq/scripts/generate-rfq.py | 10 +- tests/fixtures/pipeline/README.md | 10 + .../fixtures/pipeline/synthetic-estimate.json | 229 +++++ .../fixtures/pipeline/synthetic-profile.json | 13 + tests/golden/pipeline/ready-artifacts.json | 13 + tests/test_estimate_pipeline.py | 329 +++++++ tests/test_installed_scripts.py | 42 + tests/test_rfq_generator.py | 18 + 13 files changed, 1710 insertions(+), 3 deletions(-) create mode 100644 skills/steel-estimate/SKILL.md create mode 100644 skills/steel-estimate/references/estimate-package-example.json create mode 100644 skills/steel-estimate/references/output-contract.md create mode 100755 skills/steel-estimate/scripts/build-estimate-package.py create mode 100644 tests/fixtures/pipeline/README.md create mode 100644 tests/fixtures/pipeline/synthetic-estimate.json create mode 100644 tests/fixtures/pipeline/synthetic-profile.json create mode 100644 tests/golden/pipeline/ready-artifacts.json create mode 100644 tests/test_estimate_pipeline.py diff --git a/skills/_shared/pi_steel/validation.py b/skills/_shared/pi_steel/validation.py index 3d0b880..4d4939a 100644 --- a/skills/_shared/pi_steel/validation.py +++ b/skills/_shared/pi_steel/validation.py @@ -12,6 +12,7 @@ from .contracts import ( ESTIMATE_PACKAGE_VERSION, + content_hash, estimate_input_hash, finding_id_for, ) @@ -201,6 +202,16 @@ def _geometry_findings( def validate_estimate_package(package: dict[str, Any]) -> ValidationResult: + if not isinstance(package, dict): + input_hash = content_hash(package) + finding = _finding( + code="schema_validation", + severity="blocker", + path="$", + message="Estimate package must be a JSON object.", + relevant_hash=input_hash, + ) + return ValidationResult("invalid", input_hash, [finding], [finding]) input_hash = estimate_input_hash(package) version = package.get("schema_version") if version != ESTIMATE_PACKAGE_VERSION: @@ -217,6 +228,8 @@ def validate_estimate_package(package: dict[str, Any]) -> ValidationResult: return ValidationResult("invalid", input_hash, [finding], [finding]) findings = _schema_findings(package, input_hash) + if findings: + return ValidationResult("invalid", input_hash, findings, findings) source_ids: dict[str, int] = {} item_ids: dict[str, int] = {} for index, item in enumerate(package.get("items", [])): diff --git a/skills/steel-estimate/SKILL.md b/skills/steel-estimate/SKILL.md new file mode 100644 index 0000000..5656290 --- /dev/null +++ b/skills/steel-estimate/SKILL.md @@ -0,0 +1,80 @@ +--- +name: steel-estimate +description: "Build a deterministic, review-gated steel estimate package from canonical estimate JSON. Use when a complete takeoff-to-nest-to-draft-RFQ workflow is needed with validation, lineage, QA, and isolated run artifacts." +--- + +# Steel Estimate Package + +## Boundary + +This skill produces estimating and draft purchasing artifacts for human review. +It does not send an RFQ, select a vendor, approve substitutions, award work, or +authorize a purchase. The workbook remains `DRAFT — NOT SENT OR AWARDED`. + +## Required inputs + +- Canonical estimate-package JSON at schema version `1.0.0`. +- Explicit prepared and RFQ issue dates. +- A runtime company profile accepted by `steel-rfq`. +- Plate stock compatible with every plate part by material, grade, thickness, + and usable dimensions. + +Use `references/estimate-package-example.json` as a synthetic input example. +The company profile resolution and approval rules are documented by +`../steel-rfq/SKILL.md`. + +## Workflow and gates + +The orchestrator composes the shared estimate validator, deterministic nesting +engine, RFQ compiler, and run publisher. It does not duplicate their +calculations. + +1. Normalize and validate the canonical package. +2. Build the typed BOM projection without converting exclusions or allowances + into vendor quantities. +3. Nest plate parts only within compatible stock groups. +4. Verify placements and retain diagnostic nest artifacts. +5. Compile a draft RFQ only when validation, placement, and company-profile + gates pass. +6. Publish an isolated run manifest and QA report. + +Validation failures, unplaced parts, invalid company data, failed nest +verification, or required rendering dependencies block workbook generation. +Complete bounding-box nests for irregular geometry may produce a workbook, but +the run and workbook remain explicitly review-required. Reference DXF is never +burn-ready DXF. + +## Run + +```bash +python3 scripts/build-estimate-package.py \ + --input \ + --out \ + --prepared-date \ + --issued-date \ + --project-location "Example City, ST" +``` + +Use `--no-render` when only JSON and workbook artifacts are needed. Use +`--no-bake` to defer formula calculation to spreadsheet open. Explicit dates +and unchanged input/configuration yield stable semantic artifacts. + +Exit codes: + +- `0`: package is ready for human RFQ review. +- `2`: a draft exists, but warnings or reference-only geometry require review. +- `3`: blocked; diagnostics are published and no workbook is produced. +- `4`: a required runtime dependency is missing; no workbook is produced. +- `1`: usage, file, or input parsing error. + +See `references/output-contract.md` for artifact and status semantics. + +## Delivery checks + +- Read `run-manifest.json` and `qa-report.json`; do not infer readiness from a + workbook filename. +- Confirm there are no blocker findings or unplaced parts. +- Confirm material, grade, thickness, source IDs, and replacement lineage. +- Surface every warning and approximation to the reviewer. +- Treat all workbook and rendered artifacts according to their manifest + readiness labels. diff --git a/skills/steel-estimate/references/estimate-package-example.json b/skills/steel-estimate/references/estimate-package-example.json new file mode 100644 index 0000000..5540146 --- /dev/null +++ b/skills/steel-estimate/references/estimate-package-example.json @@ -0,0 +1,91 @@ +{ + "schema_version": "1.0.0", + "project": { + "project_id": "SYNTHETIC-EXAMPLE-001", + "name": "Synthetic Estimate Example", + "revision": { + "revision_id": "SYNTHETIC-REV-A", + "source_document": "SYNTHETIC-SOURCE-001" + } + }, + "unit_system": "imperial", + "items": [ + { + "intent": "fabricated_part", + "source_id": "SYNTHETIC-SRC-P1", + "item_id": "item:synthetic-example-p1", + "quantity": 2, + "mark": "P1", + "description": "Synthetic rectangular plate", + "material": "carbon_steel", + "grade": "A36", + "geometry": { + "shape": "rect", + "width": 8, + "height": 6, + "thickness": 0.5, + "holes": [], + "rotatable": true + }, + "source_evidence": [ + { + "source": "SYNTHETIC-SOURCE-001", + "locator": "SYNTHETIC-DETAIL-P1" + } + ] + }, + { + "intent": "purchased_stock", + "source_id": "SYNTHETIC-SRC-S1", + "item_id": "item:synthetic-example-s1", + "quantity": 1, + "description": "Synthetic stock plate purchase", + "material": "carbon_steel", + "grade": "A36", + "specification": "PLATE", + "dimensions": { + "category": "PLATE STOCK", + "width": 24, + "height": 12, + "thickness": 0.5, + "size": "24 x 12 x 0.5", + "replaces_item_ids": [ + "item:synthetic-example-p1" + ] + }, + "source_evidence": [ + { + "source": "SYNTHETIC-SOURCE-001", + "locator": "SYNTHETIC-STOCK-S1" + } + ] + } + ], + "stock": [ + { + "stock_kind": "purchasable", + "inventory_id": "SYNTHETIC-STOCK-A36", + "material": "carbon_steel", + "grade": "A36", + "width": 24, + "height": 12, + "thickness": 0.5, + "quantity": 1, + "status": "available" + } + ], + "commercial_basis": { + "currency": "USD", + "costs": [] + }, + "review": { + "status": "validated", + "findings": [], + "acknowledgements": [] + }, + "lineage": { + "source_type": "synthetic_example", + "source_hash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "configuration_hash": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } +} diff --git a/skills/steel-estimate/references/output-contract.md b/skills/steel-estimate/references/output-contract.md new file mode 100644 index 0000000..d4436ce --- /dev/null +++ b/skills/steel-estimate/references/output-contract.md @@ -0,0 +1,47 @@ +# Estimate package output contract + +Each invocation publishes an immutable directory at `runs//` and +updates `latest-run.json` only after the run is complete. Read the manifest +before using any artifact. + +## Core artifacts + +| Artifact | Purpose | Produced when | +| --- | --- | --- | +| `estimate-package.json` | Deterministically ordered canonical input | Always | +| `normalized-bom.json` | Typed BOM and calculated weight projection | Always | +| `nest-result.json` | Placements, verification, utilization, and unplaced parts | Valid input contains plate parts | +| `rfq-nesting.json` | Versioned nesting lineage for RFQ compilation | A nest was attempted | +| `inventory-consumption.json` | Confirmed on-hand sheets consumed and corresponding RFQ demand reduction | Eligible on-hand inventory was consumed | +| `qa-report.json` | Findings, approximations, gate decisions, and recalculation status | Always | +| `run-manifest.json` | Input/configuration hashes and artifact hashes/readiness | Always | +| `_RFQ_Material_List.xlsx` | Draft RFQ workbook | All blocking gates pass | +| `workbook-semantic.json` | Stable workbook content projection | Workbook exists | + +Rendered PDF, PNG, and DXF files are optional. Reference artifacts are not +fabrication authority. Burn DXF is eligible only for a fully ready run with +verified exact geometry. + +## Outcomes + +- `ready` / `rfq_ready_for_review`: all gates pass; the workbook is still a + draft requiring human review. +- `review_required` / `rfq_draft_review_required`: a workbook exists, but + warnings or reference-only geometry must be resolved or accepted by a human. +- `blocked`: diagnostics exist, but no workbook exists. +- `dependency_missing`: required rendering support is absent and no workbook + exists. + +Unplaced parts use `blocked` with package status `nested_partial`; their nest +diagnostics remain available. A validation failure stops before nesting. + +## Determinism and lineage + +Canonical JSON is sorted before publication. The manifest records the canonical +input hash, effective configuration hash, tool/schema versions, explicit dates, +and SHA-256 for each artifact. Run IDs and filesystem paths are volatile and do +not affect the semantic hash. A changed revision, quantity, profile, date, or +nest setting changes the applicable downstream lineage. + +Never treat warnings, approximations, exclusions, allowances, or source +evidence as presentation-only metadata. They are part of the review contract. diff --git a/skills/steel-estimate/scripts/build-estimate-package.py b/skills/steel-estimate/scripts/build-estimate-package.py new file mode 100755 index 0000000..c2a5645 --- /dev/null +++ b/skills/steel-estimate/scripts/build-estimate-package.py @@ -0,0 +1,818 @@ +#!/usr/bin/env python3 +"""Build one deterministic, review-gated steel estimate package.""" + +from __future__ import annotations + +import argparse +import copy +import importlib.util +import json +import os +import re +import sys +import tempfile +import zipfile +from collections import Counter +from datetime import date +from pathlib import Path +from typing import Any + + +SHARED_ROOT = Path(__file__).resolve().parents[2] / "_shared" +if str(SHARED_ROOT) not in sys.path: + sys.path.insert(0, str(SHARED_ROOT)) +from bootstrap import bootstrap_shared # noqa: E402 + +bootstrap_shared(__file__) +from pi_steel import RunPublisher, canonical_json_bytes, outcome_exit_code, sha256_bytes # noqa: E402 +from pi_steel.contracts import ESTIMATE_PACKAGE_VERSION # noqa: E402 +from pi_steel.validation import ( # noqa: E402 + eligible_on_hand_stock, + validate_estimate_package, +) + + +PIPELINE_VERSION = "1.0.0" + + +def _load_module(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +SKILLS_ROOT = Path(__file__).resolve().parents[2] +nest_engine = _load_module( + "pi_steel_estimate_nest", + SKILLS_ROOT / "steel-nest" / "scripts" / "nest.py", +) +rfq_compiler = _load_module( + "pi_steel_estimate_rfq", + SKILLS_ROOT / "steel-rfq" / "scripts" / "generate-rfq.py", +) + + +class PipelineInputError(ValueError): + pass + + +class StageArgumentParser(argparse.ArgumentParser): + def error(self, message): + self.print_usage(sys.stderr) + self.exit(1, f"{self.prog}: error: {message}\n") + + +def normalized_package(package: dict[str, Any]) -> dict[str, Any]: + """Return a stable canonical ordering without changing supplied facts.""" + value = copy.deepcopy(package) + value["items"] = sorted( + value.get("items", []), + key=lambda item: ( + item.get("item_id", "") if isinstance(item, dict) else "" + ), + ) + value["stock"] = sorted( + value.get("stock", []), + key=lambda stock: ( + stock.get("inventory_id", "") if isinstance(stock, dict) else "", + stock.get("material", "") if isinstance(stock, dict) else "", + stock.get("grade", "") if isinstance(stock, dict) else "", + stock.get("thickness", 0) if isinstance(stock, dict) else 0, + stock.get("width", 0) if isinstance(stock, dict) else 0, + stock.get("height", 0) if isinstance(stock, dict) else 0, + ), + ) + review = value.get("review", {}) + if isinstance(review, dict): + review["findings"] = sorted( + review.get("findings", []), + key=lambda finding: finding.get("finding_id", ""), + ) + review["acknowledgements"] = sorted( + review.get("acknowledgements", []), + key=lambda acknowledgement: ( + acknowledgement.get("finding_id", ""), + acknowledgement.get("timestamp", ""), + acknowledgement.get("actor", ""), + ), + ) + return value + + +def _legacy_hole(hole: dict[str, Any]) -> dict[str, Any]: + if hole["kind"] == "round": + return { + "dia": hole["diameter"], + "x": hole["x"], + "y": hole["y"], + } + return { + "w": hole["width"], + "h": hole["height"], + "x": hole["x"], + "y": hole["y"], + } + + +def nest_job_from_package( + package: dict[str, Any], + *, + eligible_inventory_ids: set[str], + kerf_in: float, + part_gap_in: float, + edge_margin_in: float, + density_lb_in3: float, +) -> dict[str, Any] | None: + plate_items = [ + item + for item in package["items"] + if item["intent"] == "fabricated_part" and item.get("geometry") + ] + if not plate_items: + return None + stock_rows = [] + for index, stock in enumerate(package.get("stock", []), start=1): + if ( + stock["stock_kind"] == "on_hand" + and stock.get("inventory_id") not in eligible_inventory_ids + ): + continue + stock_rows.append( + { + "stock_id": stock.get("inventory_id") + or f"purchasable-stock:{index:04d}", + "name": stock.get("inventory_id") or f"Purchase Stock {index}", + "material": stock["material"], + "grade": stock["grade"], + "width": stock["width"], + "height": stock["height"], + "thickness": stock["thickness"], + "qty": stock["quantity"], + } + ) + parts = [] + for item in plate_items: + geometry = item["geometry"] + part = { + "source_id": item["source_id"], + "item_id": item["item_id"], + "name": item.get("mark") or item["item_id"], + "material": item["material"], + "grade": item["grade"], + "thickness": geometry["thickness"], + "width": geometry["width"], + "height": geometry["height"], + "qty": item["quantity"], + "shape": geometry["shape"], + "rotatable": geometry.get("rotatable", True), + "holes": [_legacy_hole(hole) for hole in geometry.get("holes", [])], + } + if geometry.get("area") is not None: + part["area"] = geometry["area"] + parts.append(part) + return { + "job_name": package["project"].get("name") + or package["project"]["project_id"], + "project_id": package["project"]["project_id"], + "revision_id": package["project"]["revision"]["revision_id"], + "unit_system": package["unit_system"], + "settings": { + "kerf_in": kerf_in, + "part_gap_in": part_gap_in, + "edge_margin_in": edge_margin_in, + "density_lb_in3": density_lb_in3, + }, + "stock": stock_rows, + "parts": parts, + } + + +def build_bom_projection( + package: dict[str, Any], nest_result: dict[str, Any] | None +) -> dict[str, Any]: + items = [] + known_member_weight = 0.0 + for item in package["items"]: + row = { + "source_id": item["source_id"], + "item_id": item["item_id"], + "intent": item["intent"], + "quantity": item["quantity"], + "description": item.get("description", ""), + } + if item.get("mark") is not None: + row["mark"] = item["mark"] + if item.get("grade") is not None: + row["grade"] = item["grade"] + if item.get("designation") is not None: + row["designation"] = item["designation"] + if item.get("geometry") is not None: + row["geometry"] = item["geometry"] + if item.get("allowance_basis") is not None: + row["allowance_basis"] = item["allowance_basis"] + if item.get("reason") is not None: + row["reason"] = item["reason"] + weight = item.get("total_weight_lbs") + if ( + weight is None + and item.get("length_ft") is not None + and item.get("unit_weight_plf") is not None + ): + weight = item["quantity"] * item["length_ft"] * item["unit_weight_plf"] + if ( + weight is not None + and item["intent"] == "fabricated_part" + and item.get("geometry") is None + ): + row["calculated_weight_lbs"] = round(float(weight), 3) + known_member_weight += float(weight) + items.append(row) + + nested_plate_weight = ( + nest_result["total_part_weight_lb"] if nest_result is not None else 0.0 + ) + fabricated_weight = known_member_weight + nested_plate_weight + allowance_weight = 0.0 + allowance_rows = [] + for item in package["items"]: + if item["intent"] != "allowance": + continue + basis = item["allowance_basis"] + if basis["kind"] == "percent" and basis["applies_to"] == "fabricated_weight": + weight = fabricated_weight * basis["value"] / 100 + elif basis["kind"] == "fixed_weight": + weight = basis["value"] + else: + weight = None + allowance_rows.append( + { + "item_id": item["item_id"], + "basis": basis, + "calculated_weight_lbs": ( + None if weight is None else round(weight, 3) + ), + "vendor_quantity": None, + } + ) + if weight is not None: + allowance_weight += weight + complete = not (nest_result and nest_result["unplaced"]) + return { + "schema_version": "1.0.0", + "project_id": package["project"]["project_id"], + "revision_id": package["project"]["revision"]["revision_id"], + "items": items, + "totals": { + "known_member_weight_lbs": round(known_member_weight, 3), + "nested_plate_weight_lbs": round(nested_plate_weight, 3), + "fabricated_weight_lbs": round(fabricated_weight, 3), + "allowance_weight_lbs": round(allowance_weight, 3), + "estimate_weight_lbs": round(fabricated_weight + allowance_weight, 3), + "status": "complete" if complete else "incomplete_unplaced", + }, + "allowances": allowance_rows, + "pricing_status": "not_calculated_without_explicit_cost_basis", + } + + +def _annotated_handoff(nest_result: dict[str, Any] | None): + if nest_result is None: + return None + handoff = copy.deepcopy(nest_result["rfq_nesting"]) + for row in handoff["rows"]: + row["geometry_readiness"] = nest_result["geometry_readiness"] + return handoff + + +def apply_inventory_consumption( + package: dict[str, Any], + rfq_normalized: dict[str, Any], + nest_result: dict[str, Any] | None, + eligible_inventory_ids: set[str], +) -> list[dict[str, Any]]: + """Reduce RFQ demand only for traceable on-hand sheets actually consumed.""" + used = Counter( + report["stock_id"] + for report in (nest_result or {}).get("plate_reports", []) + if report["stock_id"] in eligible_inventory_ids + ) + stock_by_id = { + stock["inventory_id"]: stock + for stock in package.get("stock", []) + if stock.get("inventory_id") in used + } + normalized_by_id = { + item["item_id"]: item for item in rfq_normalized.get("items", []) + } + lineage = [] + for inventory_id in sorted(used): + stock = stock_by_id[inventory_id] + remaining = used[inventory_id] + for item in sorted(package.get("items", []), key=lambda row: row["item_id"]): + if remaining <= 0 or item.get("intent") != "purchased_stock": + continue + dimensions = item.get("dimensions") or {} + if not ( + item.get("material") == stock.get("material") + and item.get("grade") == stock.get("grade") + and dimensions.get("width") == stock.get("width") + and dimensions.get("height") == stock.get("height") + and dimensions.get("thickness") == stock.get("thickness") + ): + continue + satisfied = min(item["quantity"], remaining) + remaining -= satisfied + rfq_item = normalized_by_id.get(item["item_id"]) + if rfq_item is None: + continue + original_quantity = rfq_item["quantity"] + open_quantity = original_quantity - satisfied + if open_quantity: + rfq_item["quantity"] = open_quantity + if rfq_item.get("purchase_weight_lbs") is not None: + rfq_item["purchase_weight_lbs"] = round( + rfq_item["purchase_weight_lbs"] + * open_quantity + / original_quantity, + 3, + ) + else: + rfq_normalized["items"].remove(rfq_item) + lineage.append( + { + "inventory_id": inventory_id, + "purchase_item_id": item["item_id"], + "satisfied_quantity": satisfied, + "remaining_purchase_quantity": open_quantity, + } + ) + return lineage + + +def _profile_hash(profile: dict[str, Any] | None) -> str: + if profile is None: + return sha256_bytes(b"missing-profile") + return sha256_bytes( + canonical_json_bytes( + {key: value for key, value in profile.items() if not key.startswith("_")} + ) + ) + + +def _fixed_xlsx(path: Path, issued_date: str) -> None: + """Normalize volatile XLSX metadata and ZIP timestamps for stable byte hashes.""" + timestamp = f"{issued_date}T00:00:00Z".encode() + with zipfile.ZipFile(path, "r") as source: + entries = { + info.filename: (info, source.read(info.filename)) + for info in source.infolist() + } + core_name = "docProps/core.xml" + if core_name in entries: + original, content = entries[core_name] + content = re.sub( + rb"(]*>).*?()", + lambda match: match.group(1) + timestamp + match.group(2), + content, + ) + entries[core_name] = (original, content) + with tempfile.NamedTemporaryFile( + prefix=path.stem + "-", suffix=".xlsx", dir=path.parent, delete=False + ) as temporary: + temporary_path = Path(temporary.name) + try: + with zipfile.ZipFile( + temporary_path, "w", compression=zipfile.ZIP_DEFLATED + ) as destination: + for name in sorted(entries): + original, content = entries[name] + info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) + info.compress_type = zipfile.ZIP_DEFLATED + info.external_attr = original.external_attr + info.create_system = original.create_system + destination.writestr(info, content) + os.replace(temporary_path, path) + finally: + if temporary_path.exists(): + temporary_path.unlink() + + +def _package_version() -> str: + path = Path(__file__).resolve().parents[3] / "package.json" + try: + return json.loads(path.read_text(encoding="utf-8"))["version"] + except (OSError, KeyError, json.JSONDecodeError): + return "unknown" + + +def _finding(code, severity, path, message): + return { + "code": code, + "severity": severity, + "path": path, + "message": message, + } + + +def _valid_date(value: str) -> bool: + try: + return date.fromisoformat(value).isoformat() == value + except (TypeError, ValueError): + return False + + +def build_pipeline(args) -> tuple[dict[str, Any], Path]: + if not _valid_date(args.prepared_date): + raise PipelineInputError("--prepared-date must be an ISO date (YYYY-MM-DD)") + if not _valid_date(args.issued_date): + raise PipelineInputError("--issued-date must be an ISO date (YYYY-MM-DD)") + input_path = Path(args.input) + package = json.loads(input_path.read_text(encoding="utf-8")) + validation = validate_estimate_package(package) + normalized = ( + normalized_package(package) + if isinstance(package, dict) + else {"invalid_input": package} + ) + eligible_inventory_ids = ( + { + stock["inventory_id"] + for stock in eligible_on_hand_stock(package) + } + if not validation.blockers + else set() + ) + findings = [ + { + "code": finding["code"], + "severity": finding["severity"], + "path": finding["path"], + "message": finding["message"], + } + for finding in validation.active_findings + ] + validation_blocked = bool(validation.blockers) + + try: + profile, profile_source = rfq_compiler.resolve_company_profile(input_path) + profile_findings = rfq_compiler.validate_company_profile( + profile, profile.get("_profile_path") if profile else None + ) + except rfq_compiler.RfqInputError as exc: + profile, profile_source = None, "invalid" + profile_findings = [ + { + "code": "invalid_company_profile", + "severity": "error", + "message": str(exc), + } + ] + findings.extend( + _finding( + finding["code"], + "blocker" if finding["severity"] == "error" else "warning", + "$.company_profile", + finding["message"], + ) + for finding in profile_findings + ) + profile_blocked = bool(profile_findings) + + nest_result = None + nest_job = None + if not validation_blocked: + nest_job = nest_job_from_package( + normalized, + eligible_inventory_ids=eligible_inventory_ids, + kerf_in=args.kerf_in, + part_gap_in=args.part_gap_in, + edge_margin_in=args.edge_margin_in, + density_lb_in3=args.density_lb_in3, + ) + if nest_job is not None: + nest_result = nest_engine.run_job(nest_job) + for nest_finding in nest_result["validation_findings"]: + findings.append( + _finding( + nest_finding["code"], + "blocker" + if nest_finding["severity"] == "error" + else "warning", + nest_finding["path"], + nest_finding["message"], + ) + ) + for verifier_finding in nest_result["verification"]["findings"]: + findings.append( + _finding( + verifier_finding["code"], + "blocker", + verifier_finding["path"], + verifier_finding["message"], + ) + ) + if nest_result["unplaced"]: + findings.append( + _finding( + "unplaced_parts", + "blocker", + "$.nest.unplaced", + ( + f"{sum(row['quantity'] for row in nest_result['unplaced'])} " + "required plate part(s) remain unplaced." + ), + ) + ) + unplaced = bool(nest_result and nest_result["unplaced"]) + nest_blocked = bool( + nest_result + and ( + nest_result["outcome"] == "blocked" + or nest_result["verification"]["status"] != "verified" + ) + ) + reference_only = bool( + nest_result and nest_result["geometry_readiness"] == "reference_only" + ) + approximations = [] + if reference_only: + approximations.append( + { + "code": "BOUNDING_BOX_NESTING", + "message": "Irregular plate geometry is reference-only.", + } + ) + + render_missing = [] + if not args.no_render and nest_result is not None: + render_missing = nest_engine.missing_render_dependencies() + if render_missing: + findings.append( + _finding( + "render_dependency_missing", + "blocker", + "$.render", + "Missing rendering modules: " + ", ".join(render_missing), + ) + ) + + rfq_normalized = None + inventory_consumption = [] + compiler_error = None + if not validation_blocked and not profile_blocked and not nest_blocked and not render_missing: + try: + rfq_normalized = rfq_compiler.normalize_canonical_package(package) + inventory_consumption = apply_inventory_consumption( + package, + rfq_normalized, + nest_result, + eligible_inventory_ids, + ) + if not rfq_normalized["items"]: + raise rfq_compiler.RfqInputError( + "no open vendor-supply items remain after inventory consumption" + ) + except rfq_compiler.RfqInputError as exc: + compiler_error = str(exc) + findings.append( + _finding("rfq_input_blocked", "blocker", "$.rfq", compiler_error) + ) + + blocked = ( + validation_blocked + or profile_blocked + or nest_blocked + or unplaced + or bool(render_missing) + or compiler_error is not None + ) + review_warnings = [ + finding["message"] + for finding in findings + if finding["severity"] == "warning" + ] + if blocked: + outcome = "dependency_missing" if render_missing else "blocked" + package_status = "nested_partial" if unplaced else "draft" + elif reference_only or review_warnings: + outcome = "review_required" + package_status = "rfq_draft_review_required" + else: + outcome = "ready" + package_status = "rfq_ready_for_review" + + handoff = _annotated_handoff(nest_result) + configuration = { + "pipeline_version": PIPELINE_VERSION, + "nest_algorithm_version": nest_engine.NEST_ALGORITHM_VERSION, + "rfq_compiler_version": rfq_compiler.RFQ_COMPILER_VERSION, + "prepared_date": args.prepared_date, + "issued_date": args.issued_date, + "project_location": args.project_location, + "kerf_in": args.kerf_in, + "part_gap_in": args.part_gap_in, + "edge_margin_in": args.edge_margin_in, + "density_lb_in3": args.density_lb_in3, + "render": not args.no_render, + "bake": not args.no_bake, + "profile_hash": _profile_hash(profile), + } + configuration_hash = sha256_bytes(canonical_json_bytes(configuration)) + if validation_blocked: + project = normalized.get("project", {}) + revision = project.get("revision", {}) if isinstance(project, dict) else {} + bom = { + "schema_version": "1.0.0", + "project_id": ( + project.get("project_id") if isinstance(project, dict) else None + ), + "revision_id": ( + revision.get("revision_id") if isinstance(revision, dict) else None + ), + "items": [], + "totals": {"status": "blocked_invalid_input"}, + "allowances": [], + "pricing_status": "not_calculated_without_valid_input", + } + else: + bom = build_bom_projection(normalized, nest_result) + project = normalized.get("project", {}) + project = project if isinstance(project, dict) else {} + revision = project.get("revision", {}) + revision = revision if isinstance(revision, dict) else {} + qa_report = { + "schema_version": "1.0.0", + "stage": "steel-estimate", + "run_outcome": outcome, + "package_status": package_status, + "project_id": project.get("project_id"), + "revision_id": revision.get("revision_id"), + "input_hash": validation.input_hash, + "configuration_hash": configuration_hash, + "profile_source": profile_source, + "findings": findings, + "warnings": review_warnings, + "approximations": approximations, + "nest": ( + None + if nest_result is None + else { + "outcome": nest_result["outcome"], + "geometry_readiness": nest_result["geometry_readiness"], + "unplaced": nest_result["unplaced"], + "verification": nest_result["verification"], + } + ), + "rfq": { + "generated": not blocked, + "document_status": "DRAFT — NOT SENT OR AWARDED", + "recalculation_status": None, + }, + } + + with RunPublisher( + args.out, + stage="steel-estimate", + run_outcome=outcome, + package_status=package_status, + input_hash=validation.input_hash, + configuration_hash=configuration_hash, + schema_versions={ + "run_manifest": "1.0.0", + "estimate_package": ESTIMATE_PACKAGE_VERSION, + "normalized_bom": "1.0.0", + "nest_result": nest_engine.NEST_RESULT_VERSION, + "rfq_nesting": rfq_compiler.NEST_HANDOFF_VERSION, + "rfq_workbook": rfq_compiler.RFQ_COMPILER_VERSION, + }, + tool_versions={ + "pi_steel": _package_version(), + "estimate_pipeline": PIPELINE_VERSION, + "nest_algorithm": nest_engine.NEST_ALGORITHM_VERSION, + "rfq_compiler": rfq_compiler.RFQ_COMPILER_VERSION, + }, + explicit_dates={ + "prepared_date": args.prepared_date, + "issued_date": args.issued_date, + }, + warnings=review_warnings, + approximations=approximations, + run_id=args.run_id, + ) as publisher: + publisher.write_json( + "estimate-package.json", normalized, readiness="draft" + ) + publisher.write_json("normalized-bom.json", bom, readiness="diagnostic") + if nest_result is not None: + publisher.write_json( + "nest-result.json", nest_result, readiness="diagnostic" + ) + publisher.write_json( + "rfq-nesting.json", handoff, readiness="diagnostic" + ) + if inventory_consumption: + publisher.write_json( + "inventory-consumption.json", + inventory_consumption, + readiness="diagnostic", + ) + + if not blocked: + compile_result = rfq_compiler.compile_workbook( + rfq_normalized, + profile, + nest_handoff=handoff, + issued_date=args.issued_date, + project_location=args.project_location, + output_directory=publisher.staging_path, + profile_source=profile_source, + bake=not args.no_bake, + ) + workbook_path = compile_result["workbook_path"] + _fixed_xlsx(workbook_path, args.issued_date) + qa_report["rfq"]["recalculation_status"] = compile_result[ + "recalculation_status" + ] + qa_report["rfq"]["logo_status"] = compile_result["logo_status"] + publisher.register_artifact( + workbook_path.name, + readiness="draft", + media_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", + ) + publisher.write_json( + "workbook-semantic.json", + rfq_compiler.workbook_semantic_projection(workbook_path), + readiness="diagnostic", + ) + + if not args.no_render and nest_result is not None and not render_missing: + nest_engine.render_layout(nest_result, publisher.staging_path) + publisher.register_artifact( + "layout.pdf", + readiness="reference_only", + media_type="application/pdf", + ) + for plate in nest_result["plate_reports"]: + publisher.register_artifact( + f"plate_{plate['index']}.png", + readiness="reference_only", + media_type="image/png", + ) + if nest_result["plate_reports"]: + nest_engine.render_dxf_overview(nest_result, publisher.staging_path) + nest_engine.render_reference_plate_dxfs( + nest_result, publisher.staging_path + ) + publisher.register_artifact( + "reference_nest.dxf", + readiness="reference_only", + media_type="image/vnd.dxf", + ) + for plate in nest_result["plate_reports"]: + publisher.register_artifact( + f"reference_plate_{plate['index']}.dxf", + readiness="reference_only", + media_type="image/vnd.dxf", + ) + if outcome == "ready" and nest_result["burn_dxf_eligible"]: + nest_engine.render_burn_dxfs(nest_result, publisher.staging_path) + for plate in nest_result["plate_reports"]: + publisher.register_artifact( + f"burn_plate_{plate['index']}.dxf", + readiness="geometry_verified", + media_type="image/vnd.dxf", + ) + + publisher.write_qa_report(qa_report) + final_path = publisher.publish() + return qa_report, final_path + + +def main(argv=None) -> int: + parser = StageArgumentParser(description=__doc__) + parser.add_argument("--input", required=True) + parser.add_argument("--out", default="out") + parser.add_argument("--prepared-date", required=True) + parser.add_argument("--issued-date", required=True) + parser.add_argument("--project-location", default="") + parser.add_argument("--kerf-in", type=float, default=0.06) + parser.add_argument("--part-gap-in", type=float, default=0.25) + parser.add_argument("--edge-margin-in", type=float, default=0.5) + parser.add_argument("--density-lb-in3", type=float, default=0.2836) + parser.add_argument("--no-render", action="store_true") + parser.add_argument("--no-bake", action="store_true") + parser.add_argument("--run-id", help=argparse.SUPPRESS) + args = parser.parse_args(argv) + try: + qa_report, final_path = build_pipeline(args) + except (OSError, json.JSONDecodeError, PipelineInputError) as exc: + print(f"Estimate package failed: {exc}", file=sys.stderr) + return 1 + print(f"Published {qa_report['run_outcome']} estimate package: {final_path}") + return outcome_exit_code(qa_report["run_outcome"]) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/steel-rfq/scripts/generate-rfq.py b/skills/steel-rfq/scripts/generate-rfq.py index 2f1c69d..edb3c92 100755 --- a/skills/steel-rfq/scripts/generate-rfq.py +++ b/skills/steel-rfq/scripts/generate-rfq.py @@ -107,19 +107,23 @@ def _fmt(value: Any) -> str: def _category_for(item: dict[str, Any]) -> str: + if item.get("intent") == "hardware": + return "HARDWARE" dimensions = item.get("dimensions") or {} if dimensions.get("category"): return str(dimensions["category"]).upper() designation = str(item.get("designation") or item.get("specification") or "") normalized = designation.upper().replace(" ", "") - if normalized.startswith("W"): + if re.match(r"^W\d+(?:X|×)", normalized): return "W-SHAPES" if normalized.startswith(("PL", "PLATE")) or item.get("geometry"): return "PLATE PARTS" if normalized.startswith(("FB", "FLATBAR")): return "FLAT BAR STOCK" - if item.get("intent") == "hardware": - return "HARDWARE" + if re.match(r"^(?:HSS|PIPE)\d", normalized) or re.match( + r"^(?:C|MC|L)\d+(?:X|×)", normalized + ): + return "LONG PRODUCTS" return "OTHER" diff --git a/tests/fixtures/pipeline/README.md b/tests/fixtures/pipeline/README.md new file mode 100644 index 0000000..85a6fa8 --- /dev/null +++ b/tests/fixtures/pipeline/README.md @@ -0,0 +1,10 @@ +# Synthetic estimate-pipeline fixtures + +Every file here was authored from scratch for public automated tests. Geometry, +weights, identifiers, names, and approval metadata are invented and do not +derive from a production estimate, drawing, nest, RFQ, customer, vendor, or +operating company. + +The profile uses obvious `Example` identity and explicitly synthetic test text. +No prices, rates, payment terms, vendor terms, or purchasing commitments appear +in these fixtures. diff --git a/tests/fixtures/pipeline/synthetic-estimate.json b/tests/fixtures/pipeline/synthetic-estimate.json new file mode 100644 index 0000000..94a7200 --- /dev/null +++ b/tests/fixtures/pipeline/synthetic-estimate.json @@ -0,0 +1,229 @@ +{ + "schema_version": "1.0.0", + "project": { + "project_id": "SYNTHETIC-PIPELINE-001", + "name": "Synthetic Pipeline Project", + "revision": { + "revision_id": "SYNTHETIC-REV-A", + "source_document": "SYNTHETIC-ESTIMATE-001" + } + }, + "unit_system": "imperial", + "items": [ + { + "intent": "fabricated_part", + "source_id": "SYNTHETIC-SRC-W1", + "item_id": "item:synthetic-pipeline-w1", + "quantity": 2, + "mark": "W1", + "description": "Synthetic wide-flange member", + "material": "carbon_steel", + "grade": "A992", + "designation": "W12X26", + "length_ft": 12, + "unit_weight_plf": 26, + "total_weight_lbs": 624, + "source_evidence": [ + { + "source": "SYNTHETIC-DRAWING", + "locator": "SYNTHETIC-DETAIL-W1" + } + ] + }, + { + "intent": "fabricated_part", + "source_id": "SYNTHETIC-SRC-HSS1", + "item_id": "item:synthetic-pipeline-hss1", + "quantity": 1, + "mark": "HSS1", + "description": "Synthetic long product", + "material": "carbon_steel", + "grade": "A500", + "designation": "HSS6X6X3/8", + "length_ft": 10, + "unit_weight_plf": 27.5, + "total_weight_lbs": 275, + "source_evidence": [ + { + "source": "SYNTHETIC-DRAWING", + "locator": "SYNTHETIC-DETAIL-HSS1" + } + ] + }, + { + "intent": "fabricated_part", + "source_id": "SYNTHETIC-SRC-P1", + "item_id": "item:synthetic-pipeline-p1", + "quantity": 2, + "mark": "P1", + "description": "Synthetic A36 plate part", + "material": "carbon_steel", + "grade": "A36", + "geometry": { + "shape": "rect", + "width": 8, + "height": 6, + "thickness": 0.5, + "holes": [ + { + "kind": "round", + "diameter": 1, + "x": 2, + "y": 2 + } + ] + }, + "source_evidence": [ + { + "source": "SYNTHETIC-DRAWING", + "locator": "SYNTHETIC-DETAIL-P1" + } + ] + }, + { + "intent": "fabricated_part", + "source_id": "SYNTHETIC-SRC-P2", + "item_id": "item:synthetic-pipeline-p2", + "quantity": 1, + "mark": "P2", + "description": "Synthetic A572 plate part", + "material": "carbon_steel", + "grade": "A572", + "geometry": { + "shape": "rect", + "width": 7, + "height": 5, + "thickness": 0.375, + "holes": [] + }, + "source_evidence": [ + { + "source": "SYNTHETIC-DRAWING", + "locator": "SYNTHETIC-DETAIL-P2" + } + ] + }, + { + "intent": "purchased_stock", + "source_id": "SYNTHETIC-SRC-S1", + "item_id": "item:synthetic-pipeline-s1", + "quantity": 1, + "mark": "S1", + "description": "Synthetic A36 sheet purchase", + "material": "carbon_steel", + "grade": "A36", + "specification": "PLATE", + "dimensions": { + "category": "PLATE STOCK", + "width": 24, + "height": 12, + "thickness": 0.5, + "size": "24 x 12 x 0.5", + "purchase_weight_lbs": 41, + "replaces_item_ids": ["item:synthetic-pipeline-p1"] + }, + "source_evidence": [ + { + "source": "SYNTHETIC-ESTIMATE-001", + "locator": "SYNTHETIC-STOCK-S1" + } + ] + }, + { + "intent": "purchased_stock", + "source_id": "SYNTHETIC-SRC-S2", + "item_id": "item:synthetic-pipeline-s2", + "quantity": 1, + "mark": "S2", + "description": "Synthetic A572 sheet purchase", + "material": "carbon_steel", + "grade": "A572", + "specification": "PLATE", + "dimensions": { + "category": "PLATE STOCK", + "width": 20, + "height": 10, + "thickness": 0.375, + "size": "20 x 10 x 0.375", + "purchase_weight_lbs": 21, + "replaces_item_ids": ["item:synthetic-pipeline-p2"] + }, + "source_evidence": [ + { + "source": "SYNTHETIC-ESTIMATE-001", + "locator": "SYNTHETIC-STOCK-S2" + } + ] + }, + { + "intent": "allowance", + "source_id": "SYNTHETIC-SRC-A1", + "item_id": "item:synthetic-pipeline-a1", + "quantity": 1, + "description": "Synthetic connection allowance", + "allowance_basis": { + "kind": "percent", + "value": 5, + "applies_to": "fabricated_weight" + }, + "source_evidence": [ + { + "source": "SYNTHETIC-ESTIMATE-001", + "locator": "SYNTHETIC-ALLOWANCE-A1" + } + ] + }, + { + "intent": "exclusion", + "source_id": "SYNTHETIC-SRC-X1", + "item_id": "item:synthetic-pipeline-x1", + "quantity": 1, + "description": "Synthetic excluded scope", + "reason": "Explicit synthetic exclusion", + "source_evidence": [ + { + "source": "SYNTHETIC-ESTIMATE-001", + "locator": "SYNTHETIC-EXCLUSION-X1" + } + ] + } + ], + "stock": [ + { + "stock_kind": "purchasable", + "inventory_id": "SYNTHETIC-STOCK-A36", + "material": "carbon_steel", + "grade": "A36", + "width": 24, + "height": 12, + "thickness": 0.5, + "quantity": 1, + "status": "available" + }, + { + "stock_kind": "purchasable", + "inventory_id": "SYNTHETIC-STOCK-A572", + "material": "carbon_steel", + "grade": "A572", + "width": 20, + "height": 10, + "thickness": 0.375, + "quantity": 1, + "status": "available" + } + ], + "commercial_basis": { + "currency": "USD", + "costs": [] + }, + "review": { + "status": "validated", + "findings": [], + "acknowledgements": [] + }, + "lineage": { + "source_type": "synthetic_test", + "source_hash": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + "configuration_hash": "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } +} diff --git a/tests/fixtures/pipeline/synthetic-profile.json b/tests/fixtures/pipeline/synthetic-profile.json new file mode 100644 index 0000000..1bd87d4 --- /dev/null +++ b/tests/fixtures/pipeline/synthetic-profile.json @@ -0,0 +1,13 @@ +{ + "company_name": "Example Fabricator", + "city_state": "Example City, ST", + "logo": null, + "terms_template": { + "template_id": "SYNTHETIC-PIPELINE-TEXT-001", + "content": "SYNTHETIC TEST TEXT — Human review is required.", + "content_hash": "fc48b45393a03898511da34d6d46f96371633e1f06d02d4ae515f4bc0a92e260", + "approver": "Example Approver", + "approval_date": "2026-07-28", + "status": "approved" + } +} diff --git a/tests/golden/pipeline/ready-artifacts.json b/tests/golden/pipeline/ready-artifacts.json new file mode 100644 index 0000000..0247fa3 --- /dev/null +++ b/tests/golden/pipeline/ready-artifacts.json @@ -0,0 +1,13 @@ +{ + "artifact_paths": [ + "Synthetic_Pipeline_Project_RFQ_Material_List.xlsx", + "estimate-package.json", + "nest-result.json", + "normalized-bom.json", + "qa-report.json", + "rfq-nesting.json", + "workbook-semantic.json" + ], + "run_outcome": "ready", + "package_status": "rfq_ready_for_review" +} diff --git a/tests/test_estimate_pipeline.py b/tests/test_estimate_pipeline.py new file mode 100644 index 0000000..94ae305 --- /dev/null +++ b/tests/test_estimate_pipeline.py @@ -0,0 +1,329 @@ +import hashlib +import json +import os +import subprocess +import sys +from copy import deepcopy +from pathlib import Path + +import openpyxl +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "skills" / "_shared")) +from pi_steel.contracts import estimate_input_hash + +SCRIPT = ( + ROOT + / "skills" + / "steel-estimate" + / "scripts" + / "build-estimate-package.py" +) +FIXTURES = ROOT / "tests" / "fixtures" / "pipeline" +GOLDEN = ROOT / "tests" / "golden" / "pipeline" / "ready-artifacts.json" + + +def load_package(): + return json.loads((FIXTURES / "synthetic-estimate.json").read_text()) + + +def run_pipeline(tmp_path, package, run_id, *, profile=True): + input_path = tmp_path / f"{run_id}-input.json" + input_path.write_text(json.dumps(package), encoding="utf-8") + output = tmp_path / "published" + environment = os.environ.copy() + environment.pop("PYTHONPATH", None) + environment["XDG_CONFIG_HOME"] = str(tmp_path / "empty-config") + if profile: + environment["PI_STEEL_CONFIG"] = str( + FIXTURES / "synthetic-profile.json" + ) + else: + environment.pop("PI_STEEL_CONFIG", None) + completed = subprocess.run( + [ + sys.executable, + SCRIPT, + "--input", + input_path, + "--out", + output, + "--prepared-date", + "2026-07-28", + "--issued-date", + "2026-07-29", + "--project-location", + "Example City, ST", + "--run-id", + run_id, + "--no-render", + "--no-bake", + ], + cwd=tmp_path, + env=environment, + text=True, + capture_output=True, + ) + pointer = json.loads((output / "latest-run.json").read_text()) + return completed, output / pointer["run_directory"] + + +def load_json(path): + return json.loads(Path(path).read_text()) + + +def test_ready_pipeline_matches_artifact_contract_and_preserves_scope(tmp_path): + completed, run_path = run_pipeline( + tmp_path, load_package(), "SYNTHETIC-PIPELINE-READY" + ) + assert completed.returncode == 0, completed.stdout + completed.stderr + manifest = load_json(run_path / "run-manifest.json") + golden = load_json(GOLDEN) + assert manifest["run_outcome"] == golden["run_outcome"] + assert manifest["package_status"] == golden["package_status"] + assert [artifact["path"] for artifact in manifest["artifacts"]] == golden[ + "artifact_paths" + ] + normalized = load_json(run_path / "estimate-package.json") + assert [item["item_id"] for item in normalized["items"]][:2] == [ + "item:synthetic-pipeline-a1", + "item:synthetic-pipeline-hss1", + ] + bom = load_json(run_path / "normalized-bom.json") + assert {row["intent"] for row in bom["items"]} >= { + "fabricated_part", + "purchased_stock", + "allowance", + "exclusion", + } + nest = load_json(run_path / "nest-result.json") + assert { + (group["grade"], group["thickness"]) for group in nest["groups"] + } == {("A36", 0.5), ("A572", 0.375)} + workbook = openpyxl.load_workbook(next(run_path.glob("*.xlsx"))) + assert workbook["RFQ Metadata"]["B2"].value == "DRAFT — NOT SENT OR AWARDED" + + +def test_same_inputs_and_dates_have_stable_semantics_across_isolated_runs(tmp_path): + first, first_path = run_pipeline( + tmp_path, load_package(), "SYNTHETIC-PIPELINE-RUN-A" + ) + second, second_path = run_pipeline( + tmp_path, load_package(), "SYNTHETIC-PIPELINE-RUN-B" + ) + assert first.returncode == second.returncode == 0 + for name in ( + "estimate-package.json", + "normalized-bom.json", + "nest-result.json", + "rfq-nesting.json", + "workbook-semantic.json", + ): + assert (first_path / name).read_bytes() == (second_path / name).read_bytes() + first_manifest = load_json(first_path / "run-manifest.json") + second_manifest = load_json(second_path / "run-manifest.json") + assert first_manifest["semantic_hash"] == second_manifest["semantic_hash"] + assert first_path != second_path + + +def test_revision_or_quantity_change_updates_input_and_downstream_hashes(tmp_path): + base = load_package() + _, first_path = run_pipeline(tmp_path, base, "SYNTHETIC-PIPELINE-BASE") + changed = deepcopy(base) + changed["project"]["revision"]["revision_id"] = "SYNTHETIC-REV-B" + changed["items"][2]["quantity"] = 3 + _, changed_path = run_pipeline( + tmp_path, changed, "SYNTHETIC-PIPELINE-CHANGED" + ) + first_manifest = load_json(first_path / "run-manifest.json") + changed_manifest = load_json(changed_path / "run-manifest.json") + assert first_manifest["input_hash"] != changed_manifest["input_hash"] + assert first_manifest["semantic_hash"] != changed_manifest["semantic_hash"] + first_artifacts = { + artifact["path"]: artifact["sha256"] + for artifact in first_manifest["artifacts"] + } + changed_artifacts = { + artifact["path"]: artifact["sha256"] + for artifact in changed_manifest["artifacts"] + } + assert first_artifacts["nest-result.json"] != changed_artifacts["nest-result.json"] + + +@pytest.mark.parametrize("blocker", ["validation", "unplaced", "profile"]) +def test_blockers_preserve_diagnostics_but_never_publish_workbook(tmp_path, blocker): + package = load_package() + profile = True + if blocker == "validation": + package["items"][0]["quantity"] = 0 + elif blocker == "unplaced": + package["stock"][0]["quantity"] = 0 + else: + profile = False + completed, run_path = run_pipeline( + tmp_path, + package, + f"SYNTHETIC-PIPELINE-BLOCKED-{blocker.upper()}", + profile=profile, + ) + assert completed.returncode == 3, completed.stdout + completed.stderr + assert not list(run_path.glob("*.xlsx")) + manifest = load_json(run_path / "run-manifest.json") + assert manifest["run_outcome"] == "blocked" + assert (run_path / "qa-report.json").exists() + if blocker == "unplaced": + assert (run_path / "nest-result.json").exists() + assert load_json(run_path / "nest-result.json")["unplaced"] + if blocker == "validation": + assert not (run_path / "nest-result.json").exists() + + +def test_complete_irregular_nest_yields_review_required_draft(tmp_path): + package = load_package() + package["items"][2]["geometry"].update(shape="irregular", area=30) + completed, run_path = run_pipeline( + tmp_path, package, "SYNTHETIC-PIPELINE-REFERENCE" + ) + assert completed.returncode == 2, completed.stdout + completed.stderr + manifest = load_json(run_path / "run-manifest.json") + assert manifest["run_outcome"] == "review_required" + assert manifest["package_status"] == "rfq_draft_review_required" + qa = load_json(run_path / "qa-report.json") + assert any( + approximation["code"] == "BOUNDING_BOX_NESTING" + for approximation in qa["approximations"] + ) + workbook = openpyxl.load_workbook(next(run_path.glob("*.xlsx"))) + assert any( + "REFERENCE ONLY" in str(cell.value).upper() + for row in workbook["RFQ Draft"].iter_rows() + for cell in row + if cell.value + ) + + +def test_schema_blocker_with_missing_identity_still_publishes_diagnostics(tmp_path): + package = load_package() + del package["items"][0]["item_id"] + + completed, run_path = run_pipeline( + tmp_path, package, "SYNTHETIC-PIPELINE-MISSING-IDENTITY" + ) + + assert completed.returncode == 3, completed.stdout + completed.stderr + assert not list(run_path.glob("*.xlsx")) + qa = load_json(run_path / "qa-report.json") + assert qa["run_outcome"] == "blocked" + assert any(finding["code"] == "schema_validation" for finding in qa["findings"]) + assert load_json(run_path / "normalized-bom.json")["totals"]["status"] == ( + "blocked_invalid_input" + ) + + +def test_plate_source_weight_is_not_double_counted_with_nested_weight(tmp_path): + package = load_package() + plate = next(item for item in package["items"] if item.get("geometry")) + plate["total_weight_lbs"] = 999 + + completed, run_path = run_pipeline( + tmp_path, package, "SYNTHETIC-PIPELINE-PLATE-WEIGHT" + ) + + assert completed.returncode == 0, completed.stdout + completed.stderr + bom = load_json(run_path / "normalized-bom.json") + assert bom["totals"]["known_member_weight_lbs"] == 899 + assert bom["totals"]["fabricated_weight_lbs"] == ( + bom["totals"]["known_member_weight_lbs"] + + bom["totals"]["nested_plate_weight_lbs"] + ) + + +def test_blocked_run_replaces_latest_pointer_without_reusing_ready_artifacts( + tmp_path, +): + ready, ready_path = run_pipeline( + tmp_path, load_package(), "SYNTHETIC-PIPELINE-READY-FIRST" + ) + assert ready.returncode == 0 + assert list(ready_path.glob("*.xlsx")) + + blocked_package = load_package() + blocked_package["stock"][0]["quantity"] = 0 + blocked, blocked_path = run_pipeline( + tmp_path, blocked_package, "SYNTHETIC-PIPELINE-BLOCKED-LATEST" + ) + + assert blocked.returncode == 3 + assert blocked_path != ready_path + assert not list(blocked_path.glob("*.xlsx")) + pointer = load_json(tmp_path / "published" / "latest-run.json") + assert tmp_path / "published" / pointer["run_directory"] == blocked_path + + +def test_confirmed_on_hand_stock_is_consumed_without_duplicate_rfq_demand(tmp_path): + package = load_package() + purchase_ids = { + item["item_id"] + for item in package["items"] + if item["intent"] == "purchased_stock" + } + for stock in package["stock"]: + stock.update( + stock_kind="on_hand", + measured_at="2026-07-28", + source="SYNTHETIC-INVENTORY-COUNT", + ) + confirmation_hash = estimate_input_hash(package) + for stock in package["stock"]: + stock["reviewer_confirmation"] = { + "actor": "Synthetic Reviewer", + "timestamp": "2026-07-28T12:00:00Z", + "estimate_hash": confirmation_hash, + } + + completed, run_path = run_pipeline( + tmp_path, package, "SYNTHETIC-PIPELINE-ON-HAND" + ) + + assert completed.returncode == 0, completed.stdout + completed.stderr + consumption = load_json(run_path / "inventory-consumption.json") + assert {row["purchase_item_id"] for row in consumption} == purchase_ids + assert all(row["remaining_purchase_quantity"] == 0 for row in consumption) + semantic = load_json(run_path / "workbook-semantic.json") + workbook_text = json.dumps(semantic) + assert not any(item_id in workbook_text for item_id in purchase_ids) + + +def test_wrong_container_type_still_publishes_blocked_diagnostics(tmp_path): + package = load_package() + package["project"] = [] + + completed, run_path = run_pipeline( + tmp_path, package, "SYNTHETIC-PIPELINE-WRONG-CONTAINER" + ) + + assert completed.returncode == 3, completed.stdout + completed.stderr + assert not list(run_path.glob("*.xlsx")) + qa = load_json(run_path / "qa-report.json") + assert qa["run_outcome"] == "blocked" + assert qa["project_id"] is None + assert any(finding["path"] == "$.project" for finding in qa["findings"]) + + +def test_no_vendor_supply_items_publish_blocked_diagnostics(tmp_path): + package = load_package() + package["items"] = [ + item for item in package["items"] if item["intent"] == "exclusion" + ] + package["stock"] = [] + + completed, run_path = run_pipeline( + tmp_path, package, "SYNTHETIC-PIPELINE-NO-SUPPLY" + ) + + assert completed.returncode == 3, completed.stdout + completed.stderr + assert not list(run_path.glob("*.xlsx")) + qa = load_json(run_path / "qa-report.json") + assert any(finding["code"] == "rfq_input_blocked" for finding in qa["findings"]) diff --git a/tests/test_installed_scripts.py b/tests/test_installed_scripts.py index 3e1e652..4475d38 100644 --- a/tests/test_installed_scripts.py +++ b/tests/test_installed_scripts.py @@ -30,6 +30,10 @@ def test_packed_npm_artifact_contains_runtime_and_runs_doctor(tmp_path): "package/skills/_shared/pi_steel/__init__.py", "package/skills/_shared/pi_steel/run_manifest.py", "package/skills/_shared/schemas/run-manifest.schema.json", + "package/skills/steel-estimate/SKILL.md", + "package/skills/steel-estimate/scripts/build-estimate-package.py", + "package/skills/steel-estimate/references/estimate-package-example.json", + "package/skills/steel-estimate/references/output-contract.md", "package/skills/steel-rfq/scripts/generate-rfq.py", "package/skills/steel-rfq/references/rfq-input.md", "package/pyproject.toml", @@ -56,3 +60,41 @@ def test_packed_npm_artifact_contains_runtime_and_runs_doctor(tmp_path): assert doctor.returncode == 0, doctor.stdout + doctor.stderr assert json.loads(doctor.stdout)["run_outcome"] == "ready" + + environment["PI_STEEL_CONFIG"] = str( + ROOT / "tests" / "fixtures" / "pipeline" / "synthetic-profile.json" + ) + pipeline_output = tmp_path / "installed-output" + pipeline = subprocess.run( + [ + sys.executable, + installed_root + / "skills" + / "steel-estimate" + / "scripts" + / "build-estimate-package.py", + "--input", + ROOT / "tests" / "fixtures" / "pipeline" / "synthetic-estimate.json", + "--out", + pipeline_output, + "--prepared-date", + "2026-07-28", + "--issued-date", + "2026-07-29", + "--project-location", + "Example City, ST", + "--no-render", + "--no-bake", + ], + cwd=tmp_path, + env=environment, + capture_output=True, + text=True, + ) + + assert pipeline.returncode == 0, pipeline.stdout + pipeline.stderr + pointer = json.loads((pipeline_output / "latest-run.json").read_text()) + run_path = pipeline_output / pointer["run_directory"] + manifest = json.loads((run_path / "run-manifest.json").read_text()) + assert manifest["run_outcome"] == "ready" + assert list(run_path.glob("*.xlsx")) diff --git a/tests/test_rfq_generator.py b/tests/test_rfq_generator.py index e49b41d..3b80435 100644 --- a/tests/test_rfq_generator.py +++ b/tests/test_rfq_generator.py @@ -23,6 +23,24 @@ def load(name): return json.loads((FIXTURES / name).read_text()) +@pytest.mark.parametrize( + "designation", ["HSS6X6X3/8", "C10X20", "MC12X31", "L4X4X1/2", "PIPE4"] +) +def test_long_product_designations_share_the_long_products_category(designation): + assert rfq._category_for({"designation": designation}) == "LONG PRODUCTS" + + +@pytest.mark.parametrize( + ("item", "category"), + [ + ({"specification": "CAP PLATE"}, "OTHER"), + ({"intent": "hardware", "designation": "LOCK WASHER"}, "HARDWARE"), + ], +) +def test_category_prefixes_do_not_capture_plate_words_or_hardware(item, category): + assert rfq._category_for(item) == category + + def test_canonical_normalization_uses_typed_scope_and_explicit_replacements_only(): package = load("estimate-package.json") normalized = rfq.normalize_canonical_package(package) From ba659bc86a55353ee9d1cb2fae1f750fd09ae8de Mon Sep 17 00:00:00 2001 From: Victor Garcia Date: Tue, 28 Jul 2026 13:01:28 -0600 Subject: [PATCH 10/15] ci(release): enforce privacy and provenance gates --- .github/workflows/ci.yml | 47 +++++++++++ DATA_PROVENANCE.json | 19 +++++ DATA_PROVENANCE.md | 4 + README.md | 27 +++++- package.json | 14 +++- pyproject.toml | 3 + requirements-dev.txt | 5 +- requirements-render.txt | 4 + scripts/check-data-provenance.py | 97 +++++++++++++++++++++ tests/test_data_provenance.py | 40 +++++++++ tests/test_full_render_smoke.py | 140 +++++++++++++++++++++++++++++++ tests/test_installed_scripts.py | 15 ++++ tests/test_package_contents.py | 60 +++++++++++++ 13 files changed, 466 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/ci.yml create mode 100644 DATA_PROVENANCE.json create mode 100644 requirements-render.txt create mode 100644 scripts/check-data-provenance.py create mode 100644 tests/test_data_provenance.py create mode 100644 tests/test_full_render_smoke.py create mode 100644 tests/test_package_contents.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..33e8c77 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,47 @@ +name: CI + +on: + pull_request: + push: + branches: [main] + +permissions: + contents: read + +jobs: + base: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.11", "3.13"] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: pip + - uses: actions/setup-node@v4 + with: + node-version: "20" + - run: python -m pip install -r requirements.txt -r requirements-dev.txt + - run: npm test + - run: npm run privacy:check + - run: npm run provenance:check + - run: npm run pack:check + + full-render: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + - uses: actions/setup-node@v4 + with: + node-version: "20" + - run: sudo apt-get update && sudo apt-get install -y jq libreoffice poppler-utils + - run: python -m pip install -r requirements.txt -r requirements-dev.txt -r requirements-render.txt + - run: npm run test:full + env: + PI_STEEL_REQUIRE_FULL_RENDER: "1" diff --git a/DATA_PROVENANCE.json b/DATA_PROVENANCE.json new file mode 100644 index 0000000..fc1b57b --- /dev/null +++ b/DATA_PROVENANCE.json @@ -0,0 +1,19 @@ +{ + "schema_version": "1.0.0", + "datasets": [ + { + "dataset_id": "aisc-shapes-v16-transformed", + "shipped_file": "skills/steel-takeoff/assets/aisc-shapes-database.json", + "claimed_edition": "AISC Shapes Database v16.0", + "rows": 477, + "sha256": "5a7c975c4c290c34df6f7df3b4d0d0d13a00ef7c2b1f45097a49a78d245dcc91", + "required_fields": [ + "type", + "designation", + "weight_per_ft" + ], + "redistribution_permission": "unverified", + "release_readiness": "blocked" + } + ] +} diff --git a/DATA_PROVENANCE.md b/DATA_PROVENANCE.md index 865a452..a3670ba 100644 --- a/DATA_PROVENANCE.md +++ b/DATA_PROVENANCE.md @@ -5,6 +5,10 @@ shipped by pi-steel. The repository's MIT license covers StructuPath-authored code and documentation; it must not be interpreted as granting rights in third-party data. +[`DATA_PROVENANCE.json`](DATA_PROVENANCE.json) is the machine-readable release +record checked against the shipped bytes. This document provides the supporting +human-readable evidence and decision rationale. + ## AISC shapes database | Field | Recorded value | diff --git a/README.md b/README.md index ce1f664..04d51fc 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,9 @@ Requires `ezdxf`, `matplotlib`, `numpy` (`pip install ezdxf matplotlib numpy`). Turns a steel estimate/takeoff spreadsheet into a standardized vendor RFQ (.xlsx): materials grouped the way vendors stock them (W-shapes / plate / flat bar), yellow fill-in pricing columns, nesting/drop reference, and terms & conditions — branded with **your** company profile. When `steel-nest` has run for the job, its cutting plan flows straight into the RFQ's nesting table. -The three skills chain into a full estimating pipeline: **takeoff → nest → RFQ**. +The `steel-estimate` orchestrator chains the skills into a review-gated estimating +pipeline: **takeoff → nest → draft RFQ**. It publishes immutable run directories, +QA findings, lineage, and readiness labels; blocked runs never contain a workbook. One-time setup: copy `skills/steel-rfq/assets/company-profile.example.json` to the ignored path `.pi-steel/company-profile.json` in your project and enter approved @@ -61,7 +63,7 @@ Keep company profiles, customer files, vendor information, live pricing, and gen artifacts outside this repository. Public examples are synthetic and must follow [`PUBLIC_DATA_POLICY.md`](PUBLIC_DATA_POLICY.md). -> "Send this takeoff out for pricing" +> "Prepare a draft RFQ from this takeoff" > "Generate an RFQ from this estimate" ## Requirements @@ -69,9 +71,28 @@ artifacts outside this repository. Public examples are synthetic and must follow - `jq` and `python3` (with `pandas` + `openpyxl` for RFQ generation) - macOS or Linux +## Development and release checks + +```bash +npm test # base, no-render suite +npm run test:full # optional PDF/PNG/DXF/LibreOffice smoke tests +npm run privacy:check # public-repository data guard +npm run pack:check # npm contents plus unpacked-runtime smoke test +npm run provenance:check # shape-data integrity and recorded decision +npm run release:check # complete release gate +``` + +`release:check` is intentionally blocked while redistribution permission for the +checked-in transformed AISC shape dataset remains unverified. See +[`DATA_PROVENANCE.md`](DATA_PROVENANCE.md). Do not publish a new package release +by bypassing that gate. + ## License -MIT. AISC shape data derived from the publicly available AISC Shapes Database v16.0. +StructuPath-authored code and documentation are MIT licensed. That license does +not grant rights in third-party data. The checked-in AISC-derived shape data has +a separate, currently blocked redistribution decision documented in +[`DATA_PROVENANCE.md`](DATA_PROVENANCE.md). --- diff --git a/package.json b/package.json index 90e4a01..c10de21 100644 --- a/package.json +++ b/package.json @@ -32,9 +32,16 @@ }, "scripts": { "doctor": "python3 scripts/doctor.py", - "test": "python3 -m pytest", + "test": "python3 -m pytest -m 'not full_render'", + "test:contracts": "python3 -m pytest tests/test_contracts.py tests/test_run_manifests.py tests/test_rfq_workbook_contract.py", + "test:full": "python3 -m pytest -m full_render", "test:runtime": "python3 -m pytest tests/test_runtime_bootstrap.py tests/test_run_manifests.py tests/test_installed_scripts.py", - "pack:dry-run": "npm pack --dry-run" + "privacy:check": "python3 scripts/check-public-data.py", + "provenance:check": "python3 scripts/check-data-provenance.py", + "pack:check": "python3 -m pytest tests/test_package_contents.py tests/test_installed_scripts.py", + "pack:dry-run": "npm pack --dry-run", + "release:check": "npm test && npm run privacy:check && npm run pack:check && python3 scripts/check-data-provenance.py --release", + "prepublishOnly": "npm run release:check" }, "pi": { "skills": [ @@ -47,11 +54,14 @@ "!skills/**/*.pyc", "!skills/**/company-profile.json", "scripts/doctor.py", + "scripts/check-data-provenance.py", "pyproject.toml", "requirements.txt", "requirements-tested.txt", "requirements-dev.txt", + "requirements-render.txt", "DATA_PROVENANCE.md", + "DATA_PROVENANCE.json", "README.md", "PUBLIC_DATA_POLICY.md", "LICENSE" diff --git a/pyproject.toml b/pyproject.toml index 7b69f59..3df13c1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,3 +22,6 @@ test = [ [tool.pytest.ini_options] addopts = "-ra" testpaths = ["tests"] +markers = [ + "full_render: requires optional rendering dependencies and LibreOffice", +] diff --git a/requirements-dev.txt b/requirements-dev.txt index 229facc..9fa1b91 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,4 @@ -r requirements.txt -# Test and optional rendering capabilities exercised by the development suite. +# Base test tier. Install requirements-render.txt for optional rendering. pytest>=8.3,<10 -ezdxf>=1.3,<2 -matplotlib>=3.9,<4 -numpy>=2.1,<3 diff --git a/requirements-render.txt b/requirements-render.txt new file mode 100644 index 0000000..32dcc2c --- /dev/null +++ b/requirements-render.txt @@ -0,0 +1,4 @@ +# Optional PDF, PNG, and DXF rendering tier. +ezdxf>=1.3,<2 +matplotlib>=3.9,<4 +numpy>=2.1,<3 diff --git a/scripts/check-data-provenance.py b/scripts/check-data-provenance.py new file mode 100644 index 0000000..cbd39dd --- /dev/null +++ b/scripts/check-data-provenance.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Verify shipped shape data and enforce its public-release decision.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +PROVENANCE_PATH = ROOT / "DATA_PROVENANCE.md" +PROVENANCE_RECORD_PATH = ROOT / "DATA_PROVENANCE.json" + + +def audit() -> dict: + errors: list[str] = [] + record = json.loads(PROVENANCE_RECORD_PATH.read_text(encoding="utf-8")) + datasets = record.get("datasets", []) + if len(datasets) != 1: + errors.append("expected exactly one declared shipped dataset") + dataset = {} + else: + dataset = datasets[0] + shapes_path = ROOT / dataset.get("shipped_file", "") + raw = shapes_path.read_bytes() + checksum = hashlib.sha256(raw).hexdigest() + if checksum != dataset.get("sha256"): + errors.append("shape data checksum differs from DATA_PROVENANCE.json") + + rows = json.loads(raw) + expected_rows = dataset.get("rows") + if not isinstance(rows, list) or len(rows) != expected_rows: + errors.append(f"expected {expected_rows} shape rows") + rows = rows if isinstance(rows, list) else [] + + required_fields = set(dataset.get("required_fields", [])) + designations: list[str] = [] + for index, row in enumerate(rows): + if not isinstance(row, dict): + errors.append(f"shape row {index} is not an object") + continue + missing = required_fields - row.keys() + if missing: + errors.append(f"shape row {index} is missing {sorted(missing)}") + designation = row.get("designation") + if isinstance(designation, str): + designations.append(designation) + + if len(designations) != len(set(designations)): + errors.append("shape designations are not unique") + + provenance = PROVENANCE_PATH.read_text(encoding="utf-8") + permission = dataset.get("redistribution_permission") + release_readiness = dataset.get("release_readiness") + release_blocked = permission == "unverified" and release_readiness == "blocked" + documented_values = ( + dataset.get("claimed_edition"), + str(expected_rows), + dataset.get("sha256"), + "Redistribution permission | Unverified", + ) + if not all(value and value in provenance for value in documented_values): + errors.append("DATA_PROVENANCE.md differs from the machine-readable record") + + return { + "schema_version": "1.0.0", + "integrity": "passed" if not errors else "failed", + "claimed_edition": dataset.get("claimed_edition"), + "release_readiness": release_readiness, + "redistribution_permission": permission, + "shape_rows": len(rows), + "sha256": checksum, + "errors": errors, + } + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--release", + action="store_true", + help="fail while redistribution permission is unresolved", + ) + args = parser.parse_args(argv) + report = audit() + print(json.dumps(report, indent=2, sort_keys=True)) + if report["errors"]: + return 1 + if args.release and report["release_readiness"] != "ready": + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_data_provenance.py b/tests/test_data_provenance.py new file mode 100644 index 0000000..fdcd6a7 --- /dev/null +++ b/tests/test_data_provenance.py @@ -0,0 +1,40 @@ +import importlib.util +import subprocess +import sys +import unittest +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "check-data-provenance.py" + + +def load_script(): + spec = importlib.util.spec_from_file_location("check_data_provenance", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +class DataProvenanceTests(unittest.TestCase): + def test_shape_data_matches_recorded_integrity_contract(self): + report = load_script().audit() + + self.assertEqual(report["integrity"], "passed", report["errors"]) + self.assertEqual(report["shape_rows"], 477) + self.assertEqual(report["release_readiness"], "blocked") + + def test_release_check_blocks_unverified_redistribution(self): + result = subprocess.run( + [sys.executable, SCRIPT, "--release"], + cwd=ROOT, + capture_output=True, + text=True, + ) + + self.assertEqual(result.returncode, 2, result.stdout + result.stderr) + self.assertIn('"redistribution_permission": "unverified"', result.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_full_render_smoke.py b/tests/test_full_render_smoke.py new file mode 100644 index 0000000..8d6ca4d --- /dev/null +++ b/tests/test_full_render_smoke.py @@ -0,0 +1,140 @@ +import importlib.util +import json +import os +import shutil +import subprocess +import sys +from copy import deepcopy +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "skills" / "steel-estimate" / "scripts" / "build-estimate-package.py" +FIXTURES = ROOT / "tests" / "fixtures" / "pipeline" + + +pytestmark = pytest.mark.full_render + + +def require_full_environment(): + missing = [ + module + for module in ("ezdxf", "matplotlib", "numpy") + if importlib.util.find_spec(module) is None + ] + office = shutil.which("soffice") or shutil.which("libreoffice") + pdf_text = shutil.which("pdftotext") + unavailable = missing + ([] if office else ["LibreOffice"]) + ( + [] if pdf_text else ["pdftotext"] + ) + if unavailable: + message = "full render requires " + ", ".join(unavailable) + if os.environ.get("PI_STEEL_REQUIRE_FULL_RENDER") == "1": + pytest.fail(message) + pytest.skip(message) + return office + + +def run_rendered(tmp_path, package, run_id): + input_path = tmp_path / f"{run_id}.json" + input_path.write_text(json.dumps(package), encoding="utf-8") + output = tmp_path / "published" + environment = os.environ.copy() + environment["PI_STEEL_CONFIG"] = str(FIXTURES / "synthetic-profile.json") + completed = subprocess.run( + [ + sys.executable, + SCRIPT, + "--input", + input_path, + "--out", + output, + "--prepared-date", + "2026-07-28", + "--issued-date", + "2026-07-29", + "--project-location", + "Example City, ST", + "--run-id", + run_id, + ], + cwd=tmp_path, + env=environment, + capture_output=True, + text=True, + ) + pointer = json.loads((output / "latest-run.json").read_text()) + return completed, output / pointer["run_directory"] + + +def test_ready_package_renders_reference_and_verified_outputs(tmp_path): + office = require_full_environment() + package = json.loads((FIXTURES / "synthetic-estimate.json").read_text()) + completed, run_path = run_rendered( + tmp_path, package, "SYNTHETIC-FULL-RENDER-READY" + ) + assert completed.returncode == 0, completed.stdout + completed.stderr + assert (run_path / "layout.pdf").read_bytes().startswith(b"%PDF") + assert list(run_path.glob("plate_*.png")) + assert list(run_path.glob("reference_plate_*.dxf")) + assert list(run_path.glob("burn_plate_*.dxf")) + qa = json.loads((run_path / "qa-report.json").read_text()) + assert qa["rfq"]["recalculation_status"] == "baked_via_libreoffice" + + workbook = next(run_path.glob("*.xlsx")) + pdf_output = tmp_path / "workbook-pdf" + pdf_output.mkdir() + converted = subprocess.run( + [ + office, + "--headless", + "--convert-to", + "pdf", + "--outdir", + pdf_output, + workbook, + ], + cwd=tmp_path, + capture_output=True, + text=True, + ) + assert converted.returncode == 0, converted.stdout + converted.stderr + workbook_pdf = pdf_output / f"{workbook.stem}.pdf" + assert workbook_pdf.read_bytes().startswith(b"%PDF") + assert workbook_pdf.stat().st_size > 1_000 + text_output = tmp_path / "workbook.txt" + extracted = subprocess.run( + ["pdftotext", "-layout", workbook_pdf, text_output], + capture_output=True, + text=True, + ) + assert extracted.returncode == 0, extracted.stdout + extracted.stderr + rendered_text = text_output.read_text() + for expected in ( + "DRAFT", + "Synthetic Pipeline Project", + "RFQ", + "Response Requested By", + ): + assert expected in rendered_text + + +def test_irregular_render_never_publishes_burn_authority(tmp_path): + require_full_environment() + package = json.loads((FIXTURES / "synthetic-estimate.json").read_text()) + package = deepcopy(package) + package["items"][2]["geometry"].update(shape="irregular", area=30) + completed, run_path = run_rendered( + tmp_path, package, "SYNTHETIC-FULL-RENDER-REFERENCE" + ) + assert completed.returncode == 2, completed.stdout + completed.stderr + assert list(run_path.glob("reference_plate_*.dxf")) + assert not list(run_path.glob("burn_plate_*.dxf")) + manifest = json.loads((run_path / "run-manifest.json").read_text()) + assert manifest["run_outcome"] == "review_required" + assert not any( + artifact["readiness"] == "geometry_verified" + for artifact in manifest["artifacts"] + ) diff --git a/tests/test_installed_scripts.py b/tests/test_installed_scripts.py index 4475d38..638e5da 100644 --- a/tests/test_installed_scripts.py +++ b/tests/test_installed_scripts.py @@ -40,7 +40,9 @@ def test_packed_npm_artifact_contains_runtime_and_runs_doctor(tmp_path): "package/requirements.txt", "package/requirements-tested.txt", "package/requirements-dev.txt", + "package/requirements-render.txt", "package/DATA_PROVENANCE.md", + "package/DATA_PROVENANCE.json", } assert expected <= members assert not any(name.startswith("package/tests/") for name in members) @@ -61,6 +63,19 @@ def test_packed_npm_artifact_contains_runtime_and_runs_doctor(tmp_path): assert doctor.returncode == 0, doctor.stdout + doctor.stderr assert json.loads(doctor.stdout)["run_outcome"] == "ready" + provenance = subprocess.run( + [ + sys.executable, + installed_root / "scripts" / "check-data-provenance.py", + ], + cwd=tmp_path, + env=environment, + capture_output=True, + text=True, + ) + assert provenance.returncode == 0, provenance.stdout + provenance.stderr + assert json.loads(provenance.stdout)["release_readiness"] == "blocked" + environment["PI_STEEL_CONFIG"] = str( ROOT / "tests" / "fixtures" / "pipeline" / "synthetic-profile.json" ) diff --git a/tests/test_package_contents.py b/tests/test_package_contents.py new file mode 100644 index 0000000..7f1d28a --- /dev/null +++ b/tests/test_package_contents.py @@ -0,0 +1,60 @@ +import json +import subprocess +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] + + +def test_npm_publish_lifecycle_cannot_bypass_release_gate(): + package = json.loads((ROOT / "package.json").read_text()) + + assert package["scripts"]["prepublishOnly"] == "npm run release:check" + assert "check-data-provenance.py --release" in package["scripts"]["release:check"] + + +def test_npm_dry_run_contains_runtime_contract_and_excludes_private_artifacts(): + result = subprocess.run( + ["npm", "pack", "--dry-run", "--json"], + cwd=ROOT, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + files = {entry["path"] for entry in json.loads(result.stdout)[0]["files"]} + + required = { + "DATA_PROVENANCE.md", + "DATA_PROVENANCE.json", + "PUBLIC_DATA_POLICY.md", + "scripts/check-data-provenance.py", + "scripts/doctor.py", + "skills/_shared/schemas/estimate-package.schema.json", + "skills/_shared/schemas/nest-result.schema.json", + "skills/_shared/schemas/run-manifest.schema.json", + "skills/steel-estimate/SKILL.md", + "skills/steel-estimate/scripts/build-estimate-package.py", + "skills/steel-nest/scripts/nest.py", + "skills/steel-rfq/scripts/generate-rfq.py", + "skills/steel-takeoff/assets/aisc-shapes-database.json", + } + assert required <= files + + forbidden_parts = { + ".pi-steel", + "__pycache__", + "customer-data", + "local-data", + "outputs", + "private", + "tests", + "vendor-data", + } + assert not any( + forbidden_parts.intersection(Path(name).parts) for name in files + ) + assert not any( + name.endswith((".pyc", ".xlsx", ".xls", ".pdf", ".dxf", ".png")) + for name in files + ) + assert not any(Path(name).name == "company-profile.json" for name in files) From abed0231f19ea9400bccd5eef4e530656ca8e62c Mon Sep 17 00:00:00 2001 From: Victor Garcia Date: Tue, 28 Jul 2026 13:06:23 -0600 Subject: [PATCH 11/15] refactor(runtime): simplify shared stage code --- .github/workflows/ci.yml | 1 + package.json | 1 + pyproject.toml | 3 + requirements-dev.txt | 1 + requirements-tested.txt | 1 + scripts/check-data-provenance.py | 1 - scripts/check-public-data.py | 6 +- skills/_shared/pi_steel/__init__.py | 3 + skills/_shared/pi_steel/cli.py | 24 +++++++ skills/_shared/pi_steel/validation.py | 32 ++++++---- .../scripts/build-estimate-package.py | 52 ++++++--------- skills/steel-nest/scripts/nest.py | 64 ++++++++----------- skills/steel-rfq/scripts/generate-rfq.py | 44 ++++++------- tests/test_estimate_pipeline.py | 1 - 14 files changed, 126 insertions(+), 108 deletions(-) create mode 100644 skills/_shared/pi_steel/cli.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 33e8c77..706f981 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -25,6 +25,7 @@ jobs: node-version: "20" - run: python -m pip install -r requirements.txt -r requirements-dev.txt - run: npm test + - run: npm run lint - run: npm run privacy:check - run: npm run provenance:check - run: npm run pack:check diff --git a/package.json b/package.json index c10de21..7c1b90a 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ }, "scripts": { "doctor": "python3 scripts/doctor.py", + "lint": "ruff check skills scripts tests", "test": "python3 -m pytest -m 'not full_render'", "test:contracts": "python3 -m pytest tests/test_contracts.py tests/test_run_manifests.py tests/test_rfq_workbook_contract.py", "test:full": "python3 -m pytest -m full_render", diff --git a/pyproject.toml b/pyproject.toml index 3df13c1..e04b446 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,3 +25,6 @@ testpaths = ["tests"] markers = [ "full_render: requires optional rendering dependencies and LibreOffice", ] + +[tool.ruff.lint.per-file-ignores] +"tests/*.py" = ["E402"] diff --git a/requirements-dev.txt b/requirements-dev.txt index 9fa1b91..d52a49e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,3 +2,4 @@ # Base test tier. Install requirements-render.txt for optional rendering. pytest>=8.3,<10 +ruff>=0.11,<1 diff --git a/requirements-tested.txt b/requirements-tested.txt index f7ff4d3..7465c86 100644 --- a/requirements-tested.txt +++ b/requirements-tested.txt @@ -3,6 +3,7 @@ jsonschema==4.26.0 openpyxl==3.1.5 pandas==2.3.3 pytest==9.0.2 +ruff==0.11.13 ezdxf==1.4.4 matplotlib==3.10.9 numpy==2.4.1 diff --git a/scripts/check-data-provenance.py b/scripts/check-data-provenance.py index cbd39dd..8bd3e7d 100644 --- a/scripts/check-data-provenance.py +++ b/scripts/check-data-provenance.py @@ -54,7 +54,6 @@ def audit() -> dict: provenance = PROVENANCE_PATH.read_text(encoding="utf-8") permission = dataset.get("redistribution_permission") release_readiness = dataset.get("release_readiness") - release_blocked = permission == "unverified" and release_readiness == "blocked" documented_values = ( dataset.get("claimed_edition"), str(expected_rows), diff --git a/scripts/check-public-data.py b/scripts/check-public-data.py index 00267eb..855a3af 100644 --- a/scripts/check-public-data.py +++ b/scripts/check-public-data.py @@ -6,6 +6,7 @@ import re import subprocess import sys +from bisect import bisect_left from pathlib import Path @@ -94,9 +95,12 @@ def main() -> int: if suffix not in ALLOWED_SUFFIXES and path.name not in ALLOWED_NAMES: continue text = path.read_text(encoding="utf-8", errors="replace") + newline_offsets = [ + index for index, character in enumerate(text) if character == "\n" + ] for label, pattern in patterns.items(): for match in pattern.finditer(text): - line = text.count("\n", 0, match.start()) + 1 + line = bisect_left(newline_offsets, match.start()) + 1 findings.append(f"{relative}:{line}: {label}") if findings: diff --git a/skills/_shared/pi_steel/__init__.py b/skills/_shared/pi_steel/__init__.py index fb0bdfd..83da3fe 100644 --- a/skills/_shared/pi_steel/__init__.py +++ b/skills/_shared/pi_steel/__init__.py @@ -1,5 +1,6 @@ """Shared deterministic runtime primitives for pi-steel skills.""" +from .cli import StageArgumentParser, package_version from .contracts import ( ESTIMATE_PACKAGE_VERSION, ITEM_INTENTS, @@ -32,12 +33,14 @@ "RUN_OUTCOMES", "ManifestError", "RunPublisher", + "StageArgumentParser", "canonical_json_bytes", "estimate_input_hash", "instance_ids", "item_id_for", "placement_ids", "outcome_exit_code", + "package_version", "sha256_bytes", "sha256_file", ] diff --git a/skills/_shared/pi_steel/cli.py b/skills/_shared/pi_steel/cli.py new file mode 100644 index 0000000..53cca7b --- /dev/null +++ b/skills/_shared/pi_steel/cli.py @@ -0,0 +1,24 @@ +"""Shared command-line behavior for shipped pi-steel stages.""" + +from __future__ import annotations + +import argparse +import json +import sys +from pathlib import Path + + +class StageArgumentParser(argparse.ArgumentParser): + """Map command usage errors to the shared stage-contract exit code.""" + + def error(self, message): + self.print_usage(sys.stderr) + self.exit(1, f"{self.prog}: error: {message}\n") + + +def package_version(entry_file: str) -> str: + package_path = Path(entry_file).resolve().parents[3] / "package.json" + try: + return json.loads(package_path.read_text(encoding="utf-8"))["version"] + except (OSError, KeyError, json.JSONDecodeError): + return "unknown" diff --git a/skills/_shared/pi_steel/validation.py b/skills/_shared/pi_steel/validation.py index 4d4939a..b812d0a 100644 --- a/skills/_shared/pi_steel/validation.py +++ b/skills/_shared/pi_steel/validation.py @@ -5,6 +5,7 @@ import json import math from dataclasses import dataclass +from functools import lru_cache from pathlib import Path from typing import Any @@ -19,7 +20,6 @@ from .geometry_verify import ( SUPPORTED_SHAPES, finite_positive, - gross_area, hole_within_bounds, net_area, ) @@ -104,15 +104,22 @@ def _add( ) -def _schema_findings( - package: dict[str, Any], input_hash: str -) -> list[dict[str, Any]]: +@lru_cache(maxsize=1) +def _schema_validator(): schema = json.loads(_SCHEMA_PATH.read_text(encoding="utf-8")) - validator = jsonschema.Draft202012Validator( + return jsonschema.Draft202012Validator( schema, format_checker=jsonschema.FormatChecker() ) + + +def _schema_findings( + package: dict[str, Any], input_hash: str +) -> list[dict[str, Any]]: findings = [] - for error in sorted(validator.iter_errors(package), key=lambda item: list(item.path)): + for error in sorted( + _schema_validator().iter_errors(package), + key=lambda item: list(item.path), + ): _add( findings, input_hash, @@ -377,16 +384,17 @@ def validate_estimate_package(package: dict[str, Any]) -> ValidationResult: ) acknowledgements = package.get("review", {}).get("acknowledgements", []) + accepted_findings = { + (acknowledgement.get("finding_id"), acknowledgement.get("input_hash")) + for acknowledgement in acknowledgements + if acknowledgement.get("disposition") == "accepted" + } active = [] for finding in findings: acknowledged = ( finding["severity"] != "blocker" - and any( - acknowledgement.get("finding_id") == finding["finding_id"] - and acknowledgement.get("input_hash") == finding["relevant_hash"] - and acknowledgement.get("disposition") == "accepted" - for acknowledgement in acknowledgements - ) + and (finding["finding_id"], finding["relevant_hash"]) + in accepted_findings ) if not acknowledged: active.append(finding) diff --git a/skills/steel-estimate/scripts/build-estimate-package.py b/skills/steel-estimate/scripts/build-estimate-package.py index c2a5645..6b7651b 100755 --- a/skills/steel-estimate/scripts/build-estimate-package.py +++ b/skills/steel-estimate/scripts/build-estimate-package.py @@ -24,7 +24,14 @@ from bootstrap import bootstrap_shared # noqa: E402 bootstrap_shared(__file__) -from pi_steel import RunPublisher, canonical_json_bytes, outcome_exit_code, sha256_bytes # noqa: E402 +from pi_steel import ( # noqa: E402 + RunPublisher, + StageArgumentParser, + canonical_json_bytes, + outcome_exit_code, + package_version, + sha256_bytes, +) from pi_steel.contracts import ESTIMATE_PACKAGE_VERSION # noqa: E402 from pi_steel.validation import ( # noqa: E402 eligible_on_hand_stock, @@ -58,12 +65,6 @@ class PipelineInputError(ValueError): pass -class StageArgumentParser(argparse.ArgumentParser): - def error(self, message): - self.print_usage(sys.stderr) - self.exit(1, f"{self.prog}: error: {message}\n") - - def normalized_package(package: dict[str, Any]) -> dict[str, Any]: """Return a stable canonical ordering without changing supplied facts.""" value = copy.deepcopy(package) @@ -306,13 +307,21 @@ def apply_inventory_consumption( normalized_by_id = { item["item_id"]: item for item in rfq_normalized.get("items", []) } + purchase_items = sorted( + ( + item + for item in package.get("items", []) + if item.get("intent") == "purchased_stock" + ), + key=lambda row: row["item_id"], + ) lineage = [] for inventory_id in sorted(used): stock = stock_by_id[inventory_id] remaining = used[inventory_id] - for item in sorted(package.get("items", []), key=lambda row: row["item_id"]): - if remaining <= 0 or item.get("intent") != "purchased_stock": - continue + for item in purchase_items: + if remaining <= 0: + break dimensions = item.get("dimensions") or {} if not ( item.get("material") == stock.get("material") @@ -351,16 +360,6 @@ def apply_inventory_consumption( return lineage -def _profile_hash(profile: dict[str, Any] | None) -> str: - if profile is None: - return sha256_bytes(b"missing-profile") - return sha256_bytes( - canonical_json_bytes( - {key: value for key, value in profile.items() if not key.startswith("_")} - ) - ) - - def _fixed_xlsx(path: Path, issued_date: str) -> None: """Normalize volatile XLSX metadata and ZIP timestamps for stable byte hashes.""" timestamp = f"{issued_date}T00:00:00Z".encode() @@ -399,14 +398,6 @@ def _fixed_xlsx(path: Path, issued_date: str) -> None: temporary_path.unlink() -def _package_version() -> str: - path = Path(__file__).resolve().parents[3] / "package.json" - try: - return json.loads(path.read_text(encoding="utf-8"))["version"] - except (OSError, KeyError, json.JSONDecodeError): - return "unknown" - - def _finding(code, severity, path, message): return { "code": code, @@ -481,7 +472,6 @@ def build_pipeline(args) -> tuple[dict[str, Any], Path]: profile_blocked = bool(profile_findings) nest_result = None - nest_job = None if not validation_blocked: nest_job = nest_job_from_package( normalized, @@ -617,7 +607,7 @@ def build_pipeline(args) -> tuple[dict[str, Any], Path]: "density_lb_in3": args.density_lb_in3, "render": not args.no_render, "bake": not args.no_bake, - "profile_hash": _profile_hash(profile), + "profile_hash": rfq_compiler.profile_semantic_hash(profile), } configuration_hash = sha256_bytes(canonical_json_bytes(configuration)) if validation_blocked: @@ -688,7 +678,7 @@ def build_pipeline(args) -> tuple[dict[str, Any], Path]: "rfq_workbook": rfq_compiler.RFQ_COMPILER_VERSION, }, tool_versions={ - "pi_steel": _package_version(), + "pi_steel": package_version(__file__), "estimate_pipeline": PIPELINE_VERSION, "nest_algorithm": nest_engine.NEST_ALGORITHM_VERSION, "rfq_compiler": rfq_compiler.RFQ_COMPILER_VERSION, diff --git a/skills/steel-nest/scripts/nest.py b/skills/steel-nest/scripts/nest.py index 1731b20..5af27d4 100644 --- a/skills/steel-nest/scripts/nest.py +++ b/skills/steel-nest/scripts/nest.py @@ -41,6 +41,7 @@ import math import os import sys +from collections import defaultdict from dataclasses import dataclass, field from pathlib import Path @@ -54,9 +55,11 @@ from pi_steel import ( # noqa: E402 NEST_RESULT_VERSION, RunPublisher, + StageArgumentParser, canonical_json_bytes, item_id_for, outcome_exit_code, + package_version, placement_ids, sha256_bytes, ) @@ -72,14 +75,6 @@ NEST_ALGORITHM_VERSION = "maxrects-bssf-u3" -class StageArgumentParser(argparse.ArgumentParser): - """Use the shared stage contract's exit 1 for command usage errors.""" - - def error(self, message): - self.print_usage(sys.stderr) - self.exit(1, f"{self.prog}: error: {message}\n") - - # -------------------------------------------------------------------------- # Geometry primitives # -------------------------------------------------------------------------- @@ -498,13 +493,15 @@ def number(value, path, *, positive=False, nonnegative=False): "Use either cost_per_lb or cost_per_sheet for one stock entry, not both.", ) ) - for field, value in ( + for cost_field, value in ( ("cost_per_lb", per_pound), ("cost_per_sheet", per_sheet), ): if value is not None: - parsed_cost = number(value, f"{path}.{field}", nonnegative=True) - if field == "cost_per_lb": + parsed_cost = number( + value, f"{path}.{cost_field}", nonnegative=True + ) + if cost_field == "cost_per_lb": per_pound = parsed_cost else: per_sheet = parsed_cost @@ -807,6 +804,10 @@ def _summarize( net_approximations = { part["net_area_approximation"] for part in normalized["parts"] } + net_approximation_by_item = { + part["item_id"]: part["net_area_approximation"] + for part in normalized["parts"] + } for plate in used_plates: stock = plate["stock"] @@ -846,11 +847,7 @@ def _summarize( placement.shape == "irregular" for placement in plate["placements"] ) else "exact" plate_net_statuses = { - next( - part["net_area_approximation"] - for part in normalized["parts"] - if part["item_id"] == placement.item_id - ) + net_approximation_by_item[placement.item_id] for placement in plate["placements"] } net_approximation = ( @@ -955,22 +952,22 @@ def _summarize( else: geometry_readiness = "geometry_verified" + placements_by_group = defaultdict(list) + for plate in plate_reports: + for placement in plate["placements"]: + placements_by_group[ + ( + placement["material"], + placement["grade"], + placement["thickness"], + ) + ].append(placement) + groups = [] group_keys = sorted( {_material_key(part) for part in normalized["parts"]}, key=repr ) for material, grade, thickness in group_keys: - placements = [ - placement - for plate in plate_reports - for placement in plate["placements"] - if ( - placement["material"], - placement["grade"], - placement["thickness"], - ) - == (material, grade, thickness) - ] groups.append( { "group_id": "nest-group:" @@ -984,7 +981,7 @@ def _summarize( "material": material, "grade": grade, "thickness": thickness, - "placements": placements, + "placements": placements_by_group[(material, grade, thickness)], } ) configuration_hash = sha256_bytes( @@ -1060,7 +1057,6 @@ def _fmt(v): # -------------------------------------------------------------------------- def rfq_nesting_block(res): """Build the versioned nest-to-RFQ handoff without merging stock variants.""" - from collections import defaultdict groups = defaultdict(list) for pr in res["plate_reports"]: groups[ @@ -1449,14 +1445,6 @@ def stage_decision(res, geometry_verified_only=False): return outcome, package_status, findings -def package_version(): - package_path = Path(__file__).resolve().parents[3] / "package.json" - try: - return json.loads(package_path.read_text(encoding="utf-8"))["version"] - except (OSError, KeyError, json.JSONDecodeError): - return "unknown" - - def missing_render_dependencies(): """Return optional render modules unavailable to this interpreter.""" modules = ("ezdxf", "matplotlib", "numpy") @@ -1525,7 +1513,7 @@ def publish_nest_run(job, args): "rfq_nesting": "1.0.0", }, tool_versions={ - "pi_steel": package_version(), + "pi_steel": package_version(__file__), "nest_algorithm": NEST_ALGORITHM_VERSION, }, explicit_dates={}, diff --git a/skills/steel-rfq/scripts/generate-rfq.py b/skills/steel-rfq/scripts/generate-rfq.py index edb3c92..daefad0 100755 --- a/skills/steel-rfq/scripts/generate-rfq.py +++ b/skills/steel-rfq/scripts/generate-rfq.py @@ -26,8 +26,15 @@ from bootstrap import bootstrap_shared # noqa: E402 bootstrap_shared(__file__) -from pi_steel import RunPublisher, canonical_json_bytes, outcome_exit_code, sha256_bytes # noqa: E402 -from pi_steel.contracts import ESTIMATE_PACKAGE_VERSION, estimate_input_hash # noqa: E402 +from pi_steel import ( # noqa: E402 + RunPublisher, + StageArgumentParser, + canonical_json_bytes, + outcome_exit_code, + package_version, + sha256_bytes, +) +from pi_steel.contracts import ESTIMATE_PACKAGE_VERSION # noqa: E402 from pi_steel.validation import validate_estimate_package # noqa: E402 RECALC_PATH = Path(__file__).with_name("recalc.py") @@ -85,12 +92,6 @@ class RfqInputError(ValueError): """Raised when an input cannot be mapped without guessing.""" -class StageArgumentParser(argparse.ArgumentParser): - def error(self, message): - self.print_usage(sys.stderr) - self.exit(1, f"{self.prog}: error: {message}\n") - - def _valid_date(value: Any) -> bool: try: return date.fromisoformat(str(value)).isoformat() == str(value) @@ -98,6 +99,15 @@ def _valid_date(value: Any) -> bool: return False +def profile_semantic_hash(profile: dict[str, Any] | None) -> str: + if profile is None: + return sha256_bytes(b"missing-profile") + public_profile = { + key: value for key, value in profile.items() if not key.startswith("_") + } + return sha256_bytes(canonical_json_bytes(public_profile)) + + def _fmt(value: Any) -> str: if value is None: return "" @@ -862,14 +872,6 @@ def compile_workbook( } -def _package_version() -> str: - path = Path(__file__).resolve().parents[3] / "package.json" - try: - return json.loads(path.read_text(encoding="utf-8"))["version"] - except (OSError, KeyError, json.JSONDecodeError): - return "unknown" - - def _load_normalized(input_path: Path) -> dict[str, Any]: if input_path.suffix.lower() == ".json": value = json.loads(input_path.read_text(encoding="utf-8")) @@ -961,18 +963,13 @@ def publish_rfq_run(args) -> tuple[dict[str, Any], Path]: outcome, package_status = "review_required", "rfq_draft_review_required" else: outcome, package_status = "ready", "rfq_ready_for_review" - profile_hash = ( - sha256_bytes(canonical_json_bytes({k: v for k, v in profile.items() if not k.startswith("_")})) - if profile - else sha256_bytes(b"missing-profile") - ) configuration_hash = sha256_bytes( canonical_json_bytes( { "compiler_version": RFQ_COMPILER_VERSION, "issued_date": args.issued_date, "project_location": args.project_location, - "profile_hash": profile_hash, + "profile_hash": profile_semantic_hash(profile), "nest_handoff": nest_handoff, "bake_requested": not args.no_bake, } @@ -1002,7 +999,7 @@ def publish_rfq_run(args) -> tuple[dict[str, Any], Path]: "rfq_nesting": NEST_HANDOFF_VERSION, }, tool_versions={ - "pi_steel": _package_version(), + "pi_steel": package_version(__file__), "rfq_compiler": RFQ_COMPILER_VERSION, }, explicit_dates={"issued_date": args.issued_date}, @@ -1011,7 +1008,6 @@ def publish_rfq_run(args) -> tuple[dict[str, Any], Path]: approximations=[], run_id=args.run_id, ) as publisher: - compile_result = None if not blockers: compile_result = compile_workbook( normalized, diff --git a/tests/test_estimate_pipeline.py b/tests/test_estimate_pipeline.py index 94ae305..c9438ef 100644 --- a/tests/test_estimate_pipeline.py +++ b/tests/test_estimate_pipeline.py @@ -1,4 +1,3 @@ -import hashlib import json import os import subprocess From ef4df765d3491b2fbfb28a71e87dd32254ebc860 Mon Sep 17 00:00:00 2001 From: Victor Garcia Date: Tue, 28 Jul 2026 13:38:57 -0600 Subject: [PATCH 12/15] fix(review): harden public estimate workflow Close privacy, contract, lineage, failure-reporting, workbook, and runtime reliability gaps found during the compound review. --- .gitignore | 1 + .npmignore | 1 + PUBLIC_DATA_POLICY.md | 7 +- README.md | 3 +- package.json | 2 + scripts/check-public-data.py | 317 ++++++++++++++++-- skills/_shared/pi_steel/__init__.py | 3 +- skills/_shared/pi_steel/cli.py | 145 +++++++- skills/_shared/pi_steel/geometry_verify.py | 133 +++++--- skills/_shared/pi_steel/run_manifest.py | 11 + skills/_shared/pi_steel/validation.py | 122 ++++++- .../schemas/estimate-package.schema.json | 103 ++++-- .../_shared/schemas/nest-result.schema.json | 99 +++++- skills/steel-estimate/SKILL.md | 39 +++ .../scripts/acknowledge-finding.py | 193 +++++++++++ .../scripts/build-estimate-package.py | 62 +++- skills/steel-nest/scripts/nest.py | 76 ++++- skills/steel-rfq/references/rfq-input.md | 3 +- skills/steel-rfq/scripts/generate-rfq.py | 220 ++++++++++-- skills/steel-rfq/scripts/recalc.py | 32 +- skills/steel-takeoff/SKILL.md | 8 +- .../fixtures/pipeline/synthetic-estimate.json | 2 + tests/fixtures/rfq/nest-handoff.json | 3 + tests/golden/rfq/semantic-workbook.json | 6 +- tests/test_acknowledge_finding_cli.py | 159 +++++++++ tests/test_contracts.py | 192 +++++++++++ tests/test_estimate_pipeline.py | 69 +++- tests/test_full_render_smoke.py | 145 +++++++- tests/test_installed_scripts.py | 13 + tests/test_nest_cli_contract.py | 3 + tests/test_nest_invariants.py | 89 +++++ tests/test_package_contents.py | 1 + tests/test_public_data_policy.py | 196 +++++++++++ tests/test_recalc.py | 54 +++ tests/test_rfq_generator.py | 28 ++ tests/test_rfq_workbook_contract.py | 79 +++++ tests/test_run_manifests.py | 29 ++ tests/test_runtime_bootstrap.py | 42 +++ tests/test_structured_failure_cli.py | 100 ++++++ tests/test_takeoff_cli.py | 166 +++++++++ 40 files changed, 2773 insertions(+), 183 deletions(-) create mode 100755 skills/steel-estimate/scripts/acknowledge-finding.py create mode 100644 tests/test_acknowledge_finding_cli.py create mode 100644 tests/test_recalc.py create mode 100644 tests/test_structured_failure_cli.py create mode 100644 tests/test_takeoff_cli.py diff --git a/.gitignore b/.gitignore index 0193381..00a5d45 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ node_modules/ *.tgz .DS_Store outputs/ +out/ .a5c/ __pycache__/ *.pyc diff --git a/.npmignore b/.npmignore index 4db4880..bbc58f2 100644 --- a/.npmignore +++ b/.npmignore @@ -7,6 +7,7 @@ local-data/ customer-data/ vendor-data/ outputs/ +out/ *.local.json *.private.json *.xlsx diff --git a/PUBLIC_DATA_POLICY.md b/PUBLIC_DATA_POLICY.md index 3f1ca6b..38020bb 100644 --- a/PUBLIC_DATA_POLICY.md +++ b/PUBLIC_DATA_POLICY.md @@ -37,10 +37,13 @@ distribution. required, label values as synthetic in the same fixture. 5. Keep private inputs and generated outputs outside the repository in ignored directories such as `private/`, `local-data/`, `customer-data/`, or `outputs/`. -6. Run `python3 scripts/check-public-data.py` before every commit and release. +6. Run `python3 scripts/check-public-data.py` before every commit and release. Before + pushing, also scan the outgoing range with + `python3 scripts/check-public-data.py --range ..HEAD`. 7. Put private names and identifiers, one per line, in the ignored local file `.pi-steel/private-terms.txt`; the scanner checks them without committing the denylist or echoing matched text. Git history is public too. Removing a value from the current tree does not remove it -from prior commits; history cleanup requires an explicit coordinated rewrite. +from prior commits. Use `npm run privacy:history` for a redacted audit; history +cleanup requires an explicit coordinated rewrite. diff --git a/README.md b/README.md index 04d51fc..7090177 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ pi install npm:@structupath/pi-steel Structural steel quantity takeoff with a bundled **AISC 16th Edition shapes database (477 shapes)** — W, HSS, angles, channels, pipe — plus scripts the agent runs directly: - `lookup-member.sh` — full property set for any AISC designation (`W14X30` → plf, d, bf, A, Ix, Sx, …) -- `calculate-weight.sh` — BOM totals with connection and misc-steel allowances, tonnage, cost sensitivity +- `calculate-weight.sh` — BOM weight totals with connection and misc-steel allowances and tonnage; it does not invent pricing - `validate-bom.py` — catches invalid designations, wrong grades, duplicate marks, unreasonable weights Also includes reference guides for AISC shape families, takeoff procedures with worked examples, connection types and hardware weights, material grades, and bolt capacities. @@ -77,6 +77,7 @@ artifacts outside this repository. Public examples are synthetic and must follow npm test # base, no-render suite npm run test:full # optional PDF/PNG/DXF/LibreOffice smoke tests npm run privacy:check # public-repository data guard +npm run privacy:history # redacted audit of every reachable commit npm run pack:check # npm contents plus unpacked-runtime smoke test npm run provenance:check # shape-data integrity and recorded decision npm run release:check # complete release gate diff --git a/package.json b/package.json index 7c1b90a..9a3d055 100644 --- a/package.json +++ b/package.json @@ -38,6 +38,7 @@ "test:full": "python3 -m pytest -m full_render", "test:runtime": "python3 -m pytest tests/test_runtime_bootstrap.py tests/test_run_manifests.py tests/test_installed_scripts.py", "privacy:check": "python3 scripts/check-public-data.py", + "privacy:history": "python3 scripts/check-public-data.py --history", "provenance:check": "python3 scripts/check-data-provenance.py", "pack:check": "python3 -m pytest tests/test_package_contents.py tests/test_installed_scripts.py", "pack:dry-run": "npm pack --dry-run", @@ -55,6 +56,7 @@ "!skills/**/*.pyc", "!skills/**/company-profile.json", "scripts/doctor.py", + "scripts/check-public-data.py", "scripts/check-data-provenance.py", "pyproject.toml", "requirements.txt", diff --git a/scripts/check-public-data.py b/scripts/check-public-data.py index 855a3af..8aca14c 100644 --- a/scripts/check-public-data.py +++ b/scripts/check-public-data.py @@ -3,6 +3,7 @@ from __future__ import annotations +import argparse import re import subprocess import sys @@ -11,18 +12,6 @@ ROOT = Path(__file__).resolve().parents[1] -ALLOWED_SUFFIXES = { - ".csv", - ".json", - ".md", - ".py", - ".sh", - ".toml", - ".txt", - ".yaml", - ".yml", -} -ALLOWED_NAMES = {".gitignore", "LICENSE", "package.json"} SKIP_PARTS = {".git", ".a5c", "__pycache__", "node_modules"} SKIP_FILES = {Path("scripts/check-public-data.py")} FORBIDDEN_BINARY_SUFFIXES = { @@ -36,6 +25,15 @@ ".xls", ".xlsx", } +SENSITIVE_SUFFIXES = {".key", ".p12", ".pem", ".pfx"} +PRIVATE_KEY_NAMES = { + "id_dsa", + "id_ecdsa", + "id_ed25519", + "id_rsa", + "private-key", + "private_key", +} PATTERNS = { "private operating-company claim": re.compile( r"team behind (?:a|the) production structural[- ]steel", re.I @@ -48,26 +46,39 @@ r"(? list[Path]: +def tracked_files(root: Path = ROOT) -> list[Path]: result = subprocess.run( ["git", "ls-files", "--cached", "--others", "--exclude-standard"], - cwd=ROOT, - check=True, + cwd=root, + check=False, capture_output=True, text=True, ) - return [ROOT / line for line in result.stdout.splitlines() if line] + if result.returncode == 0: + return [root / line for line in result.stdout.splitlines() if line] + return [ + path + for path in root.rglob("*") + if path.is_file() and not any(part in SKIP_PARTS for part in path.parts) + ] -def main() -> int: - findings: list[str] = [] +def scan_patterns(root: Path = ROOT) -> dict[str, re.Pattern[str]]: patterns = dict(PATTERNS) - private_terms = ROOT / ".pi-steel" / "private-terms.txt" + private_terms = root / ".pi-steel" / "private-terms.txt" if private_terms.is_file(): terms = [ line.strip() @@ -79,29 +90,285 @@ def main() -> int: "|".join(re.escape(term) for term in sorted(terms, key=len, reverse=True)), re.I, ) + return patterns + + +def sensitive_path_reason(relative: Path) -> str | None: + name = relative.name.lower() + if name == ".env" or name.startswith(".env."): + return "sensitive environment file" + if relative.suffix.lower() in SENSITIVE_SUFFIXES: + return "sensitive key or certificate file" + if name in PRIVATE_KEY_NAMES: + return "private key file" + return None + - for path in tracked_files(): - relative = path.relative_to(ROOT) +def scan_paths( + paths: list[Path], + *, + root: Path = ROOT, + patterns: dict[str, re.Pattern[str]] | None = None, +) -> list[str]: + findings: list[str] = [] + active_patterns = scan_patterns(root) if patterns is None else patterns + for path in paths: + relative = path.relative_to(root) if ( relative in SKIP_FILES or any(part in SKIP_PARTS for part in relative.parts) or not path.is_file() ): continue + sensitive_reason = sensitive_path_reason(relative) + if sensitive_reason: + findings.append(f"{relative}: {sensitive_reason}") + continue suffix = path.suffix.lower() if suffix in FORBIDDEN_BINARY_SUFFIXES: findings.append(f"{relative}: public repository must not contain {suffix} artifacts") continue - if suffix not in ALLOWED_SUFFIXES and path.name not in ALLOWED_NAMES: + try: + text = path.read_bytes().decode("utf-8") + except UnicodeDecodeError: + findings.append(f"{relative}: unknown binary file") + continue + if "\x00" in text: + findings.append(f"{relative}: unknown binary file") continue - text = path.read_text(encoding="utf-8", errors="replace") newline_offsets = [ index for index, character in enumerate(text) if character == "\n" ] - for label, pattern in patterns.items(): + for label, pattern in active_patterns.items(): for match in pattern.finditer(text): line = bisect_left(newline_offsets, match.start()) + 1 findings.append(f"{relative}:{line}: {label}") + return findings + + +def _scan_bytes( + relative: Path, + content: bytes, + *, + patterns: dict[str, re.Pattern[str]], + prefix: str = "", +) -> list[str]: + if relative in SKIP_FILES or any(part in SKIP_PARTS for part in relative.parts): + return [] + sensitive_reason = sensitive_path_reason(relative) + if sensitive_reason: + return [f"{prefix}{relative}: {sensitive_reason}"] + suffix = relative.suffix.lower() + if suffix in FORBIDDEN_BINARY_SUFFIXES: + return [ + f"{prefix}{relative}: public repository must not contain {suffix} artifacts" + ] + try: + text = content.decode("utf-8") + except UnicodeDecodeError: + return [f"{prefix}{relative}: unknown binary file"] + if "\x00" in text: + return [f"{prefix}{relative}: unknown binary file"] + + findings: list[str] = [] + newline_offsets = [ + index for index, character in enumerate(text) if character == "\n" + ] + for label, pattern in patterns.items(): + for match in pattern.finditer(text): + line = bisect_left(newline_offsets, match.start()) + 1 + findings.append(f"{prefix}{relative}:{line}: {label}") + return findings + + +def staged_findings( + root: Path = ROOT, + *, + patterns: dict[str, re.Pattern[str]] | None = None, +) -> list[str]: + """Scan the exact stage-zero blobs that would be included in the next commit.""" + changed = subprocess.run( + ["git", "diff", "--cached", "--name-only", "-z", "--diff-filter=ACMR"], + cwd=root, + check=False, + capture_output=True, + ) + if changed.returncode != 0: + return [] + + active_patterns = scan_patterns(root) if patterns is None else patterns + findings: list[str] = [] + blob_cache: dict[str, bytes] = {} + for raw_path in changed.stdout.split(b"\0"): + if not raw_path: + continue + relative = Path(raw_path.decode("utf-8", errors="surrogateescape")) + index_entry = subprocess.run( + ["git", "ls-files", "--stage", "-z", "--", str(relative)], + cwd=root, + check=False, + capture_output=True, + ) + entries = [entry for entry in index_entry.stdout.split(b"\0") if entry] + stage_zero = [ + entry for entry in entries if entry.split(b"\t", 1)[0].endswith(b" 0") + ] + if not stage_zero: + continue + metadata, _ = stage_zero[0].split(b"\t", 1) + _, object_id, _ = metadata.decode("ascii").split() + if object_id not in blob_cache: + blob = subprocess.run( + ["git", "cat-file", "blob", object_id], + cwd=root, + check=False, + capture_output=True, + ) + if blob.returncode != 0: + continue + blob_cache[object_id] = blob.stdout + findings.extend( + _scan_bytes( + relative, + blob_cache[object_id], + patterns=active_patterns, + prefix="staged ", + ) + ) + return findings + + +def revision_findings( + revision_args: list[str], + root: Path = ROOT, + *, + patterns: dict[str, re.Pattern[str]] | None = None, +) -> list[str]: + """Scan committed blobs reachable from the supplied ``git rev-list`` arguments.""" + commits = subprocess.run( + ["git", "rev-list", *revision_args], + cwd=root, + check=False, + capture_output=True, + text=True, + ) + if commits.returncode != 0: + raise ValueError(commits.stderr.strip() or "invalid Git revision") + + active_patterns = scan_patterns(root) if patterns is None else patterns + findings: list[str] = [] + blob_cache: dict[str, bytes] = {} + scanned: set[tuple[str, Path]] = set() + for commit in commits.stdout.splitlines(): + tree = subprocess.run( + ["git", "ls-tree", "-r", "-z", commit], + cwd=root, + check=False, + capture_output=True, + ) + if tree.returncode != 0: + continue + for entry in tree.stdout.split(b"\0"): + if not entry: + continue + metadata, raw_path = entry.split(b"\t", 1) + _, object_type, object_id = metadata.decode("ascii").split() + if object_type != "blob": + continue + relative = Path(raw_path.decode("utf-8", errors="surrogateescape")) + identity = (object_id, relative) + if identity in scanned: + continue + scanned.add(identity) + if object_id not in blob_cache: + blob = subprocess.run( + ["git", "cat-file", "blob", object_id], + cwd=root, + check=False, + capture_output=True, + ) + if blob.returncode != 0: + continue + blob_cache[object_id] = blob.stdout + findings.extend( + _scan_bytes( + relative, + blob_cache[object_id], + patterns=active_patterns, + prefix=f"commit {commit[:12]} ", + ) + ) + return findings + + +def revision_metadata_findings( + revision_args: list[str], + root: Path = ROOT, +) -> list[str]: + """Report non-noreply author addresses without exposing the address itself.""" + commits = subprocess.run( + ["git", "log", "--format=%H%x00%ae", *revision_args], + cwd=root, + check=False, + capture_output=True, + text=True, + ) + if commits.returncode != 0: + raise ValueError(commits.stderr.strip() or "invalid Git revision") + + findings: list[str] = [] + seen_addresses: set[str] = set() + for entry in commits.stdout.splitlines(): + if "\0" not in entry: + continue + commit, address = entry.split("\0", 1) + normalized = address.strip().lower() + if ( + not normalized + or normalized in seen_addresses + or normalized.endswith("@users.noreply.github.com") + ): + continue + seen_addresses.add(normalized) + findings.append( + f"commit {commit[:12]} author metadata: non-noreply email address" + ) + return findings + + +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Check public repository content without printing matched values." + ) + revisions = parser.add_mutually_exclusive_group() + revisions.add_argument( + "--history", + action="store_true", + help="also scan every commit reachable from every local ref", + ) + revisions.add_argument( + "--range", + metavar="REVISION_RANGE", + help="also scan commits selected by a git rev-list revision range", + ) + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + findings = scan_paths(tracked_files(), root=ROOT) + findings.extend(staged_findings(ROOT)) + if args.history: + findings.extend(revision_findings(["--all"], ROOT)) + findings.extend(revision_metadata_findings(["--all"], ROOT)) + elif args.range: + try: + findings.extend(revision_findings([args.range], ROOT)) + findings.extend(revision_metadata_findings([args.range], ROOT)) + except ValueError as error: + print(f"Public-data check could not scan revisions: {error}", file=sys.stderr) + return 2 + findings = list(dict.fromkeys(findings)) if findings: print("Public-data check failed:", file=sys.stderr) diff --git a/skills/_shared/pi_steel/__init__.py b/skills/_shared/pi_steel/__init__.py index 83da3fe..1a735a7 100644 --- a/skills/_shared/pi_steel/__init__.py +++ b/skills/_shared/pi_steel/__init__.py @@ -1,6 +1,6 @@ """Shared deterministic runtime primitives for pi-steel skills.""" -from .cli import StageArgumentParser, package_version +from .cli import StageArgumentParser, package_version, publish_failure_diagnostic from .contracts import ( ESTIMATE_PACKAGE_VERSION, ITEM_INTENTS, @@ -41,6 +41,7 @@ "placement_ids", "outcome_exit_code", "package_version", + "publish_failure_diagnostic", "sha256_bytes", "sha256_file", ] diff --git a/skills/_shared/pi_steel/cli.py b/skills/_shared/pi_steel/cli.py index 53cca7b..fdbcd2e 100644 --- a/skills/_shared/pi_steel/cli.py +++ b/skills/_shared/pi_steel/cli.py @@ -6,14 +6,82 @@ import json import sys from pathlib import Path +from typing import Any + +from .run_manifest import ( + ManifestError, + RunPublisher, + canonical_json_bytes, + sha256_bytes, + sha256_file, +) class StageArgumentParser(argparse.ArgumentParser): """Map command usage errors to the shared stage-contract exit code.""" + def configure_failure_diagnostics( + self, + *, + stage: str, + entry_file: str, + input_option: str, + date_options: dict[str, str] | None = None, + ) -> None: + self._failure_context = { + "stage": stage, + "entry_file": entry_file, + "input_option": input_option, + "date_options": date_options or {}, + } + + def parse_args(self, args=None, namespace=None): + self._active_argv = list(sys.argv[1:] if args is None else args) + return super().parse_args(args, namespace) + + def _option_value(self, option: str) -> str | None: + argv = getattr(self, "_active_argv", []) + for index, argument in enumerate(argv): + if argument == option and index + 1 < len(argv): + return str(argv[index + 1]) + if str(argument).startswith(f"{option}="): + return str(argument).split("=", 1)[1] + action = next( + (item for item in self._actions if option in item.option_strings), + None, + ) + if action is not None and action.default not in (None, argparse.SUPPRESS): + return str(action.default) + return None + def error(self, message): + diagnostic_path = None + context = getattr(self, "_failure_context", None) + if context is not None: + destination = self._option_value("--out") + if destination is not None: + explicit_dates = { + field: value + for option, field in context["date_options"].items() + if (value := self._option_value(option)) is not None + } + diagnostic_path = publish_failure_diagnostic( + destination, + stage=context["stage"], + input_path=self._option_value(context["input_option"]), + error=ValueError(message), + tool_version=package_version(context["entry_file"]), + run_id=self._option_value("--run-id"), + explicit_dates=explicit_dates, + finding_code="cli_usage_error", + ) self.print_usage(sys.stderr) - self.exit(1, f"{self.prog}: error: {message}\n") + suffix = ( + f"; diagnostic published: {diagnostic_path}" + if diagnostic_path is not None + else "" + ) + self.exit(1, f"{self.prog}: error: {message}{suffix}\n") def package_version(entry_file: str) -> str: @@ -22,3 +90,78 @@ def package_version(entry_file: str) -> str: return json.loads(package_path.read_text(encoding="utf-8"))["version"] except (OSError, KeyError, json.JSONDecodeError): return "unknown" + + +def _diagnostic_message(error: Exception) -> str: + """Describe a failure without persisting input paths in published artifacts.""" + if isinstance(error, OSError): + detail = error.strerror or "input/output error" + return f"{error.__class__.__name__}: {detail}" + return str(error) or error.__class__.__name__ + + +def publish_failure_diagnostic( + destination: str | Path, + *, + stage: str, + input_path: str | Path | None, + error: Exception, + tool_version: str, + run_id: str | None = None, + explicit_dates: dict[str, str] | None = None, + finding_code: str = "input_unreadable_or_invalid", +) -> Path | None: + """Best-effort publication for failures that occur after CLI parsing. + + The original failure remains authoritative: publication errors are contained so + callers can preserve the shared exit-code contract and report the first error. + """ + path = Path(input_path) if input_path is not None else None + try: + input_hash = ( + sha256_file(path) + if path is not None and path.is_file() + else sha256_bytes(canonical_json_bytes({"input_state": "unavailable"})) + ) + message = _diagnostic_message(error) + finding: dict[str, Any] = { + "code": finding_code, + "severity": "error", + "message": message, + "exception_type": error.__class__.__name__, + } + qa_report = { + "schema_version": "1.0.0", + "stage": stage, + "run_outcome": "usage_or_internal_error", + "package_status": "draft", + "findings": [finding], + "warnings": [], + } + configuration_hash = sha256_bytes( + canonical_json_bytes( + { + "failure_contract": "1.0.0", + "stage": stage, + "explicit_dates": explicit_dates or {}, + } + ) + ) + with RunPublisher( + destination, + stage=stage, + run_outcome="usage_or_internal_error", + package_status="draft", + input_hash=input_hash, + configuration_hash=configuration_hash, + schema_versions={"run_manifest": "1.0.0"}, + tool_versions={"pi_steel": tool_version}, + explicit_dates=explicit_dates or {}, + warnings=[message], + approximations=[], + run_id=run_id, + ) as publisher: + publisher.write_qa_report(qa_report) + return publisher.publish() + except (ManifestError, OSError, TypeError, ValueError): + return None diff --git a/skills/_shared/pi_steel/geometry_verify.py b/skills/_shared/pi_steel/geometry_verify.py index 72db803..94e5d5d 100644 --- a/skills/_shared/pi_steel/geometry_verify.py +++ b/skills/_shared/pi_steel/geometry_verify.py @@ -75,6 +75,80 @@ def group_plate_items(items: list[dict[str, Any]]) -> list[dict[str, Any]]: ] +def _finite_placement_values(placement: dict[str, Any]) -> bool: + return all( + isinstance(placement.get(field), (int, float)) + and math.isfinite(placement[field]) + for field in ("x", "y", "w", "h") + ) + + +def _placements_separated( + first: dict[str, Any], + second: dict[str, Any], + clearance: float, + epsilon: float, +) -> bool: + return ( + first["x"] + first["w"] + clearance <= second["x"] + epsilon + or second["x"] + second["w"] + clearance <= first["x"] + epsilon + or first["y"] + first["h"] + clearance <= second["y"] + epsilon + or second["y"] + second["h"] + clearance <= first["y"] + epsilon + ) + + +def _overlap_candidate_pairs( + placements: list[dict[str, Any]], + clearance: float, + epsilon: float, +) -> list[tuple[int, int]]: + """Return deterministic overlap pairs without scanning every valid pair.""" + numeric = { + index for index, placement in enumerate(placements) + if _finite_placement_values(placement) + } + sweepable = { + index for index in numeric + if placements[index]["w"] > 0 and placements[index]["h"] > 0 + } + candidates: set[tuple[int, int]] = set() + active: list[int] = [] + for current_index in sorted( + sweepable, key=lambda index: (placements[index]["x"], index) + ): + current = placements[current_index] + active = [ + index + for index in active + if ( + placements[index]["x"] + + placements[index]["w"] + + clearance + > current["x"] + epsilon + ) + ] + for other_index in active: + other = placements[other_index] + if not ( + other["y"] + other["h"] + clearance <= current["y"] + epsilon + or current["y"] + current["h"] + clearance + <= other["y"] + epsilon + ): + candidates.add( + (min(other_index, current_index), max(other_index, current_index)) + ) + active.append(current_index) + + nonsweepable = numeric - sweepable + for first_index in sorted(nonsweepable): + for second_index in sorted(numeric): + if first_index < second_index: + candidates.add((first_index, second_index)) + elif second_index < first_index and second_index in sweepable: + candidates.add((second_index, first_index)) + return sorted(candidates) + + def verify_nest_placements( plate_reports: list[dict[str, Any]], *, @@ -145,44 +219,25 @@ def verify_nest_placements( } ) - for first_index, first in enumerate(placements): - for second_index in range(first_index + 1, len(placements)): - second = placements[second_index] - values = [ - first.get(field) - for field in ("x", "y", "w", "h") - ] + [ - second.get(field) - for field in ("x", "y", "w", "h") - ] - if not all( - isinstance(value, (int, float)) - and math.isfinite(value) - for value in values - ): - continue - separated = ( - first["x"] + first["w"] + inter_part_clearance - <= second["x"] + epsilon - or second["x"] + second["w"] + inter_part_clearance - <= first["x"] + epsilon - or first["y"] + first["h"] + inter_part_clearance - <= second["y"] + epsilon - or second["y"] + second["h"] + inter_part_clearance - <= first["y"] + epsilon + for first_index, second_index in _overlap_candidate_pairs( + placements, inter_part_clearance, epsilon + ): + first = placements[first_index] + second = placements[second_index] + if not _placements_separated( + first, second, inter_part_clearance, epsilon + ): + findings.append( + { + "code": "placement_overlap", + "path": ( + f"$.plate_reports[{plate_index}].placements" + f"[{first_index},{second_index}]" + ), + "message": ( + "Placements overlap or violate the required " + "kerf-plus-gap clearance." + ), + } ) - if not separated: - findings.append( - { - "code": "placement_overlap", - "path": ( - f"$.plate_reports[{plate_index}].placements" - f"[{first_index},{second_index}]" - ), - "message": ( - "Placements overlap or violate the required " - "kerf-plus-gap clearance." - ), - } - ) return findings diff --git a/skills/_shared/pi_steel/run_manifest.py b/skills/_shared/pi_steel/run_manifest.py index d467b52..5a4107c 100644 --- a/skills/_shared/pi_steel/run_manifest.py +++ b/skills/_shared/pi_steel/run_manifest.py @@ -298,6 +298,17 @@ def publish(self) -> Path: with os.fdopen(pointer_fd, "wb") as stream: stream.write(canonical_json_bytes(pointer)) os.replace(pointer_name, self.destination / "latest-run.json") + except Exception as pointer_error: + try: + shutil.rmtree(self.final_path) + except OSError as rollback_error: + raise ManifestError( + "latest-run pointer update failed and the unpublished run " + f"could not be rolled back: {self.final_path}" + ) from rollback_error + raise ManifestError( + "latest-run pointer update failed; unpublished run was rolled back" + ) from pointer_error finally: if os.path.exists(pointer_name): os.unlink(pointer_name) diff --git a/skills/_shared/pi_steel/validation.py b/skills/_shared/pi_steel/validation.py index b812d0a..c3249ac 100644 --- a/skills/_shared/pi_steel/validation.py +++ b/skills/_shared/pi_steel/validation.py @@ -131,6 +131,83 @@ def _schema_findings( return findings +def _nonfinite_number_findings( + value: Any, + input_hash: str, + *, + path: str = "$", +) -> list[dict[str, Any]]: + """Reject non-JSON numeric values throughout the canonical package. + + Python's JSON and JSON Schema implementations can accept NaN and infinity + even though they are not interoperable JSON values. Keep this domain guard + recursive so newly added numeric contract fields are protected by default. + """ + findings: list[dict[str, Any]] = [] + if isinstance(value, dict): + for key, child in value.items(): + findings.extend( + _nonfinite_number_findings( + child, + input_hash, + path=f"{path}.{key}", + ) + ) + elif isinstance(value, list): + for index, child in enumerate(value): + findings.extend( + _nonfinite_number_findings( + child, + input_hash, + path=f"{path}[{index}]", + ) + ) + elif isinstance(value, float) and not math.isfinite(value): + _add( + findings, + input_hash, + "nonfinite_number", + "blocker", + path, + "Numeric values must be finite JSON numbers.", + ) + return findings + + +def _merge_supplied_review_findings( + package: dict[str, Any], + input_hash: str, + findings: list[dict[str, Any]], +) -> None: + """Carry canonical review findings forward and expose stale review state.""" + existing = { + (finding["finding_id"], finding["relevant_hash"]) + for finding in findings + } + for index, supplied in enumerate( + package.get("review", {}).get("findings", []) + ): + key = (supplied["finding_id"], supplied["relevant_hash"]) + if key not in existing: + findings.append(dict(supplied)) + existing.add(key) + if supplied["relevant_hash"] != input_hash: + stale = _finding( + code="stale_review_finding", + severity="blocker", + path=f"$.review.findings[{index}].relevant_hash", + message=( + f"Review finding {supplied['finding_id']!r} was produced for " + "a different estimate input and must be regenerated." + ), + relevant_hash=input_hash, + ) + stale_key = (stale["finding_id"], stale["relevant_hash"]) + if stale_key not in existing: + findings.append(stale) + existing.add(stale_key) + + def _geometry_findings( item: dict[str, Any], index: int, @@ -234,8 +311,33 @@ def validate_estimate_package(package: dict[str, Any]) -> ValidationResult: ) return ValidationResult("invalid", input_hash, [finding], [finding]) - findings = _schema_findings(package, input_hash) - if findings: + findings = _nonfinite_number_findings(package, input_hash) + schema_findings = _schema_findings(package, input_hash) + if schema_findings: + findings.extend(schema_findings) + # Retain the pre-existing field-specific diagnostics for the common + # scalar errors that are now also constrained directly by the schema. + items = package.get("items", []) + if isinstance(items, list): + for index, item in enumerate(items): + if not isinstance(item, dict): + continue + quantity = item.get("quantity") + if ( + not isinstance(quantity, int) + or isinstance(quantity, bool) + or quantity <= 0 + ): + _add( + findings, + input_hash, + "invalid_quantity", + "blocker", + f"$.items[{index}].quantity", + "Quantity must be a positive integer.", + ) + if item.get("intent") == "fabricated_part": + _geometry_findings(item, index, input_hash, findings) return ValidationResult("invalid", input_hash, findings, findings) source_ids: dict[str, int] = {} item_ids: dict[str, int] = {} @@ -383,6 +485,22 @@ def validate_estimate_package(package: dict[str, Any]) -> ValidationResult: "Cost currency, unit basis, effective date, and source must be preserved.", ) + for index, assumption in enumerate(package.get("assumptions", [])): + if assumption.get("status") == "unresolved": + _add( + findings, + input_hash, + "unresolved_assumption", + "warning", + f"$.assumptions[{index}]", + ( + f"Assumption {assumption['assumption_id']!r} remains " + "unresolved and requires estimator review." + ), + ) + + _merge_supplied_review_findings(package, input_hash, findings) + acknowledgements = package.get("review", {}).get("acknowledgements", []) accepted_findings = { (acknowledgement.get("finding_id"), acknowledgement.get("input_hash")) diff --git a/skills/_shared/schemas/estimate-package.schema.json b/skills/_shared/schemas/estimate-package.schema.json index 12b6281..4b01dc3 100644 --- a/skills/_shared/schemas/estimate-package.schema.json +++ b/skills/_shared/schemas/estimate-package.schema.json @@ -16,7 +16,7 @@ "properties": { "schema_version": { "const": "1.0.0" }, "project": { "$ref": "#/$defs/project" }, - "unit_system": { "enum": ["imperial", "metric"] }, + "unit_system": { "const": "imperial" }, "items": { "type": "array", "items": { "$ref": "#/$defs/item" } @@ -26,6 +26,11 @@ "items": { "$ref": "#/$defs/stock" } }, "commercial_basis": { "$ref": "#/$defs/commercialBasis" }, + "assumptions": { + "type": "array", + "items": { "$ref": "#/$defs/assumption" }, + "default": [] + }, "review": { "$ref": "#/$defs/review" }, "lineage": { "$ref": "#/$defs/lineage" } }, @@ -37,6 +42,8 @@ "properties": { "project_id": { "type": "string", "minLength": 1 }, "name": { "type": "string" }, + "estimator": { "type": "string", "minLength": 1 }, + "estimate_as_of": { "type": "string", "format": "date" }, "revision": { "type": "object", "required": ["revision_id"], @@ -56,6 +63,8 @@ "properties": { "source": { "type": "string", "minLength": 1 }, "locator": { "type": "string", "minLength": 1 }, + "sheet": { "type": "string", "minLength": 1 }, + "detail": { "type": "string", "minLength": 1 }, "source_hash": { "type": "string", "pattern": "^[0-9a-f]{64}$" } }, "additionalProperties": false @@ -67,7 +76,7 @@ "required": ["kind", "diameter", "x", "y"], "properties": { "kind": { "const": "round" }, - "diameter": { "type": "number" }, + "diameter": { "type": "number", "exclusiveMinimum": 0 }, "x": { "type": "number" }, "y": { "type": "number" } }, @@ -78,8 +87,8 @@ "required": ["kind", "width", "height", "x", "y"], "properties": { "kind": { "const": "rect" }, - "width": { "type": "number" }, - "height": { "type": "number" }, + "width": { "type": "number", "exclusiveMinimum": 0 }, + "height": { "type": "number", "exclusiveMinimum": 0 }, "x": { "type": "number" }, "y": { "type": "number" } }, @@ -92,10 +101,10 @@ "required": ["shape", "width", "height", "thickness"], "properties": { "shape": { "type": "string" }, - "width": { "type": "number" }, - "height": { "type": "number" }, - "thickness": { "type": "number" }, - "area": { "type": "number" }, + "width": { "type": "number", "exclusiveMinimum": 0 }, + "height": { "type": "number", "exclusiveMinimum": 0 }, + "thickness": { "type": "number", "exclusiveMinimum": 0 }, + "area": { "type": "number", "exclusiveMinimum": 0 }, "holes": { "type": "array", "items": { "$ref": "#/$defs/hole" }, @@ -121,7 +130,8 @@ }, "source_id": { "type": "string", "minLength": 1 }, "item_id": { "type": "string", "minLength": 1 }, - "quantity": { "type": "integer" }, + "quantity": { "type": "integer", "minimum": 1 }, + "quantity_basis": { "$ref": "#/$defs/quantityBasis" }, "mark": { "type": "string" }, "description": { "type": "string" }, "source_evidence": { @@ -142,9 +152,9 @@ "grade": { "type": "string" }, "geometry": { "$ref": "#/$defs/geometry" }, "designation": { "type": "string" }, - "length_ft": { "type": "number" }, - "unit_weight_plf": { "type": "number" }, - "total_weight_lbs": { "type": "number" }, + "length_ft": { "type": "number", "exclusiveMinimum": 0 }, + "unit_weight_plf": { "type": "number", "exclusiveMinimum": 0 }, + "total_weight_lbs": { "type": "number", "exclusiveMinimum": 0 }, "connections": { "type": "string" }, "notes": { "type": "string" }, "identity_warning": { "type": "boolean" } @@ -164,7 +174,7 @@ "material": { "type": "string" }, "grade": { "type": "string" }, "specification": { "type": "string" }, - "dimensions": { "type": "object" } + "dimensions": { "$ref": "#/$defs/purchaseDimensions" } } } ], @@ -183,8 +193,11 @@ "required": ["kind", "value", "applies_to"], "properties": { "kind": { "enum": ["percent", "fixed_weight"] }, - "value": { "type": "number" }, - "applies_to": { "type": "string", "minLength": 1 } + "value": { "type": "number", "exclusiveMinimum": 0 }, + "applies_to": { "type": "string", "minLength": 1 }, + "source": { "type": "string", "minLength": 1 }, + "effective_date": { "type": "string", "format": "date" }, + "currency": { "type": "string", "pattern": "^[A-Z]{3}$" } }, "additionalProperties": false } @@ -231,10 +244,10 @@ "inventory_id": { "type": "string", "minLength": 1 }, "material": { "type": "string", "minLength": 1 }, "grade": { "type": "string", "minLength": 1 }, - "width": { "type": "number" }, - "height": { "type": "number" }, - "thickness": { "type": "number" }, - "quantity": { "type": "integer" }, + "width": { "type": "number", "exclusiveMinimum": 0 }, + "height": { "type": "number", "exclusiveMinimum": 0 }, + "thickness": { "type": "number", "exclusiveMinimum": 0 }, + "quantity": { "type": "integer", "minimum": 1 }, "status": { "enum": ["available", "reserved", "unavailable"] }, "measured_at": { "type": "string", "format": "date" }, "source": { "type": "string" }, @@ -270,7 +283,7 @@ ], "properties": { "cost_id": { "type": "string", "minLength": 1 }, - "amount": { "type": "number" }, + "amount": { "type": "number", "exclusiveMinimum": 0 }, "currency": { "type": "string", "pattern": "^[A-Z]{3}$" }, "unit_basis": { "type": "string", "minLength": 1 }, "effective_date": { "type": "string", "format": "date" }, @@ -283,6 +296,56 @@ }, "additionalProperties": false }, + "quantityBasis": { + "type": "object", + "required": ["method"], + "properties": { + "method": { "type": "string", "minLength": 1 }, + "description": { "type": "string", "minLength": 1 }, + "source_evidence": { + "type": "array", + "items": { "$ref": "#/$defs/evidence" } + } + }, + "additionalProperties": false + }, + "purchaseDimensions": { + "type": "object", + "properties": { + "category": { "type": "string", "minLength": 1 }, + "size": { "type": "string", "minLength": 1 }, + "purchase_weight_lbs": { + "type": "number", + "exclusiveMinimum": 0 + }, + "stock_length": { "type": "number", "exclusiveMinimum": 0 }, + "thickness": { "type": "number", "exclusiveMinimum": 0 }, + "width": { "type": "number", "exclusiveMinimum": 0 }, + "height": { "type": "number", "exclusiveMinimum": 0 }, + "replaces_item_ids": { + "type": "array", + "items": { "type": "string", "minLength": 1 }, + "uniqueItems": true + }, + "inventory_id": { "type": "string", "minLength": 1 }, + "stock_id": { "type": "string", "minLength": 1 } + }, + "additionalProperties": false + }, + "assumption": { + "type": "object", + "required": ["assumption_id", "text", "status"], + "properties": { + "assumption_id": { "type": "string", "minLength": 1 }, + "text": { "type": "string", "minLength": 1 }, + "status": { "enum": ["unresolved", "confirmed", "rejected"] }, + "source_evidence": { + "type": "array", + "items": { "$ref": "#/$defs/evidence" } + } + }, + "additionalProperties": false + }, "finding": { "type": "object", "required": [ diff --git a/skills/_shared/schemas/nest-result.schema.json b/skills/_shared/schemas/nest-result.schema.json index b1b10b0..e141a07 100644 --- a/skills/_shared/schemas/nest-result.schema.json +++ b/skills/_shared/schemas/nest-result.schema.json @@ -64,6 +64,8 @@ "required": [ "job_name", "customer", + "project_id", + "revision_id", "kerf_in", "part_gap_in", "edge_margin_in", @@ -73,10 +75,12 @@ "properties": { "job_name": { "type": "string" }, "customer": { "type": "string" }, + "project_id": { "type": "string", "minLength": 1 }, + "revision_id": { "type": "string", "minLength": 1 }, "kerf_in": { "type": "number", "minimum": 0 }, "part_gap_in": { "type": "number", "minimum": 0 }, "edge_margin_in": { "type": "number", "minimum": 0 }, - "density_lb_in3": { "type": "number", "minimum": 0 }, + "density_lb_in3": { "type": "number", "exclusiveMinimum": 0 }, "unit_system": { "type": "string" } }, "additionalProperties": false @@ -104,7 +108,7 @@ "group_id": { "type": "string", "minLength": 1 }, "material": { "type": ["string", "null"] }, "grade": { "type": ["string", "null"] }, - "thickness": { "type": "number" }, + "thickness": { "type": "number", "exclusiveMinimum": 0 }, "placements": { "type": "array", "items": { "$ref": "#/$defs/placement" } @@ -123,9 +127,9 @@ }, "additionalProperties": false }, - "total_plate_weight_lb": { "type": "number" }, - "total_part_weight_lb": { "type": "number" }, - "total_scrap_weight_lb": { "type": "number" }, + "total_plate_weight_lb": { "type": "number", "minimum": 0 }, + "total_part_weight_lb": { "type": "number", "minimum": 0 }, + "total_scrap_weight_lb": { "type": "number", "minimum": 0 }, "cost": { "type": "object", "required": ["status", "total"], @@ -133,16 +137,19 @@ "status": { "enum": ["known", "not_provided", "incomplete_unplaced"] }, - "total": { "type": ["number", "null"] } + "total": { "type": ["number", "null"], "minimum": 0 } }, "additionalProperties": false }, - "total_material_cost": { "type": ["number", "null"] }, + "total_material_cost": { + "type": ["number", "null"], + "minimum": 0 + }, "cost_known": { "type": "boolean" }, "total_holes": { "type": "integer", "minimum": 0 }, "part_net_cost": { "type": "object", - "additionalProperties": { "type": "number" } + "additionalProperties": { "type": "number", "minimum": 0 } }, "plate_reports": { "type": "array", @@ -255,8 +262,8 @@ "placement_id": { "type": "string" }, "stock_id": { "type": "string" }, "label": { "type": "string" }, - "x": { "type": "number" }, - "y": { "type": "number" }, + "x": { "type": "number", "minimum": 0 }, + "y": { "type": "number", "minimum": 0 }, "w": { "type": "number", "exclusiveMinimum": 0 }, "h": { "type": "number", "exclusiveMinimum": 0 }, "rotated": { "type": "boolean" }, @@ -264,8 +271,8 @@ "ow": { "type": "number", "exclusiveMinimum": 0 }, "oh": { "type": "number", "exclusiveMinimum": 0 }, "holes": { "type": "array", "items": { "type": "object" } }, - "base_area": { "type": "number" }, - "holes_area": { "type": "number" }, + "base_area": { "type": "number", "exclusiveMinimum": 0 }, + "holes_area": { "type": "number", "minimum": 0 }, "material": { "type": "string" }, "grade": { "type": "string" }, "thickness": { "type": "number", "exclusiveMinimum": 0 } @@ -326,10 +333,13 @@ "num_holes": { "type": "integer", "minimum": 0 }, "packing_utilization_pct": { "$ref": "#/$defs/metric" }, "net_material_yield_pct": { "$ref": "#/$defs/metric" }, - "plate_weight_lb": { "type": "number" }, - "part_weight_lb": { "type": "number" }, - "scrap_weight_lb": { "type": "number" }, - "plate_cost": { "type": ["number", "null"] }, + "plate_weight_lb": { "type": "number", "minimum": 0 }, + "part_weight_lb": { "type": "number", "minimum": 0 }, + "scrap_weight_lb": { "type": "number", "minimum": 0 }, + "plate_cost": { + "type": ["number", "null"], + "minimum": 0 + }, "cost_basis": { "type": ["string", "null"], "enum": ["per_sheet", "per_pound", null] @@ -361,16 +371,71 @@ "required": [ "schema_version", "source_nest_result_version", + "project_id", + "revision_id", + "estimate_input_hash", "geometry_readiness", "rows" ], "properties": { "schema_version": { "const": "1.0.0" }, "source_nest_result_version": { "const": "1.0.0" }, + "project_id": { "type": "string", "minLength": 1 }, + "revision_id": { "type": "string", "minLength": 1 }, + "estimate_input_hash": { "$ref": "#/$defs/hash" }, "geometry_readiness": { "enum": ["geometry_verified", "reference_only", "diagnostic"] }, - "rows": { "type": "array", "items": { "type": "object" } } + "rows": { + "type": "array", + "items": { "$ref": "#/$defs/rfqHandoffRow" } + } + }, + "additionalProperties": false + }, + "rfqHandoffRow": { + "type": "object", + "required": [ + "stock_id", + "stock_name", + "material", + "grade", + "thickness", + "sheets_needed", + "sheet_size", + "packing_utilization_pct", + "nesting_plan", + "drop_notes", + "remnant_candidates", + "total_cost", + "geometry_readiness" + ], + "properties": { + "stock_id": { "type": "string", "minLength": 1 }, + "stock_name": { "type": "string", "minLength": 1 }, + "material": { "type": "string", "minLength": 1 }, + "grade": { "type": "string", "minLength": 1 }, + "thickness": { "type": "number", "exclusiveMinimum": 0 }, + "sheets_needed": { "type": "integer", "minimum": 1 }, + "sheet_size": { "type": "string", "minLength": 1 }, + "packing_utilization_pct": { + "type": "number", + "minimum": 0, + "maximum": 100 + }, + "nesting_plan": { "type": "string", "minLength": 1 }, + "drop_notes": { "type": "string", "minLength": 1 }, + "remnant_candidates": { + "type": "array", + "items": { "$ref": "#/$defs/remnantCandidate" } + }, + "total_cost": { + "type": ["number", "null"], + "minimum": 0 + }, + "geometry_readiness": { + "enum": ["geometry_verified", "reference_only", "diagnostic"] + } }, "additionalProperties": false } diff --git a/skills/steel-estimate/SKILL.md b/skills/steel-estimate/SKILL.md index 5656290..94262c2 100644 --- a/skills/steel-estimate/SKILL.md +++ b/skills/steel-estimate/SKILL.md @@ -69,6 +69,45 @@ Exit codes: See `references/output-contract.md` for artifact and status semantics. +## Review acknowledgements + +The acknowledgement helper records a decision already made by a named human. +It never infers a disposition, chooses an actor or timestamp, or changes source +facts. Read the finding in `qa-report.json`, then supply every decision field +explicitly and write a new package: + +```bash +python3 scripts/acknowledge-finding.py \ + --input \ + --output \ + --finding-id \ + --input-hash \ + --actor "" \ + --timestamp \ + --disposition accepted +``` + +Allowed dispositions are `accepted`, `rejected`, and `deferred`. An accepted +warning is cleared only while its relevant input hash still matches. Recording +acceptance of a blocker does not waive that blocker. Re-run the estimate +pipeline with the newly written package and inspect the new QA report. + +## Dependency recovery + +If a command reports exit `4` or names a missing capability, run the package +doctor from the package root: + +```bash +python3 scripts/doctor.py --json +``` + +Install the base calculation and workbook dependencies from +`requirements.txt`. Install `requirements-render.txt` only when PDF, PNG, DXF, +or LibreOffice-assisted output is needed. `requirements-tested.txt` records the +exact dependency versions exercised by CI; it is a reproducibility reference, +not the general installation range. Run the doctor again after installation +before retrying the estimate. + ## Delivery checks - Read `run-manifest.json` and `qa-report.json`; do not infer readiness from a diff --git a/skills/steel-estimate/scripts/acknowledge-finding.py b/skills/steel-estimate/scripts/acknowledge-finding.py new file mode 100755 index 0000000..e0aa037 --- /dev/null +++ b/skills/steel-estimate/scripts/acknowledge-finding.py @@ -0,0 +1,193 @@ +#!/usr/bin/env python3 +"""Record one explicit human review decision in a canonical estimate package.""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +import tempfile +from datetime import datetime +from pathlib import Path +from typing import Any + + +SHARED_ROOT = Path(__file__).resolve().parents[2] / "_shared" +if str(SHARED_ROOT) not in sys.path: + sys.path.insert(0, str(SHARED_ROOT)) +from bootstrap import bootstrap_shared # noqa: E402 + +bootstrap_shared(__file__) +from pi_steel import canonical_json_bytes # noqa: E402 +from pi_steel.validation import ( # noqa: E402 + acknowledge_finding, + validate_estimate_package, +) + + +class AcknowledgementInputError(ValueError): + """Raised when an explicit acknowledgement cannot be recorded safely.""" + + +def _valid_timestamp(value: str) -> bool: + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return False + return "T" in value and parsed.tzinfo is not None + + +def _findings_for_decision( + package: dict[str, Any], current_hash: str +) -> list[dict[str, Any]]: + generated = validate_estimate_package(package).findings + supplied = package.get("review", {}).get("findings", []) + combined: dict[tuple[str, str], dict[str, Any]] = {} + for finding in [*generated, *supplied]: + finding_id = finding.get("finding_id") + relevant_hash = finding.get("relevant_hash") + if isinstance(finding_id, str) and isinstance(relevant_hash, str): + combined[(finding_id, relevant_hash)] = finding + return [ + finding + for finding in combined.values() + if finding.get("relevant_hash") == current_hash + ] + + +def record_acknowledgement(args: argparse.Namespace) -> dict[str, Any]: + input_path = Path(args.input).resolve() + output_path = Path(args.output).resolve() + if input_path == output_path: + raise AcknowledgementInputError( + "--output must be a new file; the source package is never overwritten" + ) + if output_path.exists(): + raise AcknowledgementInputError( + f"output already exists: {output_path}" + ) + if not args.actor.strip(): + raise AcknowledgementInputError("--actor must name the human reviewer") + if not _valid_timestamp(args.timestamp): + raise AcknowledgementInputError( + "--timestamp must be an explicit ISO-8601 date-time" + ) + + try: + package = json.loads(input_path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as exc: + raise AcknowledgementInputError( + f"estimate package could not be read: {exc}" + ) from exc + if not isinstance(package, dict): + raise AcknowledgementInputError("estimate package must be a JSON object") + + validation = validate_estimate_package(package) + if args.input_hash != validation.input_hash: + raise AcknowledgementInputError( + "--input-hash is stale for the current estimate input" + ) + matches = [ + finding + for finding in _findings_for_decision(package, validation.input_hash) + if finding["finding_id"] == args.finding_id + ] + if not matches: + stale = [ + finding + for finding in package.get("review", {}).get("findings", []) + if finding.get("finding_id") == args.finding_id + ] + if stale: + raise AcknowledgementInputError( + "finding exists but is stale for the current estimate input hash" + ) + raise AcknowledgementInputError( + "finding ID is not active for the current estimate input" + ) + if len(matches) != 1: + raise AcknowledgementInputError( + "finding ID is ambiguous for the current estimate input" + ) + finding = matches[0] + duplicate = any( + acknowledgement.get("finding_id") == args.finding_id + and acknowledgement.get("input_hash") == validation.input_hash + and acknowledgement.get("actor") == args.actor.strip() + and acknowledgement.get("timestamp") == args.timestamp + and acknowledgement.get("disposition") == args.disposition + for acknowledgement in package.get("review", {}).get( + "acknowledgements", [] + ) + ) + if duplicate: + raise AcknowledgementInputError( + "this exact acknowledgement is already recorded" + ) + + acknowledgement = acknowledge_finding( + package, + finding, + actor=args.actor.strip(), + timestamp=args.timestamp, + disposition=args.disposition, + ) + output_path.parent.mkdir(parents=True, exist_ok=True) + file_descriptor, temporary_name = tempfile.mkstemp( + prefix=f".{output_path.name}.", dir=output_path.parent + ) + try: + with os.fdopen(file_descriptor, "wb") as stream: + stream.write(canonical_json_bytes(package)) + os.link(temporary_name, output_path) + finally: + if os.path.exists(temporary_name): + os.unlink(temporary_name) + return acknowledgement + + +def parser() -> argparse.ArgumentParser: + result = argparse.ArgumentParser(description=__doc__) + result.add_argument("--input", required=True) + result.add_argument("--output", required=True) + result.add_argument("--finding-id", required=True) + result.add_argument( + "--input-hash", + required=True, + help="Relevant estimate input hash copied from the reviewed finding.", + ) + result.add_argument("--actor", required=True) + result.add_argument("--timestamp", required=True) + result.add_argument( + "--disposition", + required=True, + choices=("accepted", "rejected", "deferred"), + ) + return result + + +def main(argv: list[str] | None = None) -> int: + args = parser().parse_args(argv) + try: + acknowledgement = record_acknowledgement(args) + except AcknowledgementInputError as exc: + print(f"Acknowledgement failed: {exc}", file=sys.stderr) + return 1 + print( + json.dumps( + { + "finding_id": acknowledgement["finding_id"], + "disposition": acknowledgement["disposition"], + "input_hash": acknowledgement["input_hash"], + "recorded": True, + }, + sort_keys=True, + separators=(",", ":"), + ) + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/steel-estimate/scripts/build-estimate-package.py b/skills/steel-estimate/scripts/build-estimate-package.py index 6b7651b..1ecddf5 100755 --- a/skills/steel-estimate/scripts/build-estimate-package.py +++ b/skills/steel-estimate/scripts/build-estimate-package.py @@ -30,6 +30,7 @@ canonical_json_bytes, outcome_exit_code, package_version, + publish_failure_diagnostic, sha256_bytes, ) from pi_steel.contracts import ESTIMATE_PACKAGE_VERSION # noqa: E402 @@ -120,6 +121,7 @@ def _legacy_hole(hole: dict[str, Any]) -> dict[str, Any]: def nest_job_from_package( package: dict[str, Any], *, + estimate_input_hash: str, eligible_inventory_ids: set[str], kerf_in: float, part_gap_in: float, @@ -178,6 +180,7 @@ def nest_job_from_package( or package["project"]["project_id"], "project_id": package["project"]["project_id"], "revision_id": package["project"]["revision"]["revision_id"], + "estimate_input_hash": estimate_input_hash, "unit_system": package["unit_system"], "settings": { "kerf_in": kerf_in, @@ -299,11 +302,6 @@ def apply_inventory_consumption( for report in (nest_result or {}).get("plate_reports", []) if report["stock_id"] in eligible_inventory_ids ) - stock_by_id = { - stock["inventory_id"]: stock - for stock in package.get("stock", []) - if stock.get("inventory_id") in used - } normalized_by_id = { item["item_id"]: item for item in rfq_normalized.get("items", []) } @@ -312,31 +310,33 @@ def apply_inventory_consumption( item for item in package.get("items", []) if item.get("intent") == "purchased_stock" + and ( + (item.get("dimensions") or {}).get("inventory_id") + or (item.get("dimensions") or {}).get("stock_id") + ) ), key=lambda row: row["item_id"], ) lineage = [] for inventory_id in sorted(used): - stock = stock_by_id[inventory_id] remaining = used[inventory_id] for item in purchase_items: if remaining <= 0: break dimensions = item.get("dimensions") or {} - if not ( - item.get("material") == stock.get("material") - and item.get("grade") == stock.get("grade") - and dimensions.get("width") == stock.get("width") - and dimensions.get("height") == stock.get("height") - and dimensions.get("thickness") == stock.get("thickness") - ): + linked_stock_id = ( + dimensions.get("inventory_id") or dimensions.get("stock_id") + ) + if linked_stock_id != inventory_id: continue - satisfied = min(item["quantity"], remaining) - remaining -= satisfied rfq_item = normalized_by_id.get(item["item_id"]) if rfq_item is None: continue original_quantity = rfq_item["quantity"] + satisfied = min(original_quantity, remaining) + if satisfied <= 0: + continue + remaining -= satisfied open_quantity = original_quantity - satisfied if open_quantity: rfq_item["quantity"] = open_quantity @@ -349,6 +349,7 @@ def apply_inventory_consumption( ) else: rfq_normalized["items"].remove(rfq_item) + normalized_by_id.pop(item["item_id"], None) lineage.append( { "inventory_id": inventory_id, @@ -475,6 +476,7 @@ def build_pipeline(args) -> tuple[dict[str, Any], Path]: if not validation_blocked: nest_job = nest_job_from_package( normalized, + estimate_input_hash=validation.input_hash, eligible_inventory_ids=eligible_inventory_ids, kerf_in=args.kerf_in, part_gap_in=args.part_gap_in, @@ -782,8 +784,17 @@ def build_pipeline(args) -> tuple[dict[str, Any], Path]: def main(argv=None) -> int: parser = StageArgumentParser(description=__doc__) + parser.configure_failure_diagnostics( + stage="steel-estimate", + entry_file=__file__, + input_option="--input", + date_options={ + "--prepared-date": "prepared_date", + "--issued-date": "issued_date", + }, + ) parser.add_argument("--input", required=True) - parser.add_argument("--out", default="out") + parser.add_argument("--out", default="outputs") parser.add_argument("--prepared-date", required=True) parser.add_argument("--issued-date", required=True) parser.add_argument("--project-location", default="") @@ -798,7 +809,24 @@ def main(argv=None) -> int: try: qa_report, final_path = build_pipeline(args) except (OSError, json.JSONDecodeError, PipelineInputError) as exc: - print(f"Estimate package failed: {exc}", file=sys.stderr) + diagnostic_path = publish_failure_diagnostic( + args.out, + stage="steel-estimate", + input_path=args.input, + error=exc, + tool_version=package_version(__file__), + run_id=args.run_id, + explicit_dates={ + "prepared_date": args.prepared_date, + "issued_date": args.issued_date, + }, + ) + suffix = ( + f"; diagnostic published: {diagnostic_path}" + if diagnostic_path is not None + else "; diagnostic publication unavailable" + ) + print(f"Estimate package failed: {exc}{suffix}", file=sys.stderr) return 1 print(f"Published {qa_report['run_outcome']} estimate package: {final_path}") return outcome_exit_code(qa_report["run_outcome"]) diff --git a/skills/steel-nest/scripts/nest.py b/skills/steel-nest/scripts/nest.py index 5af27d4..826ac31 100644 --- a/skills/steel-nest/scripts/nest.py +++ b/skills/steel-nest/scripts/nest.py @@ -61,6 +61,7 @@ outcome_exit_code, package_version, placement_ids, + publish_failure_diagnostic, sha256_bytes, ) from pi_steel.contracts import content_hash, fallback_source_id, instance_ids # noqa: E402 @@ -75,6 +76,14 @@ NEST_ALGORITHM_VERSION = "maxrects-bssf-u3" +def _valid_hash(value): + return ( + isinstance(value, str) + and len(value) == 64 + and all(character in "0123456789abcdef" for character in value) + ) + + # -------------------------------------------------------------------------- # Geometry primitives # -------------------------------------------------------------------------- @@ -314,6 +323,32 @@ def number(value, path, *, positive=False, nonnegative=False): project_id = job.get("project_id") or job.get("job_name") or "LEGACY-NEST" revision_id = job.get("revision_id", "LEGACY-REVISION") + for identity_field, value in ( + ("project_id", project_id), + ("revision_id", revision_id), + ): + if not isinstance(value, str) or not value: + findings.append( + _validation_finding( + f"invalid_{identity_field}", + f"$.{identity_field}", + f"{identity_field} must be a non-empty string.", + ) + ) + if identity_field == "project_id": + project_id = "LEGACY-NEST" + else: + revision_id = "LEGACY-REVISION" + estimate_input_hash = job.get("estimate_input_hash") + if estimate_input_hash is not None and not _valid_hash(estimate_input_hash): + findings.append( + _validation_finding( + "invalid_estimate_input_hash", + "$.estimate_input_hash", + "Estimate input hash must be a lowercase SHA-256 value.", + ) + ) + estimate_input_hash = None default_material = job.get("material") default_grade = job.get("grade") default_thickness = settings.get("thickness_in") @@ -566,6 +601,9 @@ def number(value, path, *, positive=False, nonnegative=False): normalized = { "job_name": job.get("job_name", "Nesting job"), "customer": job.get("customer", ""), + "project_id": project_id, + "revision_id": revision_id, + "estimate_input_hash": estimate_input_hash, "unit_system": unit_system or "unspecified", "settings": { "kerf_in": kerf, @@ -794,6 +832,7 @@ def _summarize( validation_findings, normalized_hash, ): + estimate_input_hash = normalized.get("estimate_input_hash") or normalized_hash plate_reports = [] total_plate_area = total_packing_area = total_net_area = 0.0 total_plate_weight = total_part_weight = 0.0 @@ -997,13 +1036,15 @@ def _summarize( "schema_version": NEST_RESULT_VERSION, "algorithm_version": NEST_ALGORITHM_VERSION, "normalized_input_hash": normalized_hash, - "estimate_input_hash": normalized_hash, + "estimate_input_hash": estimate_input_hash, "configuration_hash": configuration_hash, "outcome": "blocked", "package_status": "draft", "meta": { "job_name": normalized["job_name"], "customer": normalized["customer"], + "project_id": normalized["project_id"], + "revision_id": normalized["revision_id"], "kerf_in": kerf, "part_gap_in": gap, "edge_margin_in": margin, @@ -1122,6 +1163,9 @@ def rfq_nesting_block(res): return { "schema_version": "1.0.0", "source_nest_result_version": NEST_RESULT_VERSION, + "project_id": res["meta"]["project_id"], + "revision_id": res["meta"]["revision_id"], + "estimate_input_hash": res["estimate_input_hash"], "geometry_readiness": res["geometry_readiness"], "rows": blocks, } @@ -1581,10 +1625,15 @@ def publish_nest_run(job, args): # -------------------------------------------------------------------------- def main(argv=None): ap = StageArgumentParser(description="Steel plate nesting engine") + ap.configure_failure_diagnostics( + stage="steel-nest", + entry_file=__file__, + input_option="--job", + ) ap.add_argument("--job", required=True) ap.add_argument( "--out", - default="out", + default="outputs", help="Publication root; each invocation writes an isolated runs//", ) ap.add_argument("--no-render", action="store_true", help="Skip PDF/PNG/DXF") @@ -1596,9 +1645,26 @@ def main(argv=None): ap.add_argument("--run-id", help=argparse.SUPPRESS) args = ap.parse_args(argv) - with open(args.job, encoding="utf-8") as f: - job = json.load(f) - result, qa_report, report, final_path = publish_nest_run(job, args) + try: + with open(args.job, encoding="utf-8") as f: + job = json.load(f) + result, qa_report, report, final_path = publish_nest_run(job, args) + except (OSError, json.JSONDecodeError, TypeError, ValueError) as exc: + diagnostic_path = publish_failure_diagnostic( + args.out, + stage="steel-nest", + input_path=args.job, + error=exc, + tool_version=package_version(__file__), + run_id=args.run_id, + ) + suffix = ( + f"; diagnostic published: {diagnostic_path}" + if diagnostic_path is not None + else "; diagnostic publication unavailable" + ) + print(f"Nesting failed: {exc}{suffix}", file=sys.stderr) + return 1 print(report) print(f"\nPublished {qa_report['run_outcome']} run: {final_path}") if result["burn_dxf_warnings"]: diff --git a/skills/steel-rfq/references/rfq-input.md b/skills/steel-rfq/references/rfq-input.md index 161a0b2..6c059e2 100644 --- a/skills/steel-rfq/references/rfq-input.md +++ b/skills/steel-rfq/references/rfq-input.md @@ -7,7 +7,7 @@ The deterministic compiler accepts either: these exact row-1 headers: `Source_ID`, `Item_ID`, `Scope`, `Description`, `Material`, `Grade`, - `Thickness`, `Size`, `Qty`, `Purchase Weight`. + `Thickness`, `Size`, `Qty`, `Currency`, `Purchase Weight`. The legacy adapter is intentionally narrow. `Scope` must be exactly `IN SCOPE`, `BY OTHERS`, or `EXCLUDED`; descriptions never control scope. Missing stable @@ -25,6 +25,7 @@ The optional nesting input is the versioned `rfq_nesting.json` object: { "schema_version": "1.0.0", "source_nest_result_version": "1.0.0", + "geometry_readiness": "geometry_verified", "rows": [] } ``` diff --git a/skills/steel-rfq/scripts/generate-rfq.py b/skills/steel-rfq/scripts/generate-rfq.py index daefad0..3774e8a 100755 --- a/skills/steel-rfq/scripts/generate-rfq.py +++ b/skills/steel-rfq/scripts/generate-rfq.py @@ -15,6 +15,7 @@ from typing import Any import openpyxl +import jsonschema from openpyxl import Workbook from openpyxl.drawing.image import Image from openpyxl.styles import Alignment, Border, Font, PatternFill, Side @@ -32,6 +33,7 @@ canonical_json_bytes, outcome_exit_code, package_version, + publish_failure_diagnostic, sha256_bytes, ) from pi_steel.contracts import ESTIMATE_PACKAGE_VERSION # noqa: E402 @@ -45,6 +47,9 @@ RFQ_COMPILER_VERSION = "1.0.0" NEST_HANDOFF_VERSION = "1.0.0" +NEST_RESULT_SCHEMA_PATH = ( + SHARED_ROOT / "schemas" / "nest-result.schema.json" +) HEADERS = [ "Item", "Category", @@ -235,6 +240,14 @@ def normalize_canonical_package(package: dict[str, Any]) -> dict[str, Any]: "currency": package["commercial_basis"]["currency"], "items": included, "warnings": [finding["message"] for finding in result.warnings], + "review_findings": [ + dict(finding) + for finding in result.active_findings + if finding["severity"] == "warning" + ], + "assumptions": [ + dict(assumption) for assumption in package.get("assumptions", []) + ], } @@ -356,6 +369,8 @@ def normalize_legacy_xlsx(path: str | Path) -> dict[str, Any]: "warnings": [ "Legacy XLSX mapping used exact headers; confirm typed scope before issue." ], + "review_findings": [], + "assumptions": [], } @@ -445,46 +460,90 @@ def add(code, message): return findings -def validate_nest_handoff(value: dict[str, Any] | None) -> list[dict[str, str]]: +def _nest_handoff_validator() -> jsonschema.Draft202012Validator: + schema = json.loads(NEST_RESULT_SCHEMA_PATH.read_text(encoding="utf-8")) + handoff_schema = { + "$schema": schema["$schema"], + "$ref": "#/$defs/rfqHandoff", + "$defs": schema["$defs"], + } + return jsonschema.Draft202012Validator(handoff_schema) + + +def _json_path(parts: Any) -> str: + result = "$" + for part in parts: + result += f"[{part}]" if isinstance(part, int) else f".{part}" + return result + + +def validate_nest_handoff( + value: dict[str, Any] | None, + *, + expected: dict[str, Any] | None = None, +) -> list[dict[str, str]]: if value is None: return [] - findings = [] - if value.get("schema_version") != NEST_HANDOFF_VERSION: - findings.append( + if not isinstance(value, dict): + return [ { - "code": "unsupported_nest_handoff_version", + "code": "invalid_nest_handoff", "severity": "error", - "message": "Migrate nesting handoff to version 1.0.0.", + "path": "$", + "message": "Nesting handoff must be a JSON object.", } - ) - if value.get("source_nest_result_version") != "1.0.0": + ] + findings: list[dict[str, str]] = [] + for error in sorted( + _nest_handoff_validator().iter_errors(value), + key=lambda item: tuple(str(part) for part in item.absolute_path), + ): findings.append( { - "code": "unsupported_nest_result_version", + "code": "invalid_nest_handoff_contract", "severity": "error", - "message": "Nesting result version must be 1.0.0.", + "path": _json_path(error.absolute_path), + "message": error.message, } ) - if value.get("geometry_readiness") not in { - "geometry_verified", - "reference_only", - "diagnostic", - }: - findings.append( - { - "code": "invalid_geometry_readiness", - "severity": "error", - "message": "Nesting handoff requires explicit geometry_readiness.", - } + rows = value.get("rows") + readiness_values = [value.get("geometry_readiness")] + if isinstance(rows, list): + readiness_values.extend( + row.get("geometry_readiness") + for row in rows + if isinstance(row, dict) ) - if not isinstance(value.get("rows"), list): + if "diagnostic" in readiness_values: findings.append( { - "code": "invalid_nest_rows", + "code": "diagnostic_nest_handoff", "severity": "error", - "message": "Nesting handoff rows must be an array.", + "path": "$.geometry_readiness", + "message": ( + "Diagnostic nesting output is not eligible for an RFQ; " + "resolve nest blockers and regenerate the handoff." + ), } ) + if expected is not None: + for handoff_field, expected_field in ( + ("project_id", "project_id"), + ("revision_id", "revision_id"), + ("estimate_input_hash", "input_hash"), + ): + if value.get(handoff_field) != expected.get(expected_field): + findings.append( + { + "code": "stale_nest_handoff", + "severity": "error", + "path": f"$.{handoff_field}", + "message": ( + f"Nesting handoff {handoff_field} does not match " + "the current estimate package." + ), + } + ) return findings @@ -573,6 +632,24 @@ def _safe_filename(project_name: str) -> str: return f"{stem or 'RFQ'}_RFQ_Material_List.xlsx" +def _escape_untrusted_formulas( + workbook: Workbook, + owned_formula_cells: set[tuple[str, str]], +) -> None: + """Force user-controlled formula-like strings to remain literal cells.""" + dangerous = re.compile(r"^\s*[=+\-@]") + for worksheet in workbook.worksheets: + for row in worksheet.iter_rows(): + for cell in row: + value = cell.value + if ( + isinstance(value, str) + and dangerous.match(value) + and (worksheet.title, cell.coordinate) not in owned_formula_cells + ): + cell.value = "'" + value + + def compile_workbook( normalized: dict[str, Any], profile: dict[str, Any], @@ -592,7 +669,7 @@ def compile_workbook( "company profile blocked: " + "; ".join(finding["message"] for finding in profile_findings) ) - nest_findings = validate_nest_handoff(nest_handoff) + nest_findings = validate_nest_handoff(nest_handoff, expected=normalized) if nest_findings: raise RfqInputError( "nest handoff blocked: " @@ -670,6 +747,7 @@ def compile_workbook( row = 9 material_rows = [] + owned_formula_cells: set[tuple[str, str]] = set() grouped: dict[tuple[Any, ...], list[dict[str, Any]]] = {} for item in normalized["items"]: key = ( @@ -718,6 +796,7 @@ def compile_workbook( for column, value in enumerate(values, start=1): sheet.cell(row, column, value) sheet.cell(row, 10, f'=IF(I{row}="","",F{row}*I{row})') + owned_formula_cells.add((sheet.title, f"J{row}")) for column in range(1, 15): cell = sheet.cell(row, column) cell.border = border @@ -745,6 +824,12 @@ def compile_workbook( sheet.cell(total_row, 1).alignment = Alignment(horizontal="right") sheet.cell(total_row, 8, f"=SUM(H{first_material_row}:H{last_material_row})") sheet.cell(total_row, 10, f"=SUM(J{first_material_row}:J{last_material_row})") + owned_formula_cells.update( + { + (sheet.title, f"H{total_row}"), + (sheet.title, f"J{total_row}"), + } + ) for column in (8, 10): sheet.cell(total_row, column).fill = PatternFill("solid", fgColor=green) sheet.cell(total_row, column).font = Font(name="Arial", bold=True) @@ -801,7 +886,55 @@ def compile_workbook( cell.alignment = Alignment(vertical="top", wrap_text=True) nest_row += 1 - terms_header_row = nest_row + 1 + review_header_row = nest_row + 1 + sheet.merge_cells( + start_row=review_header_row, + start_column=1, + end_row=review_header_row, + end_column=14, + ) + sheet.cell(review_header_row, 1, "REVIEW NOTES / ASSUMPTIONS") + sheet.cell(review_header_row, 1).font = Font( + name="Arial", bold=True, color=dark_blue + ) + review_row = review_header_row + 1 + review_notes = [ + f"{finding.get('finding_id', finding.get('code', 'review'))}: " + f"{finding.get('message', '')}" + for finding in normalized.get("review_findings", []) + ] + known_messages = { + finding.get("message") + for finding in normalized.get("review_findings", []) + } + review_notes.extend( + warning + for warning in normalized.get("warnings", []) + if warning not in known_messages + ) + review_notes.extend( + ( + f"Assumption {assumption.get('assumption_id', 'unidentified')} " + f"[{assumption.get('status', 'unknown')}]: {assumption.get('text', '')}" + ) + for assumption in normalized.get("assumptions", []) + ) + if not review_notes: + review_notes.append("No active review warnings or assumptions.") + for note in review_notes: + sheet.merge_cells( + start_row=review_row, + start_column=1, + end_row=review_row, + end_column=14, + ) + sheet.cell(review_row, 1, note) + sheet.cell(review_row, 1).alignment = Alignment( + wrap_text=True, vertical="top" + ) + review_row += 1 + + terms_header_row = review_row + 1 sheet.merge_cells( start_row=terms_header_row, start_column=1, end_row=terms_header_row, end_column=14 ) @@ -852,6 +985,7 @@ def compile_workbook( else: logo_status = "text_fallback" + _escape_untrusted_formulas(workbook, owned_formula_cells) output_directory = Path(output_directory) output_directory.mkdir(parents=True, exist_ok=True) workbook_path = output_directory / _safe_filename(normalized["project_name"]) @@ -867,6 +1001,7 @@ def compile_workbook( "last_material_row": last_material_row, "total_row": total_row, "nest_header_row": nest_header_row, + "review_header_row": review_header_row, "terms_header_row": terms_header_row, }, } @@ -894,7 +1029,7 @@ def publish_rfq_run(args) -> tuple[dict[str, Any], Path]: ) try: normalized = _load_normalized(input_path) - except (RfqInputError, json.JSONDecodeError) as exc: + except RfqInputError as exc: normalized = { "input_kind": "invalid", "input_version": "unknown", @@ -905,6 +1040,8 @@ def publish_rfq_run(args) -> tuple[dict[str, Any], Path]: "currency": "USD", "items": [], "warnings": [], + "review_findings": [], + "assumptions": [], } input_findings.append( { @@ -933,7 +1070,10 @@ def publish_rfq_run(args) -> tuple[dict[str, Any], Path]: if args.nest else None ) - nest_findings = validate_nest_handoff(nest_handoff) + nest_findings = validate_nest_handoff( + nest_handoff, + expected=normalized, + ) except (OSError, json.JSONDecodeError) as exc: nest_handoff = None nest_findings = [ @@ -1039,9 +1179,15 @@ def publish_rfq_run(args) -> tuple[dict[str, Any], Path]: def main(argv=None) -> int: parser = StageArgumentParser(description=__doc__) + parser.configure_failure_diagnostics( + stage="steel-rfq", + entry_file=__file__, + input_option="--input", + date_options={"--issued-date": "issued_date"}, + ) parser.add_argument("--input", required=True) parser.add_argument("--nest") - parser.add_argument("--out", default="out") + parser.add_argument("--out", default="outputs") parser.add_argument("--issued-date", required=True) parser.add_argument("--project-location", default="") parser.add_argument("--no-bake", action="store_true") @@ -1050,7 +1196,21 @@ def main(argv=None) -> int: try: qa_report, final_path = publish_rfq_run(args) except (OSError, json.JSONDecodeError, RfqInputError) as exc: - print(f"RFQ generation failed: {exc}", file=sys.stderr) + diagnostic_path = publish_failure_diagnostic( + args.out, + stage="steel-rfq", + input_path=args.input, + error=exc, + tool_version=package_version(__file__), + run_id=args.run_id, + explicit_dates={"issued_date": args.issued_date}, + ) + suffix = ( + f"; diagnostic published: {diagnostic_path}" + if diagnostic_path is not None + else "; diagnostic publication unavailable" + ) + print(f"RFQ generation failed: {exc}{suffix}", file=sys.stderr) return 1 print(f"Published {qa_report['run_outcome']} draft RFQ run: {final_path}") return outcome_exit_code(qa_report["run_outcome"]) diff --git a/skills/steel-rfq/scripts/recalc.py b/skills/steel-rfq/scripts/recalc.py index 5c635d6..5d0c432 100644 --- a/skills/steel-rfq/scripts/recalc.py +++ b/skills/steel-rfq/scripts/recalc.py @@ -21,14 +21,23 @@ import tempfile +class RecalculationError(RuntimeError): + """Raised when recalculate-on-open cannot be recorded safely.""" + + def flag_recalc_on_load(path): import openpyxl - wb = openpyxl.load_workbook(path) + try: + wb = openpyxl.load_workbook(path) wb.calculation.fullCalcOnLoad = True - except Exception: - pass # older openpyxl — the LibreOffice bake below still fixes values - wb.save(path) + wb.calculation.forceFullCalc = True + wb.calculation.calcMode = "auto" + wb.save(path) + except Exception as exc: + raise RecalculationError( + f"could not record recalculate-on-open state for {path}: {exc}" + ) from exc def libreoffice_bake(path): @@ -62,19 +71,26 @@ def recalculate(path, *, bake=True): def main(): if len(sys.argv) < 2: - sys.exit("usage: python3 recalc.py ") + print("usage: python3 recalc.py ", file=sys.stderr) + return 1 path = sys.argv[1] if not os.path.exists(path): - sys.exit(f"file not found: {path}") + print(f"file not found: {path}", file=sys.stderr) + return 1 # Set recalc-on-open FIRST so LibreOffice honors it, then bake. Do NOT # reload with openpyxl afterward — that would strip the cached values # LibreOffice just computed. - status = recalculate(path) + try: + status = recalculate(path) + except RecalculationError as exc: + print(f"recalc: failed — {exc}", file=sys.stderr) + return 1 if status == "baked_via_libreoffice": print(f"recalc: values computed and baked via LibreOffice — {path}") else: print(f"recalc: recalc-on-open set (open in Excel to compute values) — {path}") + return 0 if __name__ == "__main__": - main() + raise SystemExit(main()) diff --git a/skills/steel-takeoff/SKILL.md b/skills/steel-takeoff/SKILL.md index 449ab1e..7f7c7ca 100644 --- a/skills/steel-takeoff/SKILL.md +++ b/skills/steel-takeoff/SKILL.md @@ -72,7 +72,9 @@ For each unique member mark, create a line item: cat assets/bom-template.csv ``` -Fields: Mark, Qty, Size, Grade, Length_ft, Unit_Wt_plf, Total_Wt_lbs, Connections, Notes +Fields: Source_ID, Mark, Qty, Size, Grade, Length_ft, Unit_Wt_plf, +Total_Wt_lbs, Connections, Notes, Source_Sheet, Source_Detail, Intent. +Use stable synthetic or source-system IDs; do not invent drawing evidence. ### Step 3 — Look Up Unit Weights For every member size in the BOM, look up the unit weight from the AISC database: @@ -96,7 +98,9 @@ For every member size in the BOM, look up the unit weight from the AISC database # Total weight in lbs and tons # Connection allowance (12%) # Misc steel allowance (5%) -# Grand total with estimated cost +# Grand total weight +# No pricing unless a separate project input supplies an explicit currency, +# unit basis, effective date, and source ``` ### Step 5 — Add Connections & Misc Steel diff --git a/tests/fixtures/pipeline/synthetic-estimate.json b/tests/fixtures/pipeline/synthetic-estimate.json index 94a7200..fbabbcf 100644 --- a/tests/fixtures/pipeline/synthetic-estimate.json +++ b/tests/fixtures/pipeline/synthetic-estimate.json @@ -120,6 +120,7 @@ "thickness": 0.5, "size": "24 x 12 x 0.5", "purchase_weight_lbs": 41, + "inventory_id": "SYNTHETIC-STOCK-A36", "replaces_item_ids": ["item:synthetic-pipeline-p1"] }, "source_evidence": [ @@ -146,6 +147,7 @@ "thickness": 0.375, "size": "20 x 10 x 0.375", "purchase_weight_lbs": 21, + "inventory_id": "SYNTHETIC-STOCK-A572", "replaces_item_ids": ["item:synthetic-pipeline-p2"] }, "source_evidence": [ diff --git a/tests/fixtures/rfq/nest-handoff.json b/tests/fixtures/rfq/nest-handoff.json index 4c69f66..f6c89a3 100644 --- a/tests/fixtures/rfq/nest-handoff.json +++ b/tests/fixtures/rfq/nest-handoff.json @@ -1,6 +1,9 @@ { "schema_version": "1.0.0", "source_nest_result_version": "1.0.0", + "project_id": "SYNTHETIC-RFQ-001", + "revision_id": "SYNTHETIC-REV-A", + "estimate_input_hash": "9200952fc62e897d5ddd2247cf0b10061c62decaed9a392ca2134aba12576903", "geometry_readiness": "reference_only", "rows": [ { diff --git a/tests/golden/rfq/semantic-workbook.json b/tests/golden/rfq/semantic-workbook.json index 100fbeb..cd46a20 100644 --- a/tests/golden/rfq/semantic-workbook.json +++ b/tests/golden/rfq/semantic-workbook.json @@ -1,12 +1,12 @@ { "projection_format": "1.0.0", - "semantic_sha256": "2414a162bfd1b2efe4c270232353fed71f3ca8f578a9848262127a1aca7b2a25", + "semantic_sha256": "e5a2dae82dfb653cfb951ed273993e719005e4d8d3a768ec468dd27cc2e9e22f", "sheets": [ { "title": "RFQ Draft", - "max_row": 23, + "max_row": 26, "max_column": 14, - "cell_count": 95, + "cell_count": 97, "style_count": 25 }, { diff --git a/tests/test_acknowledge_finding_cli.py b/tests/test_acknowledge_finding_cli.py new file mode 100644 index 0000000..4e472d2 --- /dev/null +++ b/tests/test_acknowledge_finding_cli.py @@ -0,0 +1,159 @@ +import copy +import json +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SHARED = ROOT / "skills" / "_shared" +sys.path.insert(0, str(SHARED)) + +from pi_steel.validation import validate_estimate_package + + +SCRIPT = ( + ROOT + / "skills" + / "steel-estimate" + / "scripts" + / "acknowledge-finding.py" +) +FIXTURE = ROOT / "tests" / "fixtures" / "pipeline" / "synthetic-estimate.json" + + +def package_with_warning(): + package = json.loads(FIXTURE.read_text(encoding="utf-8")) + package["items"][0].pop("source_evidence") + finding = next( + finding + for finding in validate_estimate_package(package).active_findings + if finding["code"] == "missing_source_evidence" + ) + return package, finding + + +def run_acknowledgement( + input_path, + output_path, + finding_id, + input_hash, + *, + disposition="accepted", +): + return subprocess.run( + [ + sys.executable, + SCRIPT, + "--input", + input_path, + "--output", + output_path, + "--finding-id", + finding_id, + "--input-hash", + input_hash, + "--actor", + "Synthetic Reviewer", + "--timestamp", + "2026-07-28T12:00:00Z", + "--disposition", + disposition, + ], + capture_output=True, + text=True, + ) + + +def test_acknowledgement_cli_records_explicit_decision_in_new_package(tmp_path): + package, finding = package_with_warning() + source = tmp_path / "synthetic-source.json" + output = tmp_path / "synthetic-reviewed.json" + source.write_text(json.dumps(package), encoding="utf-8") + original = source.read_bytes() + + result = run_acknowledgement( + source, + output, + finding["finding_id"], + finding["relevant_hash"], + disposition="accepted", + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert source.read_bytes() == original + response = json.loads(result.stdout) + assert response == { + "disposition": "accepted", + "finding_id": finding["finding_id"], + "input_hash": finding["relevant_hash"], + "recorded": True, + } + reviewed = json.loads(output.read_text(encoding="utf-8")) + assert reviewed["review"]["acknowledgements"] == [ + { + "actor": "Synthetic Reviewer", + "disposition": "accepted", + "finding_id": finding["finding_id"], + "input_hash": finding["relevant_hash"], + "timestamp": "2026-07-28T12:00:00Z", + } + ] + assert not any( + active["finding_id"] == finding["finding_id"] + for active in validate_estimate_package(reviewed).active_findings + ) + + +def test_rejected_decision_is_recorded_without_clearing_warning(tmp_path): + package, finding = package_with_warning() + source = tmp_path / "synthetic-source.json" + output = tmp_path / "synthetic-reviewed.json" + source.write_text(json.dumps(package), encoding="utf-8") + + result = run_acknowledgement( + source, + output, + finding["finding_id"], + finding["relevant_hash"], + disposition="rejected", + ) + + assert result.returncode == 0, result.stdout + result.stderr + reviewed = json.loads(output.read_text(encoding="utf-8")) + assert any( + active["finding_id"] == finding["finding_id"] + for active in validate_estimate_package(reviewed).active_findings + ) + + +def test_acknowledgement_cli_rejects_stale_finding_and_existing_output( + tmp_path, +): + package, finding = package_with_warning() + stale = copy.deepcopy(package) + stale["project"]["revision"]["revision_id"] = "SYNTHETIC-REV-CHANGED" + source = tmp_path / "synthetic-source.json" + source.write_text(json.dumps(stale), encoding="utf-8") + output = tmp_path / "synthetic-reviewed.json" + + stale_result = run_acknowledgement( + source, output, finding["finding_id"], finding["relevant_hash"] + ) + + assert stale_result.returncode == 1 + assert not output.exists() + assert "--input-hash is stale" in stale_result.stderr + + package_source = tmp_path / "synthetic-current.json" + package_source.write_text(json.dumps(package), encoding="utf-8") + output.write_text("synthetic existing output", encoding="utf-8") + existing_result = run_acknowledgement( + package_source, + output, + finding["finding_id"], + finding["relevant_hash"], + ) + assert existing_result.returncode == 1 + assert output.read_text(encoding="utf-8") == "synthetic existing output" + assert "output already exists" in existing_result.stderr diff --git a/tests/test_contracts.py b/tests/test_contracts.py index b44ceeb..a147859 100644 --- a/tests/test_contracts.py +++ b/tests/test_contracts.py @@ -106,6 +106,198 @@ def test_contract_schemas_are_draft_2020_12_and_accept_canonical_package(): ) +def test_v1_contract_rejects_metric_and_nonpositive_commercial_values(): + package = valid_package() + package["unit_system"] = "metric" + assert "schema_validation" in blocker_codes( + validate_estimate_package(package) + ) + + package = valid_package() + package["commercial_basis"]["costs"].append( + { + "cost_id": "SYNTHETIC-ZERO-COST", + "amount": 0, + "currency": "USD", + "unit_basis": "per_pound", + "effective_date": "2026-07-28", + "source": "synthetic_test_input", + } + ) + assert "schema_validation" in blocker_codes( + validate_estimate_package(package) + ) + + +def test_purchase_dimensions_are_closed_typed_and_support_inventory_linkage(): + package = valid_package() + package["items"].append( + { + "intent": "purchased_stock", + "source_id": "SYNTHETIC-SRC-STOCK", + "item_id": "item:synthetic-stock", + "quantity": 1, + "material": "carbon_steel", + "grade": "A36", + "dimensions": { + "category": "PLATE STOCK", + "size": "48 x 96 x 0.5", + "purchase_weight_lbs": 652, + "stock_length": 96, + "width": 48, + "height": 96, + "thickness": 0.5, + "replaces_item_ids": [package["items"][0]["item_id"]], + "inventory_id": "SYNTHETIC-INV-1", + "stock_id": "SYNTHETIC-STOCK-1", + }, + } + ) + assert validate_estimate_package(package).status == "review_required" + + package["items"][-1]["dimensions"]["purchase_weight_lbs"] = -1 + assert "schema_validation" in blocker_codes( + validate_estimate_package(package) + ) + package["items"][-1]["dimensions"]["purchase_weight_lbs"] = 652 + package["items"][-1]["dimensions"]["untyped_business_field"] = "private" + assert "schema_validation" in blocker_codes( + validate_estimate_package(package) + ) + + +def test_nonfinite_number_guard_covers_nested_non_geometry_numbers(): + package = valid_package() + package["commercial_basis"]["costs"].append( + { + "cost_id": "SYNTHETIC-NAN-COST", + "amount": math.nan, + "currency": "USD", + "unit_basis": "per_pound", + "effective_date": "2026-07-28", + "source": "synthetic_test_input", + } + ) + result = validate_estimate_package(package) + assert "nonfinite_number" in blocker_codes(result) + assert any( + finding["path"] == "$.commercial_basis.costs[0].amount" + for finding in result.findings + ) + + +def test_traceability_fields_and_unresolved_assumptions_are_review_visible(): + package = valid_package() + package["project"].update( + estimator="Synthetic Estimator", + estimate_as_of="2026-07-28", + ) + package["items"][0]["quantity_basis"] = { + "method": "drawing_count", + "description": "Counted from the synthetic detail.", + "source_evidence": [ + { + "source": "SYNTHETIC-DRAWING", + "locator": "SYNTHETIC-DETAIL-1", + "sheet": "S1", + "detail": "1", + } + ], + } + package["assumptions"] = [ + { + "assumption_id": "SYNTHETIC-ASSUMPTION-1", + "text": "Synthetic finish remains to be confirmed.", + "status": "unresolved", + } + ] + result = validate_estimate_package(package) + assert result.status == "review_required" + assert "unresolved_assumption" in warning_codes(result) + + +def test_supplied_review_findings_merge_and_stale_hash_is_a_blocker(): + package = valid_package() + current_hash = validate_estimate_package(package).input_hash + supplied = { + "finding_id": "finding:synthetic-supplied", + "code": "synthetic_review_note", + "severity": "warning", + "path": "$.items[0]", + "message": "Synthetic estimator note.", + "relevant_hash": current_hash, + } + package["review"]["findings"] = [supplied] + current = validate_estimate_package(package) + assert supplied in current.findings + assert supplied in current.active_findings + + package["review"]["findings"][0]["relevant_hash"] = "c" * 64 + stale = validate_estimate_package(package) + assert "stale_review_finding" in blocker_codes(stale) + stale_finding = next( + finding + for finding in stale.findings + if finding["code"] == "stale_review_finding" + ) + acknowledge_finding( + package, + stale_finding, + actor="Example Reviewer", + timestamp="2026-07-28T12:00:00Z", + disposition="accepted", + ) + assert "stale_review_finding" in blocker_codes( + validate_estimate_package(package) + ) + + +def test_rfq_handoff_schema_requires_lineage_and_closed_typed_rows(): + nest_schema = json.loads( + (SHARED / "schemas" / "nest-result.schema.json").read_text() + ) + handoff_schema = { + "$schema": nest_schema["$schema"], + "$ref": "#/$defs/rfqHandoff", + "$defs": nest_schema["$defs"], + } + validator = jsonschema.Draft202012Validator(handoff_schema) + handoff = { + "schema_version": "1.0.0", + "source_nest_result_version": "1.0.0", + "project_id": "SYNTHETIC-CONTRACT-001", + "revision_id": "SYNTHETIC-REV-A", + "estimate_input_hash": "a" * 64, + "geometry_readiness": "geometry_verified", + "rows": [ + { + "stock_id": "SYNTHETIC-STOCK-1", + "stock_name": "Synthetic Plate", + "material": "carbon_steel", + "grade": "A36", + "thickness": 0.5, + "sheets_needed": 1, + "sheet_size": "48x96", + "packing_utilization_pct": 75, + "nesting_plan": "1 synthetic sheet", + "drop_notes": "No certified remnants.", + "remnant_candidates": [], + "total_cost": None, + "geometry_readiness": "geometry_verified", + } + ], + } + validator.validate(handoff) + + missing_lineage = copy.deepcopy(handoff) + del missing_lineage["estimate_input_hash"] + assert list(validator.iter_errors(missing_lineage)) + malformed_row = copy.deepcopy(handoff) + malformed_row["rows"][0]["sheets_needed"] = "one" + malformed_row["rows"][0]["private_note"] = "must not pass through" + assert list(validator.iter_errors(malformed_row)) + + def test_legacy_bom_preserves_marks_grades_lengths_weights_and_explicit_ids(): package = adapt_legacy_bom_csv( FIXTURES / "legacy-bom.csv", diff --git a/tests/test_estimate_pipeline.py b/tests/test_estimate_pipeline.py index c9438ef..acc3375 100644 --- a/tests/test_estimate_pipeline.py +++ b/tests/test_estimate_pipeline.py @@ -1,4 +1,5 @@ import json +import importlib.util import os import subprocess import sys @@ -22,6 +23,10 @@ ) FIXTURES = ROOT / "tests" / "fixtures" / "pipeline" GOLDEN = ROOT / "tests" / "golden" / "pipeline" / "ready-artifacts.json" +SPEC = importlib.util.spec_from_file_location("pi_steel_estimate_pipeline", SCRIPT) +estimate_pipeline = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = estimate_pipeline +SPEC.loader.exec_module(estimate_pipeline) def load_package(): @@ -98,6 +103,12 @@ def test_ready_pipeline_matches_artifact_contract_and_preserves_scope(tmp_path): "exclusion", } nest = load_json(run_path / "nest-result.json") + handoff = load_json(run_path / "rfq-nesting.json") + assert nest["estimate_input_hash"] == manifest["input_hash"] + assert nest["normalized_input_hash"] != nest["estimate_input_hash"] + assert handoff["project_id"] == "SYNTHETIC-PIPELINE-001" + assert handoff["revision_id"] == "SYNTHETIC-REV-A" + assert handoff["estimate_input_hash"] == manifest["input_hash"] assert { (group["grade"], group["thickness"]) for group in nest["groups"] } == {("A36", 0.5), ("A572", 0.375)} @@ -158,7 +169,7 @@ def test_blockers_preserve_diagnostics_but_never_publish_workbook(tmp_path, bloc if blocker == "validation": package["items"][0]["quantity"] = 0 elif blocker == "unplaced": - package["stock"][0]["quantity"] = 0 + package["items"][2]["quantity"] = 100 else: profile = False completed, run_path = run_pipeline( @@ -295,6 +306,62 @@ def test_confirmed_on_hand_stock_is_consumed_without_duplicate_rfq_demand(tmp_pa assert not any(item_id in workbook_text for item_id in purchase_ids) +def test_inventory_consumption_uses_explicit_links_and_never_removes_twice(): + package = { + "items": [ + { + "intent": "purchased_stock", + "item_id": "item:synthetic-stock-a", + "quantity": 1, + "dimensions": {"inventory_id": "SYNTHETIC-INV-A"}, + }, + { + "intent": "purchased_stock", + "item_id": "item:synthetic-stock-b", + "quantity": 1, + "dimensions": {"inventory_id": "SYNTHETIC-INV-B"}, + }, + { + "intent": "purchased_stock", + "item_id": "item:synthetic-unlinked", + "quantity": 1, + "dimensions": {}, + }, + ] + } + normalized = { + "items": [ + { + "item_id": item["item_id"], + "quantity": item["quantity"], + "purchase_weight_lbs": 10, + } + for item in package["items"] + ] + } + nest_result = { + "plate_reports": [ + {"stock_id": "SYNTHETIC-INV-A"}, + {"stock_id": "SYNTHETIC-INV-B"}, + ] + } + + lineage = estimate_pipeline.apply_inventory_consumption( + package, + normalized, + nest_result, + {"SYNTHETIC-INV-A", "SYNTHETIC-INV-B"}, + ) + + assert {row["purchase_item_id"] for row in lineage} == { + "item:synthetic-stock-a", + "item:synthetic-stock-b", + } + assert [item["item_id"] for item in normalized["items"]] == [ + "item:synthetic-unlinked" + ] + + def test_wrong_container_type_still_publishes_blocked_diagnostics(tmp_path): package = load_package() package["project"] = [] diff --git a/tests/test_full_render_smoke.py b/tests/test_full_render_smoke.py index 8d6ca4d..823d829 100644 --- a/tests/test_full_render_smoke.py +++ b/tests/test_full_render_smoke.py @@ -4,6 +4,7 @@ import shutil import subprocess import sys +from xml.etree import ElementTree from copy import deepcopy from pathlib import Path @@ -25,24 +26,43 @@ def require_full_environment(): if importlib.util.find_spec(module) is None ] office = shutil.which("soffice") or shutil.which("libreoffice") - pdf_text = shutil.which("pdftotext") - unavailable = missing + ([] if office else ["LibreOffice"]) + ( - [] if pdf_text else ["pdftotext"] + commands = { + "pdftotext": shutil.which("pdftotext"), + "pdfinfo": shutil.which("pdfinfo"), + "pdftoppm": shutil.which("pdftoppm"), + "pdfimages": shutil.which("pdfimages"), + } + unavailable = ( + missing + + ([] if office else ["LibreOffice"]) + + [name for name, path in commands.items() if path is None] ) if unavailable: message = "full render requires " + ", ".join(unavailable) if os.environ.get("PI_STEEL_REQUIRE_FULL_RENDER") == "1": pytest.fail(message) pytest.skip(message) - return office + return {"office": office, **commands} def run_rendered(tmp_path, package, run_id): + import matplotlib.image + import numpy + input_path = tmp_path / f"{run_id}.json" input_path.write_text(json.dumps(package), encoding="utf-8") + logo_path = tmp_path / "synthetic-logo.png" + logo = numpy.ones((40, 120, 3), dtype=float) + logo[4:36, 4:116] = (1.0, 0.0, 1.0) + logo[12:28, 16:104] = (0.0, 1.0, 1.0) + matplotlib.image.imsave(logo_path, logo) + profile = json.loads((FIXTURES / "synthetic-profile.json").read_text()) + profile["logo"] = str(logo_path) + profile_path = tmp_path / "synthetic-profile.json" + profile_path.write_text(json.dumps(profile), encoding="utf-8") output = tmp_path / "published" environment = os.environ.copy() - environment["PI_STEEL_CONFIG"] = str(FIXTURES / "synthetic-profile.json") + environment["PI_STEEL_CONFIG"] = str(profile_path) completed = subprocess.run( [ sys.executable, @@ -70,7 +90,7 @@ def run_rendered(tmp_path, package, run_id): def test_ready_package_renders_reference_and_verified_outputs(tmp_path): - office = require_full_environment() + tools = require_full_environment() package = json.loads((FIXTURES / "synthetic-estimate.json").read_text()) completed, run_path = run_rendered( tmp_path, package, "SYNTHETIC-FULL-RENDER-READY" @@ -82,13 +102,14 @@ def test_ready_package_renders_reference_and_verified_outputs(tmp_path): assert list(run_path.glob("burn_plate_*.dxf")) qa = json.loads((run_path / "qa-report.json").read_text()) assert qa["rfq"]["recalculation_status"] == "baked_via_libreoffice" + assert qa["rfq"]["logo_status"] == "embedded" workbook = next(run_path.glob("*.xlsx")) pdf_output = tmp_path / "workbook-pdf" pdf_output.mkdir() converted = subprocess.run( [ - office, + tools["office"], "--headless", "--convert-to", "pdf", @@ -104,9 +125,23 @@ def test_ready_package_renders_reference_and_verified_outputs(tmp_path): workbook_pdf = pdf_output / f"{workbook.stem}.pdf" assert workbook_pdf.read_bytes().startswith(b"%PDF") assert workbook_pdf.stat().st_size > 1_000 + + info = subprocess.run( + [tools["pdfinfo"], workbook_pdf], + capture_output=True, + text=True, + check=True, + ).stdout + page_count = next( + int(line.split(":", 1)[1]) + for line in info.splitlines() + if line.startswith("Pages:") + ) + assert page_count == 1 + text_output = tmp_path / "workbook.txt" extracted = subprocess.run( - ["pdftotext", "-layout", workbook_pdf, text_output], + [tools["pdftotext"], "-layout", workbook_pdf, text_output], capture_output=True, text=True, ) @@ -120,6 +155,100 @@ def test_ready_package_renders_reference_and_verified_outputs(tmp_path): ): assert expected in rendered_text + bbox_output = tmp_path / "workbook-bbox.html" + subprocess.run( + [tools["pdftotext"], "-bbox-layout", workbook_pdf, bbox_output], + capture_output=True, + text=True, + check=True, + ) + root = ElementTree.parse(bbox_output).getroot() + pages = [element for element in root.iter() if element.tag.endswith("page")] + words = [element for element in root.iter() if element.tag.endswith("word")] + assert len(pages) == 1 + page_width = float(pages[0].attrib["width"]) + page_height = float(pages[0].attrib["height"]) + assert words + for word in words: + assert 0 <= float(word.attrib["xMin"]) < float(word.attrib["xMax"]) <= page_width + assert 0 <= float(word.attrib["yMin"]) < float(word.attrib["yMax"]) <= page_height + + word_positions = { + (word.text or "").upper(): float(word.attrib["yMin"]) for word in words + } + assert word_positions["REQUEST"] < page_height * 0.2 + assert word_positions["RESPONSE"] < word_positions["TOTAL"] + assert word_positions["TOTAL"] < word_positions["TERMS"] + assert word_positions["TERMS"] < page_height * 0.95 + + images = subprocess.run( + [tools["pdfimages"], "-list", workbook_pdf], + capture_output=True, + text=True, + check=True, + ).stdout + image_rows = [ + line for line in images.splitlines() + if line.strip() and line.lstrip()[0].isdigit() + ] + assert image_rows, "the runtime synthetic logo was not embedded in the PDF" + + raster_base = tmp_path / "workbook-page" + subprocess.run( + [ + tools["pdftoppm"], + "-f", + "1", + "-l", + "1", + "-singlefile", + "-png", + workbook_pdf, + raster_base, + ], + capture_output=True, + text=True, + check=True, + ) + import matplotlib.image + import numpy + + raster = matplotlib.image.imread(raster_base.with_suffix(".png")) + occupied = numpy.any(raster[..., :3] < 0.98, axis=2) + rows, columns = numpy.where(occupied) + assert rows.size and columns.size + assert rows.min() > 1 and columns.min() > 1 + assert rows.max() < occupied.shape[0] - 2 + assert columns.max() < occupied.shape[1] - 2 + + synthetic_logo = ( + (raster[..., 0] > 0.8) + & (raster[..., 1] < 0.2) + & (raster[..., 2] > 0.8) + ) + logo_rows, logo_columns = numpy.where(synthetic_logo) + assert logo_rows.size and logo_columns.size + logo_bounds = ( + logo_columns.min() / raster.shape[1] * page_width, + logo_rows.min() / raster.shape[0] * page_height, + logo_columns.max() / raster.shape[1] * page_width, + logo_rows.max() / raster.shape[0] * page_height, + ) + assert logo_bounds[1] < page_height * 0.2 + for word in words: + if (word.text or "").upper() not in {"RESPONSE", "TOTAL", "TERMS"}: + continue + word_bounds = tuple( + float(word.attrib[field]) + for field in ("xMin", "yMin", "xMax", "yMax") + ) + assert ( + logo_bounds[2] <= word_bounds[0] + or word_bounds[2] <= logo_bounds[0] + or logo_bounds[3] <= word_bounds[1] + or word_bounds[3] <= logo_bounds[1] + ), f"synthetic logo overlaps key text {word.text!r}" + def test_irregular_render_never_publishes_burn_authority(tmp_path): require_full_environment() diff --git a/tests/test_installed_scripts.py b/tests/test_installed_scripts.py index 638e5da..e857f1f 100644 --- a/tests/test_installed_scripts.py +++ b/tests/test_installed_scripts.py @@ -26,6 +26,7 @@ def test_packed_npm_artifact_contains_runtime_and_runs_doctor(tmp_path): expected = { "package/scripts/doctor.py", + "package/scripts/check-public-data.py", "package/skills/_shared/bootstrap.py", "package/skills/_shared/pi_steel/__init__.py", "package/skills/_shared/pi_steel/run_manifest.py", @@ -76,6 +77,18 @@ def test_packed_npm_artifact_contains_runtime_and_runs_doctor(tmp_path): assert provenance.returncode == 0, provenance.stdout + provenance.stderr assert json.loads(provenance.stdout)["release_readiness"] == "blocked" + privacy = subprocess.run( + [ + sys.executable, + installed_root / "scripts" / "check-public-data.py", + ], + cwd=installed_root, + env=environment, + capture_output=True, + text=True, + ) + assert privacy.returncode == 0, privacy.stdout + privacy.stderr + environment["PI_STEEL_CONFIG"] = str( ROOT / "tests" / "fixtures" / "pipeline" / "synthetic-profile.json" ) diff --git a/tests/test_nest_cli_contract.py b/tests/test_nest_cli_contract.py index 6459d3e..b6ff4de 100644 --- a/tests/test_nest_cli_contract.py +++ b/tests/test_nest_cli_contract.py @@ -87,6 +87,9 @@ def test_ready_cli_publishes_versioned_result_handoff_and_manifest(tmp_path): assert len(result["normalized_input_hash"]) == 64 handoff = json.loads((run_path / "rfq_nesting.json").read_text()) assert handoff["schema_version"] == "1.0.0" + assert handoff["project_id"] == "SYNTHETIC-CLI-CONTRACT" + assert handoff["revision_id"] == "LEGACY-REVISION" + assert handoff["estimate_input_hash"] == result["normalized_input_hash"] assert handoff["rows"][0]["grade"] == "A36" manifest = json.loads((run_path / "run-manifest.json").read_text()) assert manifest["schema_versions"]["nest_result"] == "1.0.0" diff --git a/tests/test_nest_invariants.py b/tests/test_nest_invariants.py index f9ebad0..45143aa 100644 --- a/tests/test_nest_invariants.py +++ b/tests/test_nest_invariants.py @@ -1,4 +1,5 @@ import importlib.util +import random import sys from copy import deepcopy from pathlib import Path @@ -13,6 +14,7 @@ sys.modules[SPEC.name] = nest SPEC.loader.exec_module(nest) +import pi_steel.geometry_verify as geometry_verify from pi_steel.geometry_verify import verify_nest_placements @@ -87,3 +89,90 @@ def test_verifier_rejects_overlap_bounds_clearance_and_material_mismatch(): ) } assert {"placement_out_of_bounds", "placement_overlap", "material_mismatch"} <= codes + + +def _brute_force_overlap_paths(placements, clearance, epsilon=1e-9): + paths = [] + for first_index, first in enumerate(placements): + for second_index in range(first_index + 1, len(placements)): + second = placements[second_index] + if not ( + geometry_verify._finite_placement_values(first) + and geometry_verify._finite_placement_values(second) + ): + continue + if not geometry_verify._placements_separated( + first, second, clearance, epsilon + ): + paths.append( + f"$.plate_reports[0].placements[{first_index},{second_index}]" + ) + return paths + + +def test_sweep_line_overlap_results_match_all_pairs_reference(): + generator = random.Random(20260728) + placements = [ + { + "x": generator.uniform(0, 90), + "y": generator.uniform(0, 40), + "w": generator.uniform(0.25, 8), + "h": generator.uniform(0.25, 8), + "material": "carbon_steel", + "grade": "A36", + "thickness": 0.5, + } + for _ in range(150) + ] + plate = { + "W": 100, + "H": 50, + "material": "carbon_steel", + "grade": "A36", + "thickness": 0.5, + "placements": placements, + } + actual = [ + finding["path"] + for finding in verify_nest_placements( + [plate], edge_margin=0, inter_part_clearance=0.25 + ) + if finding["code"] == "placement_overlap" + ] + assert actual == _brute_force_overlap_paths(placements, 0.25) + + +def test_sweep_line_avoids_all_pairs_on_large_separated_layout(monkeypatch): + placements = [ + { + "x": index * 2.0, + "y": 0.0, + "w": 1.0, + "h": 1.0, + "material": "carbon_steel", + "grade": "A36", + "thickness": 0.5, + } + for index in range(4_000) + ] + plate = { + "W": 8_001, + "H": 2, + "material": "carbon_steel", + "grade": "A36", + "thickness": 0.5, + "placements": placements, + } + checks = 0 + original = geometry_verify._placements_separated + + def counted(*args, **kwargs): + nonlocal checks + checks += 1 + return original(*args, **kwargs) + + monkeypatch.setattr(geometry_verify, "_placements_separated", counted) + assert verify_nest_placements( + [plate], edge_margin=0, inter_part_clearance=0.25 + ) == [] + assert checks < len(placements) * 4 diff --git a/tests/test_package_contents.py b/tests/test_package_contents.py index 7f1d28a..75b1a90 100644 --- a/tests/test_package_contents.py +++ b/tests/test_package_contents.py @@ -33,6 +33,7 @@ def test_npm_dry_run_contains_runtime_contract_and_excludes_private_artifacts(): "skills/_shared/schemas/nest-result.schema.json", "skills/_shared/schemas/run-manifest.schema.json", "skills/steel-estimate/SKILL.md", + "skills/steel-estimate/scripts/acknowledge-finding.py", "skills/steel-estimate/scripts/build-estimate-package.py", "skills/steel-nest/scripts/nest.py", "skills/steel-rfq/scripts/generate-rfq.py", diff --git a/tests/test_public_data_policy.py b/tests/test_public_data_policy.py index 2ac4ef4..a99d430 100644 --- a/tests/test_public_data_policy.py +++ b/tests/test_public_data_policy.py @@ -1,10 +1,37 @@ +import ast +import importlib.util import subprocess import sys +import tempfile import unittest from pathlib import Path ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "scripts" / "check-public-data.py" + + +def load_scanner(): + spec = importlib.util.spec_from_file_location("check_public_data", SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def out_defaults(path): + tree = ast.parse(path.read_text(encoding="utf-8")) + defaults = [] + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + if not node.args or not isinstance(node.args[0], ast.Constant): + continue + if node.args[0].value != "--out": + continue + for keyword in node.keywords: + if keyword.arg == "default" and isinstance(keyword.value, ast.Constant): + defaults.append(keyword.value.value) + return defaults class PublicDataPolicyTests(unittest.TestCase): @@ -18,6 +45,175 @@ def test_repository_passes_public_data_check(self): self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + def test_scanner_fails_closed_for_sensitive_and_unknown_files(self): + scanner = load_scanner() + with self.subTest("environment filename"): + root = self._fixture_root() + path = root / ("." + "env" + ".local") + path.write_text("placeholder", encoding="utf-8") + findings = scanner.scan_paths([path], root=root, patterns={}) + self.assertTrue(any("environment file" in item for item in findings)) + + with self.subTest("private key extension"): + root = self._fixture_root() + path = root / ("deploy." + "p" + "em") + path.write_text("placeholder", encoding="utf-8") + findings = scanner.scan_paths([path], root=root, patterns={}) + self.assertTrue(any("key or certificate" in item for item in findings)) + + with self.subTest("unknown binary"): + root = self._fixture_root() + path = root / "fixture.bin" + path.write_bytes(bytes((0, 159, 255, 0))) + findings = scanner.scan_paths([path], root=root, patterns={}) + self.assertTrue(any("unknown binary" in item for item in findings)) + + def test_scanner_checks_unlisted_text_and_unquoted_credentials(self): + scanner = load_scanner() + root = self._fixture_root() + source = root / "fixture.ts" + credential = "API_" + "KEY" + "=" + "live-value" + source.write_text(credential, encoding="utf-8") + + findings = scanner.scan_paths([source], root=root) + + self.assertTrue(any("credential assignment" in item for item in findings)) + + def test_scanner_detects_high_confidence_secret_formats_without_echoing_value(self): + scanner = load_scanner() + root = self._fixture_root() + source = root / "fixture.txt" + generated_secret = "gh" + "p_" + ("a" * 36) + source.write_text(generated_secret, encoding="utf-8") + + findings = scanner.scan_paths([source], root=root) + + self.assertTrue(any("GitHub access token" in item for item in findings)) + self.assertFalse(any(generated_secret in item for item in findings)) + + def test_scanner_checks_staged_blob_not_only_worktree_copy(self): + scanner = load_scanner() + root = self._git_fixture_root() + source = root / "settings.txt" + staged_value = "to" + "ken=staged-value" + source.write_text(staged_value, encoding="utf-8") + subprocess.run(["git", "add", "settings.txt"], cwd=root, check=True) + source.write_text("safe placeholder", encoding="utf-8") + + findings = scanner.staged_findings(root) + + self.assertTrue( + any( + item.startswith("staged settings.txt:") + and item.endswith("credential assignment") + for item in findings + ) + ) + self.assertFalse(any(staged_value in item for item in findings)) + + def test_history_scan_reports_commit_path_and_category_without_value(self): + scanner = load_scanner() + root = self._git_fixture_root() + source = root / "retired.txt" + retired_value = "pass" + "word=retired-value" + source.write_text(retired_value, encoding="utf-8") + subprocess.run(["git", "add", "retired.txt"], cwd=root, check=True) + subprocess.run( + [ + "git", + "-c", + "user.name=Synthetic Test", + "-c", + "user.email=" + "synthetic" + "@" + "example.invalid", + "commit", + "-m", + "synthetic history fixture", + ], + cwd=root, + check=True, + capture_output=True, + ) + source.write_text("safe placeholder", encoding="utf-8") + subprocess.run(["git", "add", "retired.txt"], cwd=root, check=True) + subprocess.run( + [ + "git", + "-c", + "user.name=Synthetic Test", + "-c", + "user.email=" + "synthetic" + "@" + "example.invalid", + "commit", + "-m", + "remove synthetic credential", + ], + cwd=root, + check=True, + capture_output=True, + ) + + findings = scanner.revision_findings(["--all"], root) + + self.assertTrue( + any( + item.startswith("commit ") + and " retired.txt:" in item + and item.endswith("credential assignment") + for item in findings + ) + ) + self.assertFalse(any(retired_value in item for item in findings)) + + def test_history_scan_reports_non_noreply_author_without_address(self): + scanner = load_scanner() + root = self._git_fixture_root() + (root / "safe.txt").write_text("safe placeholder", encoding="utf-8") + subprocess.run(["git", "add", "safe.txt"], cwd=root, check=True) + address = "synthetic" + "@" + "example.invalid" + subprocess.run( + [ + "git", + "-c", + "user.name=Synthetic Test", + "-c", + "user.email=" + address, + "commit", + "-m", + "synthetic metadata fixture", + ], + cwd=root, + check=True, + capture_output=True, + ) + + findings = scanner.revision_metadata_findings(["--all"], root) + + self.assertTrue(any("non-noreply email address" in item for item in findings)) + self.assertFalse(any(address in item for item in findings)) + + def test_artifact_cli_defaults_are_git_ignored(self): + scripts = [ + ROOT / "skills" / "steel-estimate" / "scripts" / "build-estimate-package.py", + ROOT / "skills" / "steel-nest" / "scripts" / "nest.py", + ROOT / "skills" / "steel-rfq" / "scripts" / "generate-rfq.py", + ] + self.assertEqual([out_defaults(path) for path in scripts], [["outputs"]] * 3) + for directory in ("outputs", "out"): + ignored = subprocess.run( + ["git", "check-ignore", "-q", f"{directory}/synthetic-probe"], + cwd=ROOT, + ) + self.assertEqual(ignored.returncode, 0, directory) + + def _fixture_root(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + return Path(temporary.name) + + def _git_fixture_root(self): + root = self._fixture_root() + subprocess.run(["git", "init", "-q"], cwd=root, check=True) + return root + if __name__ == "__main__": unittest.main() diff --git a/tests/test_recalc.py b/tests/test_recalc.py new file mode 100644 index 0000000..8a99e8f --- /dev/null +++ b/tests/test_recalc.py @@ -0,0 +1,54 @@ +import importlib.util +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "skills" / "steel-rfq" / "scripts" / "recalc.py" +SPEC = importlib.util.spec_from_file_location("pi_steel_recalc_reliability", SCRIPT) +recalc = importlib.util.module_from_spec(SPEC) +sys.modules[SPEC.name] = recalc +SPEC.loader.exec_module(recalc) + + +def test_recalculation_flag_failure_is_distinct_and_prevents_bake(monkeypatch): + class BrokenWorkbook: + calculation = SimpleNamespace() + + def save(self, _path): + raise OSError("synthetic save failure") + + import openpyxl + + monkeypatch.setattr(openpyxl, "load_workbook", lambda _path: BrokenWorkbook()) + bake_calls = [] + monkeypatch.setattr( + recalc, "libreoffice_bake", lambda path: bake_calls.append(path) or True + ) + + with pytest.raises(recalc.RecalculationError, match="synthetic save failure"): + recalc.recalculate("synthetic.xlsx") + assert bake_calls == [] + + +def test_cli_reports_recalculation_failure_without_deferred_label( + monkeypatch, capsys, tmp_path +): + workbook = tmp_path / "synthetic.xlsx" + workbook.write_bytes(b"synthetic") + monkeypatch.setattr( + recalc, + "recalculate", + lambda _path: (_ for _ in ()).throw( + recalc.RecalculationError("synthetic flag failure") + ), + ) + monkeypatch.setattr(sys, "argv", ["recalc.py", str(workbook)]) + + assert recalc.main() == 1 + captured = capsys.readouterr() + assert "synthetic flag failure" in captured.err + assert "deferred_recalculate_on_open" not in captured.out + captured.err diff --git a/tests/test_rfq_generator.py b/tests/test_rfq_generator.py index 3b80435..68ab78f 100644 --- a/tests/test_rfq_generator.py +++ b/tests/test_rfq_generator.py @@ -244,3 +244,31 @@ def test_cli_ready_and_review_runs_publish_draft_workbooks( ) workbook = openpyxl.load_workbook(next(run_path.glob("*.xlsx"))) assert workbook["RFQ Metadata"]["B2"].value == "DRAFT — NOT SENT OR AWARDED" + + +def test_nest_handoff_blocks_diagnostic_stale_and_malformed_contracts(): + normalized = rfq.normalize_canonical_package(load("estimate-package.json")) + diagnostic = load("nest-handoff.json") + diagnostic["geometry_readiness"] = "diagnostic" + assert "diagnostic_nest_handoff" in { + finding["code"] + for finding in rfq.validate_nest_handoff( + diagnostic, expected=normalized + ) + } + + stale = load("nest-handoff.json") + stale["revision_id"] = "SYNTHETIC-STALE-REVISION" + assert "stale_nest_handoff" in { + finding["code"] + for finding in rfq.validate_nest_handoff(stale, expected=normalized) + } + + malformed = load("nest-handoff.json") + malformed["rows"] = [{"stock_id": "SYNTHETIC-INCOMPLETE"}] + findings = rfq.validate_nest_handoff(malformed, expected=normalized) + assert any( + finding["code"] == "invalid_nest_handoff_contract" + and finding["path"].startswith("$.rows[0]") + for finding in findings + ) diff --git a/tests/test_rfq_workbook_contract.py b/tests/test_rfq_workbook_contract.py index e2cb805..1b5dc75 100644 --- a/tests/test_rfq_workbook_contract.py +++ b/tests/test_rfq_workbook_contract.py @@ -106,3 +106,82 @@ def test_semantic_projection_matches_golden(tmp_path, monkeypatch): } for sheet in actual["sheets"] ] == expected["sheets"] + + +def test_formula_like_input_is_literal_but_compiler_formulas_remain_formulas( + tmp_path, monkeypatch +): + package = load("estimate-package.json") + package["project"]["name"] = "=1+1" + package["items"][0]["description"] = "@SUM(1,1)" + normalized = rfq.normalize_canonical_package(package) + profile = load("synthetic-profile.json") + profile["terms_template"]["content"] = "+DANGEROUS" + profile["terms_template"]["content_hash"] = hashlib.sha256( + profile["terms_template"]["content"].encode() + ).hexdigest() + monkeypatch.setattr(rfq.recalc, "libreoffice_bake", lambda path: False) + + result = rfq.compile_workbook( + normalized, + profile, + nest_handoff=None, + issued_date="2026-07-28", + project_location="-1+1", + output_directory=tmp_path, + profile_source="environment", + bake=True, + ) + + workbook = openpyxl.load_workbook(result["workbook_path"], data_only=False) + sheet = workbook["RFQ Draft"] + untrusted = [ + cell + for row in sheet.iter_rows() + for cell in row + if isinstance(cell.value, str) + and any(token in cell.value for token in ("=1+1", "@SUM", "+DANGEROUS")) + ] + assert untrusted + assert all(cell.data_type == "s" and cell.value.startswith("'") for cell in untrusted) + total_row = result["contract"]["total_row"] + assert sheet[f"H{total_row}"].data_type == "f" + assert sheet[f"J{total_row}"].data_type == "f" + + +def test_canonical_review_findings_and_assumptions_are_visible( + tmp_path, monkeypatch +): + package = load("estimate-package.json") + package["assumptions"] = [ + { + "assumption_id": "SYNTHETIC-ASSUMPTION-1", + "text": "Synthetic basis requires confirmation.", + "status": "unresolved", + } + ] + normalized = rfq.normalize_canonical_package(package) + profile = load("synthetic-profile.json") + monkeypatch.setattr(rfq.recalc, "libreoffice_bake", lambda path: False) + + result = rfq.compile_workbook( + normalized, + profile, + nest_handoff=None, + issued_date="2026-07-28", + project_location="Example City, ST", + output_directory=tmp_path, + profile_source="environment", + bake=True, + ) + + workbook = openpyxl.load_workbook(result["workbook_path"], data_only=False) + visible_text = " ".join( + str(cell.value) + for row in workbook["RFQ Draft"].iter_rows() + for cell in row + if cell.value is not None + ) + assert "REVIEW NOTES / ASSUMPTIONS" in visible_text + assert "SYNTHETIC-ASSUMPTION-1" in visible_text + assert "unresolved" in visible_text diff --git a/tests/test_run_manifests.py b/tests/test_run_manifests.py index 30c4aa3..60d9b71 100644 --- a/tests/test_run_manifests.py +++ b/tests/test_run_manifests.py @@ -1,4 +1,5 @@ import json +import os import sys from pathlib import Path @@ -18,6 +19,7 @@ sha256_bytes, sha256_file, ) +import pi_steel.run_manifest as run_manifest INPUT_HASH = sha256_bytes(b"SYNTHETIC-INPUT") @@ -140,3 +142,30 @@ def test_ready_publication_requires_qa_report(tmp_path): with publisher(tmp_path, "SYNTHETIC-RUN-NO-QA") as run: with pytest.raises(ManifestError, match="require qa-report.json"): run.publish() + + +def test_pointer_update_failure_rolls_back_run_and_preserves_prior_pointer( + tmp_path, monkeypatch +): + with publisher(tmp_path, "SYNTHETIC-RUN-PRIOR") as prior: + prior.write_qa_report({"outcome": "ready"}) + prior.publish() + pointer_before = (tmp_path / "latest-run.json").read_bytes() + + original_replace = os.replace + + def fail_latest_pointer(source, destination): + if Path(destination).name == "latest-run.json": + raise OSError("synthetic pointer failure") + return original_replace(source, destination) + + monkeypatch.setattr(run_manifest.os, "replace", fail_latest_pointer) + failed = publisher(tmp_path, "SYNTHETIC-RUN-POINTER-FAIL") + with failed: + failed.write_qa_report({"outcome": "ready"}) + with pytest.raises(ManifestError, match="unpublished run was rolled back"): + failed.publish() + + assert not failed.final_path.exists() + assert (tmp_path / "latest-run.json").read_bytes() == pointer_before + assert json.loads(pointer_before)["run_id"] == "SYNTHETIC-RUN-PRIOR" diff --git a/tests/test_runtime_bootstrap.py b/tests/test_runtime_bootstrap.py index 526b441..5e01cfd 100644 --- a/tests/test_runtime_bootstrap.py +++ b/tests/test_runtime_bootstrap.py @@ -6,6 +6,7 @@ from pathlib import Path import openpyxl +import pytest ROOT = Path(__file__).resolve().parents[1] @@ -106,6 +107,47 @@ def test_doctor_returns_machine_readable_dependency_missing_for_unsupported_pyth assert "python" in report["missing_required"] +@pytest.mark.parametrize("missing_module", ["jsonschema", "openpyxl", "pandas"]) +def test_doctor_maps_each_missing_required_module_to_dependency_missing( + missing_module, +): + doctor = load_doctor() + + report = doctor.diagnose( + version_info=(3, 12, 1), + module_finder=lambda name: ( + None if name == missing_module else object() + ), + command_finder=lambda name: "synthetic-jq" if name == "jq" else None, + ) + + assert report["run_outcome"] == "dependency_missing" + assert report["exit_code"] == 4 + assert report["missing_required"] == [missing_module] + dependency = next( + item for item in report["required"] if item["name"] == missing_module + ) + assert dependency["available"] is False + assert dependency["purpose"] == doctor.REQUIRED_MODULES[missing_module] + + +def test_doctor_maps_missing_jq_to_dependency_missing(): + doctor = load_doctor() + + report = doctor.diagnose( + version_info=(3, 12, 1), + module_finder=lambda _name: object(), + command_finder=lambda _name: None, + ) + + assert report["run_outcome"] == "dependency_missing" + assert report["exit_code"] == 4 + assert report["missing_required"] == ["jq"] + jq = next(item for item in report["required"] if item["name"] == "jq") + assert jq["available"] is False + assert jq["kind"] == "command" + + def test_doctor_reports_optional_capabilities_without_blocking_base_runtime(): doctor = load_doctor() diff --git a/tests/test_structured_failure_cli.py b/tests/test_structured_failure_cli.py new file mode 100644 index 0000000..2492ad1 --- /dev/null +++ b/tests/test_structured_failure_cli.py @@ -0,0 +1,100 @@ +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +ENTRYPOINTS = { + "steel-estimate": ( + ROOT + / "skills" + / "steel-estimate" + / "scripts" + / "build-estimate-package.py", + [ + "--input", + "{input}", + "--prepared-date", + "2026-07-28", + "--issued-date", + "2026-07-29", + ], + ), + "steel-nest": ( + ROOT / "skills" / "steel-nest" / "scripts" / "nest.py", + ["--job", "{input}", "--no-render"], + ), + "steel-rfq": ( + ROOT / "skills" / "steel-rfq" / "scripts" / "generate-rfq.py", + ["--input", "{input}", "--issued-date", "2026-07-29", "--no-bake"], + ), +} + + +@pytest.mark.parametrize("stage", ENTRYPOINTS) +@pytest.mark.parametrize( + "input_state", ["unavailable_file", "malformed", "missing_argument"] +) +def test_post_parse_input_failures_publish_machine_readable_run( + tmp_path, stage, input_state +): + script, stage_args = ENTRYPOINTS[stage] + input_path = tmp_path / f"{input_state}.json" + if input_state == "malformed": + input_path.write_text('{"not": "complete"', encoding="utf-8") + rendered_stage_args = [ + str(input_path) if argument == "{input}" else argument + for argument in stage_args + ] + if input_state == "missing_argument": + input_index = rendered_stage_args.index(str(input_path)) + del rendered_stage_args[input_index - 1 : input_index + 1] + output = tmp_path / f"{stage}-{input_state}-published" + run_id = f"SYNTHETIC-{stage.upper()}-{input_state.upper()}" + command = [ + sys.executable, + script, + *rendered_stage_args, + "--out", + output, + "--run-id", + run_id, + ] + environment = os.environ.copy() + environment.pop("PYTHONPATH", None) + environment["XDG_CONFIG_HOME"] = str(tmp_path / "empty-config") + + completed = subprocess.run( + command, + cwd=tmp_path, + env=environment, + capture_output=True, + text=True, + ) + + assert completed.returncode == 1 + pointer = json.loads((output / "latest-run.json").read_text(encoding="utf-8")) + assert pointer["run_id"] == run_id + run_path = output / pointer["run_directory"] + manifest = json.loads( + (run_path / "run-manifest.json").read_text(encoding="utf-8") + ) + qa_report = json.loads( + (run_path / "qa-report.json").read_text(encoding="utf-8") + ) + assert manifest["stage"] == stage + assert manifest["run_outcome"] == "usage_or_internal_error" + assert manifest["package_status"] == "draft" + assert qa_report["run_outcome"] == "usage_or_internal_error" + expected_code = ( + "cli_usage_error" + if input_state == "missing_argument" + else "input_unreadable_or_invalid" + ) + assert qa_report["findings"][0]["code"] == expected_code + assert str(input_path) not in json.dumps(qa_report) + assert "diagnostic published:" in completed.stderr diff --git a/tests/test_takeoff_cli.py b/tests/test_takeoff_cli.py new file mode 100644 index 0000000..8fa5803 --- /dev/null +++ b/tests/test_takeoff_cli.py @@ -0,0 +1,166 @@ +import json +import subprocess +import sys +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +CALCULATE_WEIGHT = ( + ROOT / "skills" / "steel-takeoff" / "scripts" / "calculate-weight.sh" +) +VALIDATE_BOM = ( + ROOT / "skills" / "steel-takeoff" / "scripts" / "validate-bom.py" +) +HEADER = ( + "Source_ID,Mark,Qty,Size,Grade,Length_ft,Unit_Wt_plf," + "Total_Wt_lbs,Connections,Notes,Source_Sheet,Source_Detail,Intent\n" +) + + +def run_validate(path: Path, *, json_output: bool = True): + command = [sys.executable, VALIDATE_BOM, path] + if json_output: + command.append("--json") + return subprocess.run(command, capture_output=True, text=True) + + +def write_bom(tmp_path: Path, row: str) -> Path: + path = tmp_path / "synthetic-bom.csv" + path.write_text(HEADER + row + "\n", encoding="utf-8") + return path + + +def test_calculate_weight_reports_exact_synthetic_totals_without_pricing( + tmp_path, +): + bom = tmp_path / "synthetic-weight.csv" + bom.write_text( + "Source_ID,Mark,Qty,Size,Grade,Length_ft,Unit_Wt_plf\n" + "SYNTHETIC-W1,SYNTHETIC-W1,2,W14X30,A992,10,30\n" + "SYNTHETIC-H1,SYNTHETIC-H1,1,HSS4X4X1/4,,5'-6\",20\n", + encoding="utf-8", + ) + + result = subprocess.run( + ["bash", CALCULATE_WEIGHT, bom], + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stdout + result.stderr + assert "Member Weight:" in result.stdout + assert "710 lb" in result.stdout + assert "Connection Allow (12%):" in result.stdout + assert "85 lb" in result.stdout + assert "Misc Steel Allow (5%):" in result.stdout + assert "36 lb" in result.stdout + assert "GRAND TOTAL:" in result.stdout + assert "831 lb" in result.stdout + assert "UNSPECIFIED" in result.stdout + assert "No price was calculated." in result.stdout + assert "$" not in result.stdout + assert "cost sensitivity" not in result.stdout.lower() + assert "estimated cost" not in result.stdout.lower() + + +def test_validate_bom_missing_file_returns_text_error_and_exit_one(tmp_path): + result = run_validate(tmp_path / "missing-synthetic.csv") + + assert result.returncode == 1 + assert result.stdout == "" + assert "ERROR: File not found:" in result.stderr + + +def test_validate_bom_malformed_quantity_returns_parse_error_and_exit_one( + tmp_path, +): + path = write_bom( + tmp_path, + "SYNTHETIC-SRC-1,SYNTHETIC-M1,not-a-number,W14X30,A992,10,30," + "300,None,Synthetic,SYNTHETIC-SHEET,SYNTHETIC-DETAIL,fabricated_part", + ) + + result = run_validate(path) + + assert result.returncode == 1 + assert result.stdout == "" + assert "ERROR: Could not parse BOM:" in result.stderr + + +@pytest.mark.parametrize( + ("row", "expected_code", "expected_status", "expected_exit"), + [ + ( + "SYNTHETIC-SRC-1,SYNTHETIC-M1,0,W14X30,A992,10,30,0,None," + "Synthetic,SYNTHETIC-SHEET,SYNTHETIC-DETAIL,fabricated_part", + "invalid_quantity", + "invalid", + 1, + ), + ( + "SYNTHETIC-SRC-1,SYNTHETIC-M1,1,W99X99,A992,10,99,990,None," + "Synthetic,SYNTHETIC-SHEET,SYNTHETIC-DETAIL,fabricated_part", + "unverified_designation", + "review_required", + 0, + ), + ( + "SYNTHETIC-SRC-1,SYNTHETIC-M1,1,W14X30,A992,10,31,310,None," + "Synthetic,SYNTHETIC-SHEET,SYNTHETIC-DETAIL,fabricated_part", + "unit_weight_mismatch", + "review_required", + 0, + ), + ( + "SYNTHETIC-SRC-1,SYNTHETIC-M1,1,W14X30,A500,10,30,300,None," + "Synthetic,SYNTHETIC-SHEET,SYNTHETIC-DETAIL,fabricated_part", + "unusual_grade", + "review_required", + 0, + ), + ( + "SYNTHETIC-SRC-1,SYNTHETIC-M1,1,W14X30,A992,81,30,2430,None," + "Synthetic,SYNTHETIC-SHEET,SYNTHETIC-DETAIL,fabricated_part", + "unusual_member_length", + "review_required", + 0, + ), + ], +) +def test_validate_bom_json_reports_branch_status_and_exit_semantics( + tmp_path, + row, + expected_code, + expected_status, + expected_exit, +): + result = run_validate(write_bom(tmp_path, row)) + + assert result.returncode == expected_exit, result.stdout + result.stderr + report = json.loads(result.stdout) + assert report["status"] == expected_status + assert expected_code in { + finding["code"] for finding in report["findings"] + } + assert len(report["input_hash"]) == 64 + + +def test_validate_bom_json_valid_case_is_machine_readable_and_exits_zero( + tmp_path, +): + path = write_bom( + tmp_path, + "SYNTHETIC-SRC-1,SYNTHETIC-M1,1,W14X30,A992,10,30,300,None," + "Synthetic,SYNTHETIC-SHEET,SYNTHETIC-DETAIL,fabricated_part", + ) + + result = run_validate(path) + + assert result.returncode == 0, result.stdout + result.stderr + report = json.loads(result.stdout) + assert report["status"] == "validated" + assert report["line_count"] == 1 + assert report["total_weight_lbs"] == 300 + assert report["findings"] == [] From 8009363e24ea5f280d73107ad6f308ca7e7a5a54 Mon Sep 17 00:00:00 2001 From: Victor Garcia Date: Tue, 28 Jul 2026 13:58:06 -0600 Subject: [PATCH 13/15] fix(ci): install DXF runtime and expose draft status --- requirements-render.txt | 3 +-- requirements.txt | 1 + skills/steel-rfq/scripts/generate-rfq.py | 1 + tests/golden/rfq/semantic-workbook.json | 2 +- tests/test_rfq_workbook_contract.py | 1 + 5 files changed, 5 insertions(+), 3 deletions(-) diff --git a/requirements-render.txt b/requirements-render.txt index 32dcc2c..99b6d1d 100644 --- a/requirements-render.txt +++ b/requirements-render.txt @@ -1,4 +1,3 @@ -# Optional PDF, PNG, and DXF rendering tier. -ezdxf>=1.3,<2 +# Optional PDF and PNG rendering tier. matplotlib>=3.9,<4 numpy>=2.1,<3 diff --git a/requirements.txt b/requirements.txt index 5a1e10e..c77cf32 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ # Required by the validated estimate and RFQ runtime. +ezdxf>=1.3,<2 jsonschema>=4.23,<5 openpyxl>=3.1,<4 pandas>=2.2,<3 diff --git a/skills/steel-rfq/scripts/generate-rfq.py b/skills/steel-rfq/scripts/generate-rfq.py index 3774e8a..74747e1 100755 --- a/skills/steel-rfq/scripts/generate-rfq.py +++ b/skills/steel-rfq/scripts/generate-rfq.py @@ -707,6 +707,7 @@ def compile_workbook( f"{profile['city_state']}" ) sheet["A3"] = ( + "Document Status: DRAFT — NOT SENT OR AWARDED | " f"Date Issued: {issued_date} | Response Requested By: _______________ | " f"Project Location: {project_location}" ) diff --git a/tests/golden/rfq/semantic-workbook.json b/tests/golden/rfq/semantic-workbook.json index cd46a20..a9b492a 100644 --- a/tests/golden/rfq/semantic-workbook.json +++ b/tests/golden/rfq/semantic-workbook.json @@ -1,6 +1,6 @@ { "projection_format": "1.0.0", - "semantic_sha256": "e5a2dae82dfb653cfb951ed273993e719005e4d8d3a768ec468dd27cc2e9e22f", + "semantic_sha256": "f58a92fcaf25db8b4d7354fb20090358648bc1b73251c5756887a5aed2e13c53", "sheets": [ { "title": "RFQ Draft", diff --git a/tests/test_rfq_workbook_contract.py b/tests/test_rfq_workbook_contract.py index 1b5dc75..8fa59f7 100644 --- a/tests/test_rfq_workbook_contract.py +++ b/tests/test_rfq_workbook_contract.py @@ -54,6 +54,7 @@ def test_workbook_structure_formulas_styles_and_draft_metadata(tmp_path, monkeyp assert sheet.column_dimensions["C"].width == 38 assert sheet["I8"].fill.fgColor.rgb.endswith("BF8F00") assert sheet["I10"].fill.fgColor.rgb.endswith("FFF2CC") + assert "DRAFT — NOT SENT OR AWARDED" in sheet["A3"].value total_row = result["contract"]["total_row"] first_row = result["contract"]["first_material_row"] last_row = result["contract"]["last_material_row"] From 29862f469cc6666057e7f2850e0050991b104fe3 Mon Sep 17 00:00:00 2001 From: Victor Garcia Date: Tue, 28 Jul 2026 13:59:57 -0600 Subject: [PATCH 14/15] fix(ci): install complete nest runtime --- requirements-render.txt | 4 +--- requirements.txt | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements-render.txt b/requirements-render.txt index 99b6d1d..5adea52 100644 --- a/requirements-render.txt +++ b/requirements-render.txt @@ -1,3 +1 @@ -# Optional PDF and PNG rendering tier. -matplotlib>=3.9,<4 -numpy>=2.1,<3 +# Full-render tests additionally require LibreOffice and Poppler system tools. diff --git a/requirements.txt b/requirements.txt index c77cf32..2b4de91 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,7 @@ # Required by the validated estimate and RFQ runtime. ezdxf>=1.3,<2 jsonschema>=4.23,<5 +matplotlib>=3.9,<4 +numpy>=2.1,<3 openpyxl>=3.1,<4 pandas>=2.2,<3 From 7102481509b20fac5c56301106ad9eeac4874388 Mon Sep 17 00:00:00 2001 From: Victor Garcia Date: Tue, 28 Jul 2026 14:02:26 -0600 Subject: [PATCH 15/15] fix(ci): pin tested lint and render contracts --- pyproject.toml | 10 ++++------ requirements-dev.txt | 4 ++-- skills/steel-estimate/SKILL.md | 12 ++++++------ tests/test_full_render_smoke.py | 2 +- 4 files changed, 13 insertions(+), 15 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index e04b446..bad9a8f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,17 +4,15 @@ version = "0.2.2" description = "Python runtime dependencies and test configuration for pi-steel" requires-python = ">=3.11,<3.14" dependencies = [ + "ezdxf>=1.3,<2", "jsonschema>=4.23,<5", + "matplotlib>=3.9,<4", + "numpy>=2.1,<3", "openpyxl>=3.1,<4", "pandas>=2.2,<3", ] [project.optional-dependencies] -render = [ - "ezdxf>=1.3,<2", - "matplotlib>=3.9,<4", - "numpy>=2.1,<3", -] test = [ "pytest>=8.3,<10", ] @@ -23,7 +21,7 @@ test = [ addopts = "-ra" testpaths = ["tests"] markers = [ - "full_render: requires optional rendering dependencies and LibreOffice", + "full_render: requires LibreOffice and Poppler system tools", ] [tool.ruff.lint.per-file-ignores] diff --git a/requirements-dev.txt b/requirements-dev.txt index d52a49e..cbfc571 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,5 @@ -r requirements.txt -# Base test tier. Install requirements-render.txt for optional rendering. +# Base test tier; full-render tests also require the documented system tools. pytest>=8.3,<10 -ruff>=0.11,<1 +ruff>=0.11,<0.12 diff --git a/skills/steel-estimate/SKILL.md b/skills/steel-estimate/SKILL.md index 94262c2..3701011 100644 --- a/skills/steel-estimate/SKILL.md +++ b/skills/steel-estimate/SKILL.md @@ -101,12 +101,12 @@ doctor from the package root: python3 scripts/doctor.py --json ``` -Install the base calculation and workbook dependencies from -`requirements.txt`. Install `requirements-render.txt` only when PDF, PNG, DXF, -or LibreOffice-assisted output is needed. `requirements-tested.txt` records the -exact dependency versions exercised by CI; it is a reproducibility reference, -not the general installation range. Run the doctor again after installation -before retrying the estimate. +Install the calculation, workbook, PNG, and DXF dependencies from +`requirements.txt`. Full PDF and LibreOffice-assisted checks also require the +system tools documented by `requirements-render.txt`. +`requirements-tested.txt` records the exact dependency versions exercised by +CI; it is a reproducibility reference, not the general installation range. Run +the doctor again after installation before retrying the estimate. ## Delivery checks diff --git a/tests/test_full_render_smoke.py b/tests/test_full_render_smoke.py index 823d829..28c8d4f 100644 --- a/tests/test_full_render_smoke.py +++ b/tests/test_full_render_smoke.py @@ -150,7 +150,7 @@ def test_ready_package_renders_reference_and_verified_outputs(tmp_path): for expected in ( "DRAFT", "Synthetic Pipeline Project", - "RFQ", + "REQUEST FOR QUOTATION", "Response Requested By", ): assert expected in rendered_text