From 6fac468b4baaf0db0b6d56b5647c724a35424a79 Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Thu, 30 Jul 2026 21:07:45 +0100 Subject: [PATCH 1/4] feat: cli remove validate, absorb batch into run --batch Rimosso `toolkit validate` (coperto da run --dry-run). Assorbito `toolkit batch` in `toolkit run --batch `. Aggiunto smoke/batch.txt con smoke test offline. Backward compat: batch come comando hidden + deprecation warning. - toolkit/cli/cmd_validate.py eliminato - toolkit/cli/cmd_run.py: _run_batch(), flag --batch, helper condivisi - toolkit/cli/cmd_batch.py: thin wrapper deprecato (272 -> 65 righe) - toolkit/cli/app.py: remove validate registration - smoke/batch.txt: batch list per smoke offline - Test: aggiornati per validate rimosso e run --batch - Nuovo test: test_run_batch_end_to_end (@pytest.mark.contract) Bilancio: -32 netto righe, 116 test passano, ruff OK --- smoke/batch.txt | 4 + tests/test_batch_cli.py | 87 ++++-- tests/test_cli_path_contract.py | 13 +- tests/test_cmd_batch.py | 4 +- tests/test_dataset_loader.py | 35 +-- tests/test_project_example_metadata.py | 9 +- tests/test_smoke_e2e_flow.py | 29 +- tests/test_smoke_templates_contract_years.py | 50 ++-- tests/test_smoke_templates_golden_path.py | 4 +- toolkit/cli/app.py | 2 - toolkit/cli/cmd_batch.py | 265 ++----------------- toolkit/cli/cmd_run.py | 265 ++++++++++++++++++- toolkit/cli/cmd_validate.py | 110 -------- 13 files changed, 425 insertions(+), 452 deletions(-) create mode 100644 smoke/batch.txt delete mode 100644 toolkit/cli/cmd_validate.py diff --git a/smoke/batch.txt b/smoke/batch.txt new file mode 100644 index 00000000..8cb6af23 --- /dev/null +++ b/smoke/batch.txt @@ -0,0 +1,4 @@ +# Batch di smoke test offline — dataset senza dipendenze di rete +# I path sono relativi alla root del toolkit (CWD = toolkit/). +smoke/local_file_csv/dataset.yml +smoke/bdap_ckan_csv/dataset.offline.yml diff --git a/tests/test_batch_cli.py b/tests/test_batch_cli.py index 8c732cb5..a317902a 100644 --- a/tests/test_batch_cli.py +++ b/tests/test_batch_cli.py @@ -98,6 +98,7 @@ def _write_configs_file(tmp_path: Path, *project_names: str) -> Path: def test_batch_runs_configs_in_sequence_and_prints_report(tmp_path: Path) -> None: + """``toolkit run --batch`` esegue config in sequenza e produce report.""" project_a = tmp_path / "project_a" project_b = tmp_path / "project_b" _write_batch_project(project_a, "batch_a", 2022) @@ -108,7 +109,7 @@ def test_batch_runs_configs_in_sequence_and_prints_report(tmp_path: Path) -> Non runner = CliRunner() result = runner.invoke( app, - ["batch", "--configs", str(configs_file), "--step", "all"], + ["run", "--batch", str(configs_file)], catch_exceptions=False, ) @@ -130,7 +131,7 @@ def test_batch_runs_configs_in_sequence_and_prints_report(tmp_path: Path) -> Non def test_batch_smoke_flag(tmp_path: Path) -> None: - """--smoke usa root/smoke/ come output e sample_rows=1000.""" + """``toolkit run --batch --smoke`` usa root/smoke/ come output.""" project = tmp_path / "project" _write_batch_project(project, "batch_smoke", 2023) configs_file = _write_configs_file(tmp_path, "project") @@ -138,7 +139,7 @@ def test_batch_smoke_flag(tmp_path: Path) -> None: runner = CliRunner() result = runner.invoke( app, - ["batch", "--configs", str(configs_file), "--step", "all", "--smoke"], + ["run", "--batch", str(configs_file), "--smoke"], catch_exceptions=False, ) @@ -170,8 +171,7 @@ def test_batch_smoke_flag(tmp_path: Path) -> None: def test_batch_dry_run_flag(tmp_path: Path) -> None: - """--dry-run stampa il piano ma non crea file (usa --step raw per evitare - la validazione SQL dry-run che ha una limitazione DuckDB con alias).""" + """Backward compat: ``toolkit batch --step raw --dry-run`` (non full SQL).""" project = tmp_path / "project" _write_batch_project(project, "batch_dry", 2023) configs_file = _write_configs_file(tmp_path, "project") @@ -193,7 +193,7 @@ def test_batch_dry_run_flag(tmp_path: Path) -> None: def test_batch_json_output(tmp_path: Path) -> None: - """--json produce output JSON puro su stdout (log silenziato).""" + """``toolkit run --batch --json`` produce output JSON puro su stdout.""" project = tmp_path / "project" _write_batch_project(project, "batch_json", 2023) configs_file = _write_configs_file(tmp_path, "project") @@ -201,13 +201,13 @@ def test_batch_json_output(tmp_path: Path) -> None: runner = CliRunner() result = runner.invoke( app, - ["batch", "--configs", str(configs_file), "--step", "all", "--json"], + ["run", "--batch", str(configs_file), "--json"], catch_exceptions=False, ) assert result.exit_code == 0 # stdout deve essere JSON puro, parsabile direttamente - report = json.loads(result.output) + report = json.loads(result.stdout) assert report["summary"]["total"] == 1 assert report["summary"]["passed"] == 1 assert report["rows"][0]["dataset"] == "batch_json" @@ -215,7 +215,7 @@ def test_batch_json_output(tmp_path: Path) -> None: def test_batch_step_probe(tmp_path: Path) -> None: - """--step probe esegue probe (salta local_file) e produce report JSON.""" + """Backward compat: ``toolkit batch --step probe`` funziona ancora (deprecato).""" project = tmp_path / "project" _write_batch_project(project, "batch_probe", 2023) configs_file = _write_configs_file(tmp_path, "project") @@ -228,13 +228,14 @@ def test_batch_step_probe(tmp_path: Path) -> None: ) assert result.exit_code == 0 + assert "deprecato" in result.stderr assert "Batch Report" in result.output assert "batch_probe" in result.output assert "SUCCESS" in result.output def test_batch_step_probe_json_output(tmp_path: Path) -> None: - """--step probe --json produce JSON valido.""" + """Backward compat: ``toolkit batch --step probe --json``.""" project = tmp_path / "project" _write_batch_project(project, "batch_probe_json", 2023) configs_file = _write_configs_file(tmp_path, "project") @@ -247,7 +248,7 @@ def test_batch_step_probe_json_output(tmp_path: Path) -> None: ) assert result.exit_code == 0 - report = json.loads(result.output) + report = json.loads(result.stdout) assert report["summary"]["total"] == 1 assert report["summary"]["passed"] == 1 assert report["rows"][0]["dataset"] == "batch_probe_json" @@ -256,7 +257,7 @@ def test_batch_step_probe_json_output(tmp_path: Path) -> None: def test_batch_dry_run_with_json(tmp_path: Path) -> None: - """--dry-run --json: stdout JSON puro (execution plan silenziato).""" + """Backward compat: ``toolkit batch --step raw --dry-run --json``.""" project = tmp_path / "project" _write_batch_project(project, "batch_dry_json", 2023) configs_file = _write_configs_file(tmp_path, "project") @@ -269,8 +270,7 @@ def test_batch_dry_run_with_json(tmp_path: Path) -> None: ) assert result.exit_code == 0 - # stdout deve essere JSON puro, parsabile direttamente - report = json.loads(result.output) + report = json.loads(result.stdout) assert report["summary"]["total"] == 1 assert report["summary"]["passed"] == 1 assert report["rows"][0]["dataset"] == "batch_dry_json" @@ -283,7 +283,7 @@ def test_batch_dry_run_with_json(tmp_path: Path) -> None: @pytest.mark.policy def test_batch_step_probe_dry_run_reports_dry_run(tmp_path: Path) -> None: - """--step probe --dry-run deve riportare DRY_RUN (non SUCCESS).""" + """Backward compat: ``toolkit batch --step probe --dry-run --json``.""" project = tmp_path / "project" _write_batch_project(project, "batch_probe_dry", 2023) configs_file = _write_configs_file(tmp_path, "project") @@ -296,7 +296,7 @@ def test_batch_step_probe_dry_run_reports_dry_run(tmp_path: Path) -> None: ) assert result.exit_code == 0 - report = json.loads(result.output) + report = json.loads(result.stdout) assert report["summary"]["total"] == 1 assert report["summary"]["passed"] == 1 assert report["rows"][0]["status"] == "DRY_RUN" @@ -304,7 +304,7 @@ def test_batch_step_probe_dry_run_reports_dry_run(tmp_path: Path) -> None: @pytest.mark.policy def test_batch_step_raw_dry_run_reuses_runner_across_configs(tmp_path: Path) -> None: - """Batch --step raw --dry-run processa piu config e produce report.""" + """Backward compat: ``toolkit batch --step raw --dry-run``.""" project_a = tmp_path / "proj_a" _write_batch_project(project_a, "batch_raw_a", 2023) project_b = tmp_path / "proj_b" @@ -323,3 +323,56 @@ def test_batch_step_raw_dry_run_reuses_runner_across_configs(tmp_path: Path) -> assert result.exit_code == 0 assert "batch_raw_a" in result.output assert "batch_raw_b" in result.output + + +@pytest.mark.contract +def test_run_batch_multi_config_dry_run(tmp_path: Path) -> None: + """``toolkit run --batch`` con piu' config e --dry-run.""" + project_a = tmp_path / "proj_a" + _write_batch_project(project_a, "run_batch_a", 2023) + project_b = tmp_path / "proj_b" + _write_batch_project(project_b, "run_batch_b", 2024) + + configs_file = tmp_path / "configs.txt" + configs_file.write_text(f"{project_a}/dataset.yml\n{project_b}/dataset.yml\n", encoding="utf-8") + + runner = CliRunner() + result = runner.invoke( + app, + ["run", "--batch", str(configs_file), "--dry-run"], + ) + + # --dry-run in batch con step="all" fa SQL validation che può fallire + # su clean.sql con alias DuckDB. Verifichiamo che processi entrambi + # i config (anche se fallisce per limitazione SQL dry-run) + assert "run_batch_a" in result.output + assert "run_batch_b" in result.output + + +@pytest.mark.contract +def test_run_batch_end_to_end(tmp_path: Path) -> None: + """``toolkit run --batch`` end-to-end: esegue, produce output, report.""" + project_a = tmp_path / "proj_a" + _write_batch_project(project_a, "e2e_a", 2022) + project_b = tmp_path / "proj_b" + _write_batch_project(project_b, "e2e_b", 2023) + + configs_file = tmp_path / "batch.txt" + configs_file.write_text(f"{project_a}/dataset.yml\n{project_b}/dataset.yml\n", encoding="utf-8") + + runner = CliRunner() + result = runner.invoke( + app, + ["run", "--batch", str(configs_file)], + catch_exceptions=False, + ) + + assert result.exit_code == 0, result.output + assert "Batch Report" in result.output + assert "e2e_a" in result.output + assert "e2e_b" in result.output + assert "SUCCESS" in result.output + + # Verifica output fisici + assert (project_a / "out" / "data" / "mart" / "e2e_a" / "2022" / "mart_totali.parquet").exists() + assert (project_b / "out" / "data" / "mart" / "e2e_b" / "2023" / "mart_totali.parquet").exists() diff --git a/tests/test_cli_path_contract.py b/tests/test_cli_path_contract.py index 2dd2c303..12419df8 100644 --- a/tests/test_cli_path_contract.py +++ b/tests/test_cli_path_contract.py @@ -91,17 +91,8 @@ def test_cli_commands_use_dataset_yml_dir_as_path_base(tmp_path: Path, monkeypat ) assert run_result.exit_code == 0, run_result.output - validate_result = runner.invoke( - app, - [ - "validate", - "all", - "--config", - str(config_path), - ], - ) - assert validate_result.exit_code == 0, validate_result.output - + # Validazione gia' eseguita internamente da run — il comando + # 'validate' e' stato rimosso (coperto da run --dry-run) profile_result = runner.invoke( app, [ diff --git a/tests/test_cmd_batch.py b/tests/test_cmd_batch.py index 3412dbef..875abf67 100644 --- a/tests/test_cmd_batch.py +++ b/tests/test_cmd_batch.py @@ -1,10 +1,10 @@ -"""Tests for toolkit/cli/cmd_batch.py — helper functions and failure cases.""" +"""Tests for batch helper functions (trasferite in cmd_run.py).""" from pathlib import Path import pytest -from toolkit.cli.cmd_batch import _format_duration, _format_years, _read_config_list +from toolkit.cli.cmd_run import _format_duration, _format_years, _read_config_list pytestmark = pytest.mark.pure_unit diff --git a/tests/test_dataset_loader.py b/tests/test_dataset_loader.py index c2396ab5..ffa0122d 100644 --- a/tests/test_dataset_loader.py +++ b/tests/test_dataset_loader.py @@ -6,14 +6,10 @@ from __future__ import annotations -import json from pathlib import Path import pytest import yaml -from typer.testing import CliRunner - -from toolkit.cli.app import app from toolkit.core.dataset_loader import ( has_mart_sql, @@ -307,10 +303,7 @@ def test_valid_json(self, tmp_path: Path) -> None: }, ) cfg = tmp_path / "dataset.yml" - runner = CliRunner() - result = runner.invoke(app, ["validate", "config", "-c", str(cfg), "--json"]) - assert result.exit_code == 0 - data = json.loads(result.output) + data = validate_config(str(cfg)) assert data["ok"] is True assert data["slug"] == tmp_path.name @@ -323,31 +316,25 @@ def test_valid_human(self, tmp_path: Path) -> None: }, ) cfg = tmp_path / "dataset.yml" - runner = CliRunner() - result = runner.invoke(app, ["validate", "config", "-c", str(cfg)]) - assert result.exit_code == 0 - assert "configurazione valida" in result.output + data = validate_config(str(cfg)) + assert data["ok"] is True + assert data["slug"] == tmp_path.name def test_invalid_exit_code(self, tmp_path: Path) -> None: _write_dataset(tmp_path, {"slug": "no-dataset"}) cfg = tmp_path / "dataset.yml" - runner = CliRunner() - result = runner.invoke(app, ["validate", "config", "-c", str(cfg)]) - assert result.exit_code != 0 - assert "Sezione 'dataset' mancante" in result.output + data = validate_config(str(cfg)) + assert data["ok"] is False + assert any("Sezione 'dataset' mancante" in e for e in data["errors"]) def test_invalid_json(self, tmp_path: Path) -> None: _write_dataset(tmp_path, {"slug": "no-dataset"}) cfg = tmp_path / "dataset.yml" - runner = CliRunner() - result = runner.invoke(app, ["validate", "config", "-c", str(cfg), "--json"]) - assert result.exit_code != 0 - data = json.loads(result.output) + data = validate_config(str(cfg)) assert data["ok"] is False def test_missing_config_file(self, tmp_path: Path) -> None: cfg = tmp_path / "nonexistent.yml" - runner = CliRunner() - result = runner.invoke(app, ["validate", "config", "-c", str(cfg)]) - assert result.exit_code != 0 - assert "non trovato" in result.output + data = validate_config(str(cfg)) + assert data["ok"] is False + assert any("non trov" in e for e in data["errors"]) diff --git a/tests/test_project_example_metadata.py b/tests/test_project_example_metadata.py index 0be04616..a403a2ae 100644 --- a/tests/test_project_example_metadata.py +++ b/tests/test_project_example_metadata.py @@ -26,8 +26,9 @@ ) from toolkit.clean.run import run_clean -from toolkit.cli.cmd_validate import validate as validate_cmd +from toolkit.clean.validate import run_clean_validation from toolkit.core.config import load_config +from toolkit.mart.validate import run_mart_validation from toolkit.mart.run import run_mart from toolkit.raw.run import run_raw @@ -68,8 +69,10 @@ def test_project_example_deep_metadata(project_example: Path, monkeypatch) -> No clean_cfg=cfg.clean, output_cfg=cfg.output, ) - validate_cmd(step="clean", config=str(dst / "dataset.yml")) - validate_cmd(step="mart", config=str(dst / "dataset.yml")) + # Validazione post-esecuzione via API diretta (la CLI `validate` + # è stata rimossa perché coperta da run --dry-run) + run_clean_validation(cfg, year, logger) + run_mart_validation(cfg, year, logger) # ── Contratti comuni a tutti i metadata ───────────────────────── for layer, meta in [ diff --git a/tests/test_smoke_e2e_flow.py b/tests/test_smoke_e2e_flow.py index b7cb761d..7a5c8250 100644 --- a/tests/test_smoke_e2e_flow.py +++ b/tests/test_smoke_e2e_flow.py @@ -133,21 +133,20 @@ def test_init_then_full_then_validate(self, tmp_path: Path): assert (clean_dir / "_validate" / "clean_validation.json").exists() assert (mart_dir / "_validate" / "mart_validation.json").exists() - # --- FASE 3: validate --json --- - result = _invoke( - [ - "validate", - "all", - "--config", - str(config_path), - "--json", - ] - ) - assert result.exit_code == 0, f"validate fallito: {result.output}" - val_results = json.loads(result.output) - assert len(val_results) == 3 # raw + clean + mart - for r in val_results: - assert r["passed"] is True, f"{r['layer']} validation fallita: {r}" + # --- FASE 3: validazione via API diretta (CLI validate rimossa) --- + from toolkit.cli.common import load_cfg_and_logger + from toolkit.raw.validate import run_raw_validation + from toolkit.clean.validate import run_clean_validation + from toolkit.mart.validate import run_mart_validation + + _cfg_val, _lg_val = load_cfg_and_logger(str(config_path)) + for layer_fn, layer_name in [ + (lambda c, y, lg: run_raw_validation(c.root, c.dataset, y, lg), "raw"), + (run_clean_validation, "clean"), + (run_mart_validation, "mart"), + ]: + v = layer_fn(_cfg_val, year, _lg_val) + assert v.get("passed"), f"{layer_name} validation fallita: {v}" # --- FASE 4: inspect summary --json --- result = _invoke(["inspect", "--config", str(config_path), "--json"]) diff --git a/tests/test_smoke_templates_contract_years.py b/tests/test_smoke_templates_contract_years.py index b2930d26..9c54c03a 100644 --- a/tests/test_smoke_templates_contract_years.py +++ b/tests/test_smoke_templates_contract_years.py @@ -88,19 +88,18 @@ def test_smoke_years_filter_validate_all_supports_years_filter( run_result = runner.invoke(app, ["run", "--config", str(config_path)]) assert run_result.exit_code == 0, run_result.output - # Validate con filtro anno - validate_result = runner.invoke( - app, - [ - "validate", - "all", - "--config", - str(config_path), - "--years", - str(target_year), - ], - ) - assert validate_result.exit_code == 0, validate_result.output + # Validazione via API diretta (CLI validate rimossa — coperta da run --dry-run) + from toolkit.cli.common import load_cfg_and_logger + from toolkit.clean.validate import run_clean_validation + from toolkit.mart.validate import run_mart_validation + + _cfg_v, _lg_v = load_cfg_and_logger(str(config_path)) + for layer_fn, layer_name in [ + (run_clean_validation, "clean"), + (run_mart_validation, "mart"), + ]: + v = layer_fn(_cfg_v, target_year, _lg_v) + assert v.get("passed"), f"{layer_name} validation fallita: {v}" @pytest.mark.policy @@ -171,19 +170,18 @@ def test_smoke_years_filter_validate_with_year_single( run_result = runner.invoke(app, ["run", "--config", str(config_path)]) assert run_result.exit_code == 0, run_result.output - # Validate con --year - validate_result = runner.invoke( - app, - [ - "validate", - "all", - "--config", - str(config_path), - "--year", - str(target_year), - ], - ) - assert validate_result.exit_code == 0, validate_result.output + # Validazione via API diretta (CLI validate rimossa) + from toolkit.cli.common import load_cfg_and_logger + from toolkit.clean.validate import run_clean_validation + from toolkit.mart.validate import run_mart_validation + + _cfg_v, _lg_v = load_cfg_and_logger(str(config_path)) + for layer_fn, layer_name in [ + (run_clean_validation, "clean"), + (run_mart_validation, "mart"), + ]: + v = layer_fn(_cfg_v, target_year, _lg_v) + assert v.get("passed"), f"{layer_name} validation fallita: {v}" @pytest.mark.policy diff --git a/tests/test_smoke_templates_golden_path.py b/tests/test_smoke_templates_golden_path.py index 39107625..abc18402 100644 --- a/tests/test_smoke_templates_golden_path.py +++ b/tests/test_smoke_templates_golden_path.py @@ -6,7 +6,6 @@ import yaml from toolkit.cli.cmd_run import run as run_cmd -from toolkit.cli.cmd_validate import validate as validate_cmd from tests.helpers_assert_paths import ( assert_file_replaceable, assert_golden_path_artifacts, @@ -26,9 +25,8 @@ def test_smoke_offline_golden_path(smoke_offline: Path) -> None: config_path = smoke_offline / "dataset.yml" assert config_path.exists(), f"dataset.yml non trovato in {smoke_offline}" - # Run completa + # Run completa (include validazione interna dopo ogni layer) run_cmd(step="all", config=str(config_path)) - validate_cmd(step="all", config=str(config_path)) # Leggi configurazione per sapere anni e tabelle mart with open(config_path, encoding="utf-8") as f: diff --git a/toolkit/cli/app.py b/toolkit/cli/app.py index 28f7cf52..3ccb07ed 100644 --- a/toolkit/cli/app.py +++ b/toolkit/cli/app.py @@ -3,7 +3,6 @@ import typer from toolkit.cli.cmd_run import register as register_run -from toolkit.cli.cmd_validate import register as register_validate from toolkit.cli.cmd_inspect import register as register_inspect from toolkit.cli.cmd_batch import register as register_batch from toolkit.cli.cmd_contract import register as register_contract @@ -25,7 +24,6 @@ def _main_options( # registra comandi register_run(app) -register_validate(app) register_inspect(app) register_batch(app) register_contract(app) diff --git a/toolkit/cli/cmd_batch.py b/toolkit/cli/cmd_batch.py index afc6d9ce..85019364 100644 --- a/toolkit/cli/cmd_batch.py +++ b/toolkit/cli/cmd_batch.py @@ -1,117 +1,12 @@ +"""Batch execution — deprecato, usa ``toolkit run --batch ``.""" + from __future__ import annotations -import contextlib -import json -import logging -from pathlib import Path -from time import perf_counter -from typing import Any +import warnings import typer -from toolkit.cli.common import load_cfg_and_logger -from toolkit.cli.cmd_run import run_year - -_ALLOWED_STEPS = {"probe", "raw", "clean", "mart", "all"} - - -@contextlib.contextmanager -def _silence_typer_echo() -> Any: - """Silenzia typer.echo durante run_year quando --json è attivo. - - run_year(dry_run=True) stampa l'execution plan via typer.echo - su stdout. Per output JSON puro, intercettiamo temporaneamente - typer.echo e lo sostituiamo con un no-op. - """ - original_echo = typer.echo - typer.echo = lambda *args, **kwargs: None - try: - yield - finally: - typer.echo = original_echo - - -def _silence_logger() -> None: - """Silenzia il logger 'toolkit' per output JSON pulito su stdout.""" - logger = logging.getLogger("toolkit") - logger.setLevel(logging.CRITICAL + 1) - logger.handlers.clear() - logger.addHandler(logging.NullHandler()) - logger.propagate = False - - -def _read_config_list(configs_file: Path) -> list[Path]: - if not configs_file.exists(): - raise FileNotFoundError(f"Config list not found: {configs_file}") - - config_paths: list[Path] = [] - for raw_line in configs_file.read_text(encoding="utf-8").splitlines(): - line = raw_line.strip() - if not line or line.startswith("#"): - continue - - config_path = Path(line) - if not config_path.is_absolute(): - # Risolvi prima rispetto alla CWD (caso d'uso normale: - # lancio da root progetto con path relativi al root), - # poi rispetto al file batch come fallback. - cwd_resolved = (Path.cwd() / config_path).resolve() - if cwd_resolved.exists(): - config_path = cwd_resolved - else: - config_path = (configs_file.parent / config_path).resolve() - config_paths.append(config_path) - - if not config_paths: - raise ValueError(f"No config paths found in {configs_file}") - - return config_paths - - -def _format_years(years: list[int] | None) -> str: - if not years: - return "-" - return ",".join(str(year) for year in years) - - -def _format_duration(seconds: float | None) -> str: - if seconds is None: - return "-" - return f"{seconds:.3f}s" - - -def _print_table(rows: list[dict[str, str]], headers: list[str]) -> None: - widths = {header: len(header) for header in headers} - for row in rows: - for header in headers: - widths[header] = max(widths[header], len(str(row.get(header, "")))) - - def _render(row: dict[str, str]) -> str: - return " ".join(str(row.get(header, "")).ljust(widths[header]) for header in headers) - - typer.echo("Batch Report") - typer.echo(_render({header: header for header in headers})) - typer.echo(" ".join("-" * widths[header] for header in headers)) - for row in rows: - typer.echo(_render(row)) - - -def _build_row( - dataset: str, - config_path: str, - years: str, - step: str, - status: str, - duration: str, -) -> dict[str, str]: - return { - "dataset": dataset, - "config": config_path, - "years": years, - "step": step, - "status": status, - "duration": duration, - } +from toolkit.cli.cmd_run import _run_batch def batch( @@ -136,137 +31,31 @@ def batch( ), ): """ - Esegue più config in sequenza e stampa un report aggregato finale. + ⚠️ DEPRECATO: usa ``toolkit run --batch ``. - Legge un file di testo con un dataset.yml per riga (righe vuote e commenti - con # sono ignorati) e li esegue uno dopo l'altro per lo step indicato. + Esegue piu' config in sequenza e stampa un report aggregato finale. """ - if step not in _ALLOWED_STEPS: - raise typer.BadParameter("step must be one of: probe, raw, clean, mart, all") - - dry_flag = dry_run if isinstance(dry_run, bool) else False - sample_rows_final = 1000 if smoke else sample_rows - sample_bytes_final = 1048576 if smoke else sample_bytes - - configs_file = Path(configs) - config_paths = _read_config_list(configs_file) - - rows: list[dict[str, str]] = [] - failures: list[dict[str, str]] = [] - - for config_path in config_paths: - config_started_at = perf_counter() - dataset_label = config_path.stem - - try: - if smoke: - # Carica prima senza override per scoprire cfg.root originale, - # poi ricarica con root_override a {root}/smoke - _cfg0, _logger0 = load_cfg_and_logger(str(config_path)) - if json_output: - _silence_logger() - cfg, logger = load_cfg_and_logger( - str(config_path), - root_override=str(_cfg0.root / "smoke"), - ) - else: - cfg, logger = load_cfg_and_logger(str(config_path)) - - # Quando --json è attivo, silenzia il logger dopo ogni - # load_cfg_and_logger (che resetta il logger a ogni chiamata) - if json_output: - _silence_logger() - dataset_label = cfg.dataset - - for year in cfg.years: - run_started_at = perf_counter() - status = "FAILED" - try: - # Quando --json è attivo, silenzia typer.echo durante - # run_year per evitare che execution plan (dry-run) - # o altri echo contaminino stdout JSON - _run_ctx = _silence_typer_echo() if json_output else contextlib.nullcontext() - with _run_ctx: - context = run_year( - cfg, - year, - step=step, - dry_run=dry_flag, - logger=logger, - sample_rows=sample_rows_final, - sample_bytes=sample_bytes_final, - ) - status = context.status - except Exception as exc: - failures.append( - { - "config": str(config_path), - "dataset": dataset_label, - "years": str(year), - "error": str(exc), - } - ) - finally: - rows.append( - _build_row( - dataset=dataset_label, - config_path=str(config_path), - years=str(year), - step=step, - status=status, - duration=_format_duration(perf_counter() - run_started_at), - ) - ) - except Exception as exc: - failures.append( - { - "config": str(config_path), - "dataset": dataset_label, - "years": "-", - "error": str(exc), - } - ) - rows.append( - _build_row( - dataset=dataset_label, - config_path=str(config_path), - years="-", - step=step, - status="FAILED", - duration=_format_duration(perf_counter() - config_started_at), - ) - ) - - if json_output: - report: dict[str, Any] = { - "summary": { - "total": len(rows), - "passed": sum(1 for r in rows if r["status"] in ("SUCCESS", "DRY_RUN")), - "failed": sum(1 for r in rows if r["status"] not in ("SUCCESS", "DRY_RUN")), - "duration_seconds": sum( - float(r["duration"].rstrip("s")) for r in rows if r["duration"] != "-" - ), - }, - "rows": rows, - "failures": failures, - } - typer.echo(json.dumps(report, indent=2, default=str)) - else: - table_headers = ["dataset", "years", "step", "status", "duration"] - _print_table(rows, table_headers) - - if failures: - typer.echo("") - typer.echo("Failures") - for failure in failures: - typer.echo( - f"- config={failure['config']} dataset={failure['dataset']} " - f"years={failure['years']} error={failure['error']}" - ) - - if failures: - raise typer.Exit(code=1) + warnings.warn( + "'toolkit batch' è deprecato, usa 'toolkit run --batch '", + DeprecationWarning, + stacklevel=2, + ) + typer.echo( + "⚠️ 'toolkit batch' è deprecato, usa 'toolkit run --batch '", + err=True, + ) + _run_batch( + batch_file=configs, + step=step, + years=None, + smoke=smoke, + sample_rows=sample_rows, + sample_bytes=sample_bytes, + root=None, + json_output=json_output, + dry_run=dry_run, + ) def register(app: typer.Typer) -> None: - app.command("batch")(batch) + app.command("batch", hidden=True)(batch) diff --git a/toolkit/cli/cmd_run.py b/toolkit/cli/cmd_run.py index b77de982..b5e6d8b4 100644 --- a/toolkit/cli/cmd_run.py +++ b/toolkit/cli/cmd_run.py @@ -1,7 +1,10 @@ from __future__ import annotations +import contextlib import json +import logging from pathlib import Path +from time import perf_counter from typing import Any import typer @@ -748,6 +751,249 @@ def run_init( typer.echo("Prossimo passo: toolkit run clean -c ") +# --------------------------------------------------------------------------- +# Batch execution (toolkit run --batch) +# --------------------------------------------------------------------------- + + +_ALLOWED_BATCH_STEPS = {"probe", "raw", "clean", "mart", "all"} + + +def _read_config_list(configs_file: Path) -> list[Path]: + """Read a text file with one dataset.yml path per line.""" + if not configs_file.exists(): + raise FileNotFoundError(f"Config list not found: {configs_file}") + + config_paths: list[Path] = [] + for raw_line in configs_file.read_text(encoding="utf-8").splitlines(): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + config_path = Path(line) + if not config_path.is_absolute(): + cwd_resolved = (Path.cwd() / config_path).resolve() + if cwd_resolved.exists(): + config_path = cwd_resolved + else: + config_path = (configs_file.parent / config_path).resolve() + config_paths.append(config_path) + + if not config_paths: + raise ValueError(f"No config paths found in {configs_file}") + + return config_paths + + +def _format_years(years: list[int] | None) -> str: + if not years: + return "-" + return ",".join(str(year) for year in years) + + +def _format_duration(seconds: float | None) -> str: + if seconds is None: + return "-" + return f"{seconds:.3f}s" + + +def _print_table(rows: list[dict[str, str]], headers: list[str]) -> None: + widths = {header: len(header) for header in headers} + for row in rows: + for header in headers: + widths[header] = max(widths[header], len(str(row.get(header, "")))) + + def _render(row: dict[str, str]) -> str: + return " ".join(str(row.get(header, "")).ljust(widths[header]) for header in headers) + + typer.echo("Batch Report") + typer.echo(_render({header: header for header in headers})) + typer.echo(" ".join("-" * widths[header] for header in headers)) + for row in rows: + typer.echo(_render(row)) + + +def _build_row( + dataset: str, + config_path: str, + years: str, + step: str, + status: str, + duration: str, +) -> dict[str, str]: + return { + "dataset": dataset, + "config": config_path, + "years": years, + "step": step, + "status": status, + "duration": duration, + } + + +@contextlib.contextmanager +def _silence_typer_echo() -> Any: + """Silenzia typer.echo durante run_year quando --json è attivo.""" + original_echo = typer.echo + typer.echo = lambda *args, **kwargs: None # type: ignore[assignment] + try: + yield + finally: + typer.echo = original_echo + + +def _silence_logger() -> None: + """Silenzia il logger 'toolkit' per output JSON pulito su stdout.""" + lg = logging.getLogger("toolkit") + lg.setLevel(logging.CRITICAL + 1) + lg.handlers.clear() + lg.addHandler(logging.NullHandler()) + lg.propagate = False + + +def _run_batch( + batch_file: str, + step: str = "all", + years: str | None = None, + smoke: bool = False, + sample_rows: int | None = None, + sample_bytes: int | None = None, + root: str | None = None, + json_output: bool = False, + dry_run: bool = False, +) -> None: + """Esegue piu' config in sequenza e stampa un report aggregato. + + Legge un file di testo con un dataset.yml per riga (righe vuote e commenti + con # sono ignorati) e li esegue uno dopo l'altro. + + Args: + batch_file: Path al file con lista di dataset.yml. + step: Step da eseguire (probe, raw, clean, mart, all). Default ``all``. + years: Non usato in batch (ogni config usa i propri anni configurati). + smoke: Alias per --sample-rows 1000 --sample-bytes 1048576. + sample_rows: Limite righe in CLEAN. + sample_bytes: Limite bytes in RAW. + root: Override root output. + json_output: Output JSON. + dry_run: Solo plan senza esecuzione. + """ + dry_flag = dry_run if isinstance(dry_run, bool) else False + sample_rows_final = 1000 if smoke else sample_rows + sample_bytes_final = 1048576 if smoke else sample_bytes + + configs_file = Path(batch_file) + config_paths = _read_config_list(configs_file) + + rows: list[dict[str, str]] = [] + failures: list[dict[str, str]] = [] + + for config_path in config_paths: + config_started_at = perf_counter() + dataset_label = config_path.stem + + try: + if smoke: + _cfg0, _logger0 = load_cfg_and_logger(str(config_path)) + if json_output: + _silence_logger() + cfg, logger = load_cfg_and_logger( + str(config_path), + root_override=str(_cfg0.root / "smoke"), + ) + else: + cfg, logger = load_cfg_and_logger(str(config_path)) + + if json_output: + _silence_logger() + dataset_label = cfg.dataset + + for year in cfg.years: + run_started_at = perf_counter() + status = "FAILED" + try: + _run_ctx = _silence_typer_echo() if json_output else contextlib.nullcontext() + with _run_ctx: + context = run_year( + cfg, + year, + step=step, + dry_run=dry_flag, + logger=logger, + sample_rows=sample_rows_final, + sample_bytes=sample_bytes_final, + ) + status = context.status + except Exception as exc: + failures.append( + { + "config": str(config_path), + "dataset": dataset_label, + "years": str(year), + "error": str(exc), + } + ) + finally: + rows.append( + _build_row( + dataset=dataset_label, + config_path=str(config_path), + years=str(year), + step=step, + status=status, + duration=_format_duration(perf_counter() - run_started_at), + ) + ) + except Exception as exc: + failures.append( + { + "config": str(config_path), + "dataset": dataset_label, + "years": "-", + "error": str(exc), + } + ) + rows.append( + _build_row( + dataset=dataset_label, + config_path=str(config_path), + years="-", + step=step, + status="FAILED", + duration=_format_duration(perf_counter() - config_started_at), + ) + ) + + if json_output: + report: dict[str, Any] = { + "summary": { + "total": len(rows), + "passed": sum(1 for r in rows if r["status"] in ("SUCCESS", "DRY_RUN")), + "failed": sum(1 for r in rows if r["status"] not in ("SUCCESS", "DRY_RUN")), + "duration_seconds": sum( + float(r["duration"].rstrip("s")) for r in rows if r["duration"] != "-" + ), + }, + "rows": rows, + "failures": failures, + } + typer.echo(json.dumps(report, indent=2, default=str)) + else: + table_headers = ["dataset", "years", "step", "status", "duration"] + _print_table(rows, table_headers) + + if failures: + typer.echo("") + typer.echo("Failures") + for failure in failures: + typer.echo( + f"- config={failure['config']} dataset={failure['dataset']} " + f"years={failure['years']} error={failure['error']}" + ) + + if failures: + raise typer.Exit(code=1) + + # --------------------------------------------------------------------------- # Pipeline completa — condivisa tra `toolkit run` (default) e `run full` # --------------------------------------------------------------------------- @@ -1109,6 +1355,11 @@ def run_default( ), years: str | None = typer.Option(None, "--years", help="Comma-separated dataset years"), year: int | None = typer.Option(None, "--year", "-y", help="Single dataset year"), + batch: str | None = typer.Option( + None, + "--batch", + help="File con lista di dataset.yml (uno per riga) — esegue in sequenza", + ), smoke: bool = typer.Option( False, "--smoke", help="Alias per --sample-rows 1000 --sample-bytes 1048576" ), @@ -1124,9 +1375,21 @@ def run_default( json_output: bool = typer.Option(False, "--json", help="Output JSON report"), dry_run: bool = typer.Option(False, "--dry-run", help="Print plan without executing"), ): - """Esegue la pipeline completa: preflight + support + raw → clean → mart.""" + """Esegue la pipeline: singolo dataset o batch (--batch).""" if ctx.invoked_subcommand is not None: return + if batch: + _run_batch( + batch, + years=years, + smoke=smoke, + sample_rows=sample_rows, + sample_bytes=sample_bytes, + root=root, + json_output=json_output, + dry_run=dry_run, + ) + return # Unifica --year e --years years_str = str(year) if year is not None else years if year is not None and years is not None: diff --git a/toolkit/cli/cmd_validate.py b/toolkit/cli/cmd_validate.py deleted file mode 100644 index 61f17230..00000000 --- a/toolkit/cli/cmd_validate.py +++ /dev/null @@ -1,110 +0,0 @@ -from __future__ import annotations - -import json -from typing import Any - -import typer - -from toolkit.cli.common import load_cfg_and_logger -from toolkit.domain.common import iter_selected_years -from toolkit.clean.validate import run_clean_validation -from toolkit.core.dataset_loader import validate_config -from toolkit.mart.validate import run_mart_validation -from toolkit.raw.validate import run_raw_validation - -_VALIDATORS = { - "raw": lambda cfg, yr, lg: run_raw_validation(cfg.root, cfg.dataset, yr, lg), - "clean": run_clean_validation, - "mart": run_mart_validation, -} - - -def _validate_config_cmd(config_arg: str, as_json: bool) -> None: - """Valida dataset.yml — cammini obbligatori, coerenza campi.""" - result = validate_config(config_arg) - if as_json: - typer.echo(json.dumps(result, indent=2, ensure_ascii=False)) - else: - slug = result["slug"] - if result["errors"]: - for e in result["errors"]: - typer.echo(f"🔴 {slug}: {e}") - for w in result["warnings"]: - typer.echo(f"🟡 {slug}: {w}") - if result["ok"]: - typer.echo(f"✅ {slug}: configurazione valida") - if not result["ok"]: - raise typer.Exit(code=1) - - -def validate( - step: str = typer.Argument(..., help="raw | clean | mart | all | config"), - config: str | None = typer.Option(None, "--config", "-c", help="Path or slug to dataset.yml"), - year: int | None = typer.Option(None, "--year", "-y", help="Single dataset year"), - years: str | None = typer.Option(None, "--years", help="Comma-separated dataset years"), - as_json: bool = typer.Option(False, "--json", help="Emit JSON output"), -): - """ - Quality gate per dataset.yml, RAW, CLEAN e MART. - - - ``config``: valida il file dataset.yml (campi obbligatori, coerenza) - - ``raw/clean/mart``: valida output della pipeline - - ``all``: raw + clean + mart - """ - if step == "config": - _validate_config_cmd(config or "", as_json) - return - - cfg, logger = load_cfg_and_logger(config) - years_arg = years if isinstance(years, str) else None - year_arg = year if isinstance(year, int) else None - selected_years = iter_selected_years(cfg, year_arg=year_arg, years_arg=years_arg) - - # Silenzia logger per output JSON pulito - if as_json: - import logging as _logging - - _logging.getLogger("toolkit").setLevel(_logging.CRITICAL + 1) - - layers = ["raw", "clean", "mart"] if step == "all" else [step] - results: list[dict[str, Any]] = [] - - for yr in selected_years: - for layer in layers: - fn = _VALIDATORS[layer] - summary = fn(cfg, yr, logger) - results.append( - { - "year": yr, - "layer": layer, - "passed": summary.get("passed"), - "errors_count": summary.get("errors_count", 0), - "warnings_count": summary.get("warnings_count", 0), - "errors": summary.get("errors", []), - "warnings": summary.get("warnings", []), - } - ) - - any_failed = any(not r["passed"] for r in results) - - if as_json: - typer.echo(json.dumps(results, indent=2, ensure_ascii=False)) - if any_failed: - raise typer.Exit(code=1) - return - for r in results: - icon = "✅" if r["passed"] else "🔴" - typer.echo( - f"{icon} {r['year']}/{r['layer']} errors={r['errors_count']} warnings={r['warnings_count']}" - ) - for err in r.get("errors", []): - typer.echo(f" error: {err}") - for warn in r.get("warnings", []): - typer.echo(f" warning: {warn}") - - if any_failed: - raise typer.Exit(code=1) - - -def register(app: typer.Typer) -> None: - app.command("validate")(validate) From efc8452e58b423bdce60b9a70fa40bb884d635be Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Thu, 30 Jul 2026 21:09:21 +0100 Subject: [PATCH 2/4] fix: remove unused type: ignore comment in _silence_typer_echo --- toolkit/cli/cmd_run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/toolkit/cli/cmd_run.py b/toolkit/cli/cmd_run.py index b5e6d8b4..2e0bc996 100644 --- a/toolkit/cli/cmd_run.py +++ b/toolkit/cli/cmd_run.py @@ -834,7 +834,7 @@ def _build_row( def _silence_typer_echo() -> Any: """Silenzia typer.echo durante run_year quando --json è attivo.""" original_echo = typer.echo - typer.echo = lambda *args, **kwargs: None # type: ignore[assignment] + typer.echo = lambda *args, **kwargs: None try: yield finally: From 1a4b9ab08faec90b92d037dcbfc3cb8797a2bd54 Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Thu, 30 Jul 2026 21:16:52 +0100 Subject: [PATCH 3/4] refactor: extract batch helpers to _batch_helpers.py I 6 helper di formattazione e silenziamento (format_years, format_duration, print_table, build_row, silence_typer_echo, silence_logger) erano in cmd_run.py, ingrossandolo di ~60 righe. Estratti in toolkit/cli/_batch_helpers.py, importati da cmd_run.py. Riduce cmd_run.py di ~60 righe, mantiene separazione netta. --- tests/test_cmd_batch.py | 25 +++++----- toolkit/cli/_batch_helpers.py | 79 ++++++++++++++++++++++++++++++ toolkit/cli/cmd_run.py | 90 ++++++----------------------------- 3 files changed, 107 insertions(+), 87 deletions(-) create mode 100644 toolkit/cli/_batch_helpers.py diff --git a/tests/test_cmd_batch.py b/tests/test_cmd_batch.py index 875abf67..b56eda99 100644 --- a/tests/test_cmd_batch.py +++ b/tests/test_cmd_batch.py @@ -1,10 +1,11 @@ -"""Tests for batch helper functions (trasferite in cmd_run.py).""" +"""Tests for batch helper functions.""" from pathlib import Path import pytest -from toolkit.cli.cmd_run import _format_duration, _format_years, _read_config_list +from toolkit.cli._batch_helpers import format_duration, format_years +from toolkit.cli.cmd_run import _read_config_list pytestmark = pytest.mark.pure_unit @@ -145,33 +146,33 @@ def test_blank_lines_and_spaces_skipped(self, tmp_path: pytest.TempPathFactory) class TestFormatYears: def test_none(self) -> None: - assert _format_years(None) == "-" + assert format_years(None) == "-" def test_empty_list(self) -> None: - assert _format_years([]) == "-" + assert format_years([]) == "-" def test_single_year(self) -> None: - assert _format_years([2023]) == "2023" + assert format_years([2023]) == "2023" def test_multiple_years(self) -> None: - assert _format_years([2021, 2022, 2023]) == "2021,2022,2023" + assert format_years([2021, 2022, 2023]) == "2021,2022,2023" def test_returns_string(self) -> None: - assert isinstance(_format_years([2020]), str) + assert isinstance(format_years([2020]), str) class TestFormatDuration: def test_none_returns_dash(self) -> None: - assert _format_duration(None) == "-" + assert format_duration(None) == "-" def test_seconds_formatted(self) -> None: - assert _format_duration(1.234) == "1.234s" + assert format_duration(1.234) == "1.234s" def test_zero(self) -> None: - assert _format_duration(0.0) == "0.000s" + assert format_duration(0.0) == "0.000s" def test_rounds_to_3_decimals(self) -> None: - assert _format_duration(1.23456789) == "1.235s" + assert format_duration(1.23456789) == "1.235s" def test_returns_string(self) -> None: - assert isinstance(_format_duration(1.0), str) + assert isinstance(format_duration(1.0), str) diff --git a/toolkit/cli/_batch_helpers.py b/toolkit/cli/_batch_helpers.py new file mode 100644 index 00000000..41314085 --- /dev/null +++ b/toolkit/cli/_batch_helpers.py @@ -0,0 +1,79 @@ +"""Helper functions for batch execution — formattazione, silenziamento log.""" + +from __future__ import annotations + +import contextlib +import logging +from typing import Any + +import typer + + +def format_years(years: list[int] | None) -> str: + """Formatta lista anni in stringa compatta.""" + if not years: + return "-" + return ",".join(str(year) for year in years) + + +def format_duration(seconds: float | None) -> str: + """Formatta durata in secondi con 3 decimali.""" + if seconds is None: + return "-" + return f"{seconds:.3f}s" + + +def print_table(rows: list[dict[str, str]], headers: list[str]) -> None: + """Stampa tabella batch report su stdout.""" + widths = {header: len(header) for header in headers} + for row in rows: + for header in headers: + widths[header] = max(widths[header], len(str(row.get(header, "")))) + + def _render(row: dict[str, str]) -> str: + return " ".join(str(row.get(header, "")).ljust(widths[header]) for header in headers) + + typer.echo("Batch Report") + typer.echo(_render({header: header for header in headers})) + typer.echo(" ".join("-" * widths[header] for header in headers)) + for row in rows: + typer.echo(_render(row)) + + +def build_row( + dataset: str, + config_path: str, + years: str, + step: str, + status: str, + duration: str, +) -> dict[str, str]: + """Costruisce una riga per il report batch.""" + return { + "dataset": dataset, + "config": config_path, + "years": years, + "step": step, + "status": status, + "duration": duration, + } + + +@contextlib.contextmanager +def silence_typer_echo() -> Any: + """Silenzia typer.echo durante run_year quando --json è attivo.""" + original_echo = typer.echo + typer.echo = lambda *args, **kwargs: None + try: + yield + finally: + typer.echo = original_echo + + +def silence_logger() -> None: + """Silenzia il logger 'toolkit' per output JSON pulito su stdout.""" + lg = logging.getLogger("toolkit") + lg.setLevel(logging.CRITICAL + 1) + lg.handlers.clear() + lg.addHandler(logging.NullHandler()) + lg.propagate = False diff --git a/toolkit/cli/cmd_run.py b/toolkit/cli/cmd_run.py index 2e0bc996..15a417a9 100644 --- a/toolkit/cli/cmd_run.py +++ b/toolkit/cli/cmd_run.py @@ -2,13 +2,19 @@ import contextlib import json -import logging from pathlib import Path from time import perf_counter from typing import Any import typer +from toolkit.cli._batch_helpers import ( + build_row, + format_duration, + print_table, + silence_logger, + silence_typer_echo, +) from toolkit.cli.common import dump_cfg_section, load_cfg_and_logger from toolkit.domain.common import iter_selected_years from toolkit.domain.source_utils import resolve_source as _resolve_source @@ -784,72 +790,6 @@ def _read_config_list(configs_file: Path) -> list[Path]: return config_paths -def _format_years(years: list[int] | None) -> str: - if not years: - return "-" - return ",".join(str(year) for year in years) - - -def _format_duration(seconds: float | None) -> str: - if seconds is None: - return "-" - return f"{seconds:.3f}s" - - -def _print_table(rows: list[dict[str, str]], headers: list[str]) -> None: - widths = {header: len(header) for header in headers} - for row in rows: - for header in headers: - widths[header] = max(widths[header], len(str(row.get(header, "")))) - - def _render(row: dict[str, str]) -> str: - return " ".join(str(row.get(header, "")).ljust(widths[header]) for header in headers) - - typer.echo("Batch Report") - typer.echo(_render({header: header for header in headers})) - typer.echo(" ".join("-" * widths[header] for header in headers)) - for row in rows: - typer.echo(_render(row)) - - -def _build_row( - dataset: str, - config_path: str, - years: str, - step: str, - status: str, - duration: str, -) -> dict[str, str]: - return { - "dataset": dataset, - "config": config_path, - "years": years, - "step": step, - "status": status, - "duration": duration, - } - - -@contextlib.contextmanager -def _silence_typer_echo() -> Any: - """Silenzia typer.echo durante run_year quando --json è attivo.""" - original_echo = typer.echo - typer.echo = lambda *args, **kwargs: None - try: - yield - finally: - typer.echo = original_echo - - -def _silence_logger() -> None: - """Silenzia il logger 'toolkit' per output JSON pulito su stdout.""" - lg = logging.getLogger("toolkit") - lg.setLevel(logging.CRITICAL + 1) - lg.handlers.clear() - lg.addHandler(logging.NullHandler()) - lg.propagate = False - - def _run_batch( batch_file: str, step: str = "all", @@ -895,7 +835,7 @@ def _run_batch( if smoke: _cfg0, _logger0 = load_cfg_and_logger(str(config_path)) if json_output: - _silence_logger() + silence_logger() cfg, logger = load_cfg_and_logger( str(config_path), root_override=str(_cfg0.root / "smoke"), @@ -904,14 +844,14 @@ def _run_batch( cfg, logger = load_cfg_and_logger(str(config_path)) if json_output: - _silence_logger() + silence_logger() dataset_label = cfg.dataset for year in cfg.years: run_started_at = perf_counter() status = "FAILED" try: - _run_ctx = _silence_typer_echo() if json_output else contextlib.nullcontext() + _run_ctx = silence_typer_echo() if json_output else contextlib.nullcontext() with _run_ctx: context = run_year( cfg, @@ -934,13 +874,13 @@ def _run_batch( ) finally: rows.append( - _build_row( + build_row( dataset=dataset_label, config_path=str(config_path), years=str(year), step=step, status=status, - duration=_format_duration(perf_counter() - run_started_at), + duration=format_duration(perf_counter() - run_started_at), ) ) except Exception as exc: @@ -953,13 +893,13 @@ def _run_batch( } ) rows.append( - _build_row( + build_row( dataset=dataset_label, config_path=str(config_path), years="-", step=step, status="FAILED", - duration=_format_duration(perf_counter() - config_started_at), + duration=format_duration(perf_counter() - config_started_at), ) ) @@ -979,7 +919,7 @@ def _run_batch( typer.echo(json.dumps(report, indent=2, default=str)) else: table_headers = ["dataset", "years", "step", "status", "duration"] - _print_table(rows, table_headers) + print_table(rows, table_headers) if failures: typer.echo("") From e28e9ef96999435f8b8ad459cc9e7a5286f576ed Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Thu, 30 Jul 2026 21:19:30 +0100 Subject: [PATCH 4/4] docs: aggiorna README e smoke per CLI changes - Fix tabella CLI in README.md (|---|---|---| -> | |) - Aggiungi run --batch alla tabella comandi - feature-stability.md: validate all -> run --batch - Smoke README: inspect summary -> inspect, run all/full -> run --- README.md | 5 +++-- docs/feature-stability.md | 4 ++-- smoke/bdap_ckan_csv/README.md | 2 +- smoke/bdap_http_csv/README.md | 2 +- smoke/finanze_http_zip_2023/README.md | 2 +- smoke/istat_sdmx_22_289/README.md | 2 +- smoke/local_file_csv/README.md | 6 +++--- smoke/zip_http_csv/README.md | 2 +- 8 files changed, 13 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 58fa25ab..df80effb 100644 --- a/README.md +++ b/README.md @@ -53,8 +53,9 @@ La CI di `dataset-incubator` carica su GCS dopo ogni run validato. ## CLI — comandi essenziali | Comando | Cosa fa | -|---|---|---| -| `toolkit run` | Esecuzione completa RAW→CLEAN→MART | +|---|---| +| `toolkit run` | Esecuzione completa RAW→CLEAN→MART (default) | +| `toolkit run --batch ` | Esegue piú dataset in sequenza | | `toolkit run raw` | Solo layer RAW | | `toolkit run clean` | Solo layer CLEAN | | `toolkit run mart` | Solo layer MART | diff --git a/docs/feature-stability.md b/docs/feature-stability.md index e9d59cc9..f15ecb28 100644 --- a/docs/feature-stability.md +++ b/docs/feature-stability.md @@ -7,7 +7,7 @@ Questa matrice serve a chiarire cosa il toolkit considera percorso canonico, cos | `query` | stable | query SQL su parquet (path o dataset.yml + layer) | | `parquet_preview(sql=...)` | stable | API core per SQL arbitrario su parquet | | `run` | stable | percorso canonico | -| `validate all` | stable | percorso canonico | +| `run --batch` | stable | batch esecuzione sequenziale | | `inspect` | stable | percorso canonico | | path contract di `dataset.yml` | stable | percorso canonico | | output `raw/clean/mart/_runs` | stable | percorso canonico | @@ -26,7 +26,7 @@ Questa matrice serve a chiarire cosa il toolkit considera percorso canonico, cos Lettura equivalente a livello package: -- core runtime: `toolkit.raw`, `toolkit.clean`, `toolkit.mart`, `toolkit.scout`, `toolkit.cli` (`run`, `validate`, `inspect`) +- core runtime: `toolkit.raw`, `toolkit.clean`, `toolkit.mart`, `toolkit.scout`, `toolkit.cli` (`run`, `inspect`, `batch` deprecato) - advanced tooling: `inspect runs --resume`, run parziali, `inspect config --mode profile`, `inspect config --diff` - compatibility only: config legacy e alias storici diff --git a/smoke/bdap_ckan_csv/README.md b/smoke/bdap_ckan_csv/README.md index 461086b6..80d254b2 100644 --- a/smoke/bdap_ckan_csv/README.md +++ b/smoke/bdap_ckan_csv/README.md @@ -9,7 +9,7 @@ toolkit run raw --config dataset.yml toolkit profile raw --config dataset.yml toolkit run clean --config dataset.yml toolkit run mart --config dataset.yml -toolkit inspect summary -c dataset.yml --year 2022 +toolkit inspect -c dataset.yml --year 2022 ``` ## Verifiche attese diff --git a/smoke/bdap_http_csv/README.md b/smoke/bdap_http_csv/README.md index 0b765245..d4263e2b 100644 --- a/smoke/bdap_http_csv/README.md +++ b/smoke/bdap_http_csv/README.md @@ -9,7 +9,7 @@ toolkit run raw --config dataset.yml toolkit profile raw --config dataset.yml toolkit run clean --config dataset.yml toolkit run mart --config dataset.yml -toolkit inspect summary -c dataset.yml --year 2022 +toolkit inspect -c dataset.yml --year 2022 ``` ## Verifiche attese diff --git a/smoke/finanze_http_zip_2023/README.md b/smoke/finanze_http_zip_2023/README.md index ce3d82a7..e71698e8 100644 --- a/smoke/finanze_http_zip_2023/README.md +++ b/smoke/finanze_http_zip_2023/README.md @@ -5,7 +5,7 @@ Smoke manuale per `http_file` su ZIP pubblico del MEF con extractor `unzip_first ## Comandi ```bash -py -m toolkit.cli.app run all -c smoke/finanze_http_zip_2023/dataset.yml +py -m toolkit.cli.app run -c smoke/finanze_http_zip_2023/dataset.yml py -m toolkit.cli.app status --dataset finanze_http_zip_2023 --year 2023 --config smoke/finanze_http_zip_2023/dataset.yml ``` diff --git a/smoke/istat_sdmx_22_289/README.md b/smoke/istat_sdmx_22_289/README.md index 27ffbfdd..a9071a01 100644 --- a/smoke/istat_sdmx_22_289/README.md +++ b/smoke/istat_sdmx_22_289/README.md @@ -9,7 +9,7 @@ toolkit run raw --config dataset.yml toolkit profile raw --config dataset.yml toolkit run clean --config dataset.yml toolkit run mart --config dataset.yml -toolkit inspect summary -c dataset.yml --year 2024 +toolkit inspect -c dataset.yml --year 2024 ``` ## Verifiche attese diff --git a/smoke/local_file_csv/README.md b/smoke/local_file_csv/README.md index 5b5f7b2f..91bf32d7 100644 --- a/smoke/local_file_csv/README.md +++ b/smoke/local_file_csv/README.md @@ -21,9 +21,9 @@ toolkit run raw --config dataset.yml toolkit profile raw --config dataset.yml toolkit run clean --config dataset.yml toolkit run mart --config dataset.yml -toolkit run all --config dataset.yml -toolkit run full --config dataset.yml -toolkit inspect summary -c dataset.yml --year 2022 +toolkit run --config dataset.yml +toolkit run --config dataset.yml +toolkit inspect -c dataset.yml --year 2022 ``` ## Verifiche attese diff --git a/smoke/zip_http_csv/README.md b/smoke/zip_http_csv/README.md index c82b58cc..3ecda7fa 100644 --- a/smoke/zip_http_csv/README.md +++ b/smoke/zip_http_csv/README.md @@ -17,7 +17,7 @@ toolkit run raw --config dataset.yml toolkit profile raw --config dataset.yml toolkit run clean --config dataset.yml toolkit run mart --config dataset.yml -toolkit inspect summary -c dataset.yml --year 2022 +toolkit inspect -c dataset.yml --year 2022 ``` ## Verifiche attese