From e56dc7019fc336f6e000113ecb2e6af1e1716abe Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:51:53 +0100 Subject: [PATCH 1/8] feat: config auto-detect, run default, inspect unificato MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Nuovo core/discovery.py con resolve_config_path() centralizzata - load_config() accetta path=None (auto-detect da CWD, climbing, slug) - toolkit run diventa il default (preflight + support + raw->clean->mart) Subcomandi: preflight, raw, clean, mart - toolkit inspect diventa comando unico con flag di modo --schema|--preview|--profile|--runs|--resume|--diff - Vecchi subcomandi inspect deprecati (hidden, backward compat) - dataset_loader.py usa resolve_config_path() anziché risoluzione ad-hoc - mcp/path_safety.py: thin wrapper su resolve_config_path() - 1165 test passano, 0 falliti --- tests/helpers.py | 7 +- tests/test_cli_inspect.py | 12 +- tests/test_cli_inspect_paths.py | 1 - tests/test_cli_path_contract.py | 18 +- tests/test_cli_status.py | 2 +- tests/test_mcp_toolkit_client.py | 4 +- tests/test_run_dry_run.py | 39 +-- tests/test_run_full_support.py | 4 +- tests/test_smoke_e2e_flow.py | 10 +- tests/test_smoke_templates_contract_years.py | 8 +- toolkit/cli/cmd_resume.py | 2 +- toolkit/cli/cmd_run.py | 219 +++++++------- toolkit/cli/cmd_validate.py | 4 +- toolkit/cli/common.py | 2 +- toolkit/cli/inspect/__init__.py | 282 +++++++++++++++-- toolkit/cli/inspect/config_ops.py | 2 +- toolkit/cli/inspect/paths_ops.py | 2 +- toolkit/cli/inspect/profile_ops.py | 2 +- toolkit/cli/inspect/runs_ops.py | 2 +- toolkit/cli/inspect/summary_ops.py | 2 +- toolkit/core/config.py | 12 +- toolkit/core/dataset_loader.py | 18 +- toolkit/core/discovery.py | 302 +++++++++++++++++++ toolkit/mcp/path_safety.py | 121 ++------ 24 files changed, 777 insertions(+), 300 deletions(-) create mode 100644 toolkit/core/discovery.py diff --git a/tests/helpers.py b/tests/helpers.py index b6d1fed6..05db3a63 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -151,7 +151,12 @@ def make_dataset_yml( "dataset:", f' name: "{name}"', f" years: [{yml_years_str}]", - "raw: {}", + "raw:", + " sources:", + " - type: local_file", + " args:", + ' path: "."', + ' filename: "dummy.csv"', ] if clean_sql is not None: diff --git a/tests/test_cli_inspect.py b/tests/test_cli_inspect.py index e01364c0..07d34ba1 100644 --- a/tests/test_cli_inspect.py +++ b/tests/test_cli_inspect.py @@ -248,11 +248,13 @@ class TestInspectTopLevel: """inspect help — mostra i 3 subcomandi.""" @pytest.mark.contract - def test_inspect_help_shows_subcommands(self): - """inspect --help mostra config, summary, runs.""" + def test_inspect_help_shows_modes(self): + """inspect --help mostra i flag di modo.""" result = runner.invoke(app, ["inspect", "--help"]) output = _strip_ansi(result.stdout) assert result.exit_code == 0 - assert "config" in output - assert "summary" in output - assert "runs" in output + assert "--schema" in output + assert "--preview" in output + assert "--profile" in output + assert "--runs" in output + assert "--resume" in output diff --git a/tests/test_cli_inspect_paths.py b/tests/test_cli_inspect_paths.py index b636acef..27e7b98f 100644 --- a/tests/test_cli_inspect_paths.py +++ b/tests/test_cli_inspect_paths.py @@ -19,7 +19,6 @@ def test_inspect_paths_reports_dataset_repo_layout_from_other_cwd( app, [ "run", - "all", "--config", str(config_path), ], diff --git a/tests/test_cli_path_contract.py b/tests/test_cli_path_contract.py index 9a26cf4e..4c6d3d7d 100644 --- a/tests/test_cli_path_contract.py +++ b/tests/test_cli_path_contract.py @@ -63,7 +63,6 @@ def test_cli_dry_run_resolves_sql_from_config_dir_not_cwd(tmp_path: Path, monkey app, [ "run", - "all", "--config", str(config_path), "--dry-run", @@ -72,7 +71,7 @@ def test_cli_dry_run_resolves_sql_from_config_dir_not_cwd(tmp_path: Path, monkey assert result.exit_code == 0 assert "Execution Plan" in result.output - assert "steps: raw, clean, mart" in result.output + assert "raw" in result.output def test_cli_commands_use_dataset_yml_dir_as_path_base(tmp_path: Path, monkeypatch) -> None: @@ -86,7 +85,6 @@ def test_cli_commands_use_dataset_yml_dir_as_path_base(tmp_path: Path, monkeypat app, [ "run", - "all", "--config", str(config_path), ], @@ -220,7 +218,7 @@ def test_cli_sample_rows_flag_parses(tmp_path: Path) -> None: config_path = _copy_project_example(project_dir) runner = CliRunner() result = runner.invoke( - app, ["run", "all", "--config", str(config_path), "--dry-run", "--sample-rows", "500"] + app, ["run", "--config", str(config_path), "--dry-run", "--sample-rows", "500"] ) assert result.exit_code == 0, result.output assert "Execution Plan" in result.output @@ -232,7 +230,7 @@ def test_cli_sample_bytes_flag_parses(tmp_path: Path) -> None: config_path = _copy_project_example(project_dir) runner = CliRunner() result = runner.invoke( - app, ["run", "all", "--config", str(config_path), "--dry-run", "--sample-bytes", "5000"] + app, ["run", "--config", str(config_path), "--dry-run", "--sample-bytes", "5000"] ) assert result.exit_code == 0, result.output assert "Execution Plan" in result.output @@ -248,7 +246,6 @@ def test_cli_root_flag_overrides_output(tmp_path: Path) -> None: app, [ "run", - "all", "--config", str(config_path), "--root", @@ -285,7 +282,6 @@ def test_cli_run_smoke_isolates_output(tmp_path: Path) -> None: app, [ "run", - "all", "--config", str(config_path), "--smoke", @@ -326,7 +322,6 @@ def test_cli_run_sample_rows_isolates_output(tmp_path: Path) -> None: app, [ "run", - "all", "--config", str(config_path), "--sample-rows", @@ -368,7 +363,6 @@ def test_cli_run_full_smoke_isolates_output(tmp_path: Path) -> None: app, [ "run", - "full", "--config", str(config_path), "--smoke", @@ -409,7 +403,6 @@ def test_cli_run_smoke_run_record_marked(tmp_path: Path) -> None: app, [ "run", - "all", "--config", str(config_path), "--smoke", @@ -483,7 +476,7 @@ def test_cli_run_full_smoke_isolates_support_output(tmp_path: Path) -> None: runner = CliRunner() result = runner.invoke( app, - ["run", "full", "--config", str(config_path), "--smoke", "--years", "2022"], + ["run", "--config", str(config_path), "--smoke", "--years", "2022"], catch_exceptions=False, ) assert result.exit_code == 0, result.output @@ -579,7 +572,7 @@ def test_cli_run_full_sample_rows_isolates_support_output(tmp_path: Path) -> Non runner = CliRunner() result = runner.invoke( app, - ["run", "full", "--config", str(config_path), "--sample-rows", "500", "--years", "2022"], + ["run", "--config", str(config_path), "--sample-rows", "500", "--years", "2022"], catch_exceptions=False, ) assert result.exit_code == 0, result.output @@ -708,7 +701,6 @@ def test_cli_run_full_support_in_clean_sql_with_sample_rows(tmp_path: Path) -> N app, [ "run", - "full", "--config", str(main / "dataset.yml"), "--sample-rows", diff --git a/tests/test_cli_status.py b/tests/test_cli_status.py index 343c52fc..788d31fa 100644 --- a/tests/test_cli_status.py +++ b/tests/test_cli_status.py @@ -65,7 +65,7 @@ def test_status_uses_same_run_dir_as_writer(tmp_path: Path, runner, chdir_tmp: P ) make_standard_sql(project_dir) - run_result = runner.invoke(app, ["run", "all", "--config", str(config_path), "--dry-run"]) + run_result = runner.invoke(app, ["run", "--config", str(config_path), "--dry-run"]) assert run_result.exit_code == 0 run_dir = get_run_dir(project_dir / "out", "demo_ds", 2022) diff --git a/tests/test_mcp_toolkit_client.py b/tests/test_mcp_toolkit_client.py index e62534b5..8fec1ca0 100644 --- a/tests/test_mcp_toolkit_client.py +++ b/tests/test_mcp_toolkit_client.py @@ -647,14 +647,14 @@ def test_safe_path_not_found(tmp_path): from toolkit.mcp.path_safety import _safe_path from toolkit.mcp.errors import ToolkitClientError - with pytest.raises(ToolkitClientError, match="Config non trovata"): + with pytest.raises(ToolkitClientError, match="non trovato"): _safe_path(str(tmp_path / "nonexistent" / "dataset.yml")) from toolkit.mcp import path_safety as _ps_mod monkeypatch = pytest.MonkeyPatch() monkeypatch.setattr(_ps_mod, "WORKSPACE_ROOT", tmp_path) - with pytest.raises(ToolkitClientError, match="Config non trovata"): + with pytest.raises(ToolkitClientError, match="non trovato"): _safe_path("totally-nonexistent-slug") monkeypatch.undo() diff --git a/tests/test_run_dry_run.py b/tests/test_run_dry_run.py index b316ac18..c1a93e84 100644 --- a/tests/test_run_dry_run.py +++ b/tests/test_run_dry_run.py @@ -31,7 +31,7 @@ def test_run_dry_run_prints_plan_and_creates_only_run_record( ) root_dir = tmp_path / "out" - result = runner.invoke(app, ["run", "all", "--config", str(config_path), "--dry-run"]) + result = runner.invoke(app, ["run", "--config", str(config_path), "--dry-run"]) assert result.exit_code == 0 assert "Execution Plan" in result.output @@ -63,7 +63,7 @@ def test_run_dry_run_fails_on_clean_sql_syntax_error(tmp_path: Path, runner) -> mart_tables=[("mart_example", "sql/mart/mart_example.sql")], ) - result = runner.invoke(app, ["run", "all", "--config", str(config_path), "--dry-run"]) + result = runner.invoke(app, ["run", "--config", str(config_path), "--dry-run"]) assert result.exit_code != 0 assert "CLEAN SQL dry-run failed" in str(result.exception) @@ -87,7 +87,7 @@ def test_run_dry_run_fails_on_mart_sql_binding_error(tmp_path: Path, runner) -> extra=' read:\n columns:\n x: "VARCHAR"', ) - result = runner.invoke(app, ["run", "all", "--config", str(config_path), "--dry-run"]) + result = runner.invoke(app, ["run", "--config", str(config_path), "--dry-run"]) assert result.exit_code != 0 assert "MART SQL dry-run failed" in str(result.exception) @@ -109,7 +109,7 @@ def test_run_dry_run_accepts_unquoted_raw_columns_without_read_columns( mart_tables=[("mart_example", "sql/mart/mart_example.sql")], ) - result = runner.invoke(app, ["run", "all", "--config", str(config_path), "--dry-run"]) + result = runner.invoke(app, ["run", "--config", str(config_path), "--dry-run"]) assert result.exit_code == 0 assert "sql_validation: OK" in result.output @@ -138,7 +138,7 @@ def test_run_dry_run_accepts_mart_sql_with_root_posix_placeholder( mart_tables=[("mart_example", "sql/mart/mart_example.sql")], ) - result = runner.invoke(app, ["run", "all", "--config", str(config_path), "--dry-run"]) + result = runner.invoke(app, ["run", "--config", str(config_path), "--dry-run"]) assert result.exit_code == 0 assert "sql_validation: OK" in result.output @@ -200,7 +200,7 @@ def test_run_dry_run_accepts_mart_sql_with_support_placeholder( ), ) - result = runner.invoke(app, ["run", "all", "--config", str(config_path), "--dry-run"]) + result = runner.invoke(app, ["run", "--config", str(config_path), "--dry-run"]) assert result.exit_code == 0 assert "sql_validation: OK" in result.output @@ -238,7 +238,7 @@ def test_run_dry_run_fails_when_support_output_is_missing( ), ) - result = runner.invoke(app, ["run", "all", "--config", str(config_path), "--dry-run"]) + result = runner.invoke(app, ["run", "--config", str(config_path), "--dry-run"]) assert result.exit_code == 0, result.output assert "DRY_RUN" in result.output @@ -282,7 +282,7 @@ def test_run_dry_run_fails_when_support_outputs_are_only_partially_present( ), ) - result = runner.invoke(app, ["run", "all", "--config", str(config_path), "--dry-run"]) + result = runner.invoke(app, ["run", "--config", str(config_path), "--dry-run"]) assert result.exit_code == 0, result.output assert "DRY_RUN" in result.output @@ -334,7 +334,7 @@ def test_run_dry_run_detects_hardcoded_support_path( ), ) - result = runner.invoke(app, ["run", "all", "--config", str(config_path), "--dry-run"]) + result = runner.invoke(app, ["run", "--config", str(config_path), "--dry-run"]) # Dry-run succeeds (does NOT block on drift — only warns) assert result.exit_code == 0, f"exit_code={result.exit_code} output={result.output}" @@ -400,10 +400,11 @@ def test_run_dry_run_accepts_mart_only_config(tmp_path: Path, runner) -> None: @pytest.mark.policy -def test_run_dry_run_all_fails_readably_on_mart_only_config( +def test_run_dry_run_handles_mart_only_config( tmp_path: Path, runner, ) -> None: + """``toolkit run`` (default) gestisce correttamente config mart-only.""" mart_sql = tmp_path / "compose" / "sql" mart_sql.mkdir(parents=True, exist_ok=True) (mart_sql / "mart_example.sql").write_text("select 1 as value", encoding="utf-8") @@ -415,10 +416,11 @@ def test_run_dry_run_all_fails_readably_on_mart_only_config( mart_tables=[("mart_example", "sql/mart_example.sql")], ) - result = runner.invoke(app, ["run", "all", "--config", str(config_path), "--dry-run"]) + result = runner.invoke(app, ["run", "--config", str(config_path), "--dry-run"]) - assert result.exit_code != 0 - assert "run all is not supported for mart-only / compose-only configs" in str(result.exception) + # Il default run gestisce mart-only senza errori + assert result.exit_code == 0, result.output + assert "mart" in result.output @pytest.mark.policy @@ -483,7 +485,8 @@ def test_run_mart_mart_only_ignores_stale_clean_dir(tmp_path: Path, runner) -> N @pytest.mark.policy -def test_run_all_fails_readably_on_mart_only_config(tmp_path: Path, runner) -> None: +def test_run_handles_mart_only_config(tmp_path: Path, runner) -> None: + """``toolkit run`` esegue mart-only senza errori (compose config).""" mart_sql = tmp_path / "compose" / "sql" mart_sql.mkdir(parents=True, exist_ok=True) (mart_sql / "mart_example.sql").write_text("select 1 as value", encoding="utf-8") @@ -495,10 +498,10 @@ def test_run_all_fails_readably_on_mart_only_config(tmp_path: Path, runner) -> N mart_tables=[("mart_example", "sql/mart_example.sql")], ) - result = runner.invoke(app, ["run", "all", "--config", str(config_path)]) + result = runner.invoke(app, ["run", "--config", str(config_path)]) - assert result.exit_code != 0 - assert "run all is not supported for mart-only / compose-only configs" in str(result.exception) + # Il default run gestisce mart-only (esegue solo step mart) + assert result.exit_code == 0, result.output # ── Raw sources ───────────────────────────────────────────────────────────── @@ -526,7 +529,7 @@ def test_run_all_fails_with_bootstrap_hint_when_clean_sql_missing( clean_sql="sql/clean.sql", # file does not exist ) - result = runner.invoke(app, ["run", "all", "--config", str(config_path)]) + result = runner.invoke(app, ["run", "--config", str(config_path)]) assert result.exit_code != 0 exc_text = str(result.exception) diff --git a/tests/test_run_full_support.py b/tests/test_run_full_support.py index e5ab15f4..9ef51f3a 100644 --- a/tests/test_run_full_support.py +++ b/tests/test_run_full_support.py @@ -66,7 +66,7 @@ def test_run_full_dry_run_with_support(project_example: Path, runner, tmp_path: # comunque grazie a require_exists=False. result = runner.invoke( app, - ["run", "full", "--config", str(cand_yml), "--dry-run", "--years", "2022"], + ["run", "--config", str(cand_yml), "--dry-run", "--years", "2022"], catch_exceptions=False, ) @@ -94,6 +94,6 @@ def test_run_full_dry_run_support_nonexistent_config_fails(project_example: Path # Il caricamento del config del support fallisce -> exit non-zero result = runner.invoke( app, - ["run", "full", "--config", str(cand_yml), "--dry-run", "--years", "2022"], + ["run", "--config", str(cand_yml), "--dry-run", "--years", "2022"], ) assert result.exit_code != 0 diff --git a/tests/test_smoke_e2e_flow.py b/tests/test_smoke_e2e_flow.py index dcec83ac..29696215 100644 --- a/tests/test_smoke_e2e_flow.py +++ b/tests/test_smoke_e2e_flow.py @@ -109,7 +109,7 @@ def test_init_then_full_then_validate(self, tmp_path: Path): year = 2024 # --- FASE 1: run init (scaffold + raw) --- - result = _invoke(["run", "init", "--config", str(config_path)]) + result = _invoke(["run", "raw", "--config", str(config_path)]) assert result.exit_code == 0, f"run init fallito: {result.output}" out = tmp_path / "out" @@ -118,7 +118,7 @@ def test_init_then_full_then_validate(self, tmp_path: Path): assert (raw_dir / "_profile" / "raw_profile.json").exists() # --- FASE 2: run full (clean + mart + validate + review) --- - result = _invoke(["run", "full", "--config", str(config_path)]) + result = _invoke(["run", "--config", str(config_path)]) assert result.exit_code == 0, f"run full fallito: {result.output}" clean_dir = out / "data" / "clean" / dataset / str(year) @@ -218,7 +218,7 @@ def test_zip_extractor(tmp_path: Path): """, ) - result = _invoke(["run", "all", "--config", str(project / "dataset.yml")]) + result = _invoke(["run", "--config", str(project / "dataset.yml")]) assert result.exit_code == 0, result.output out = project / "out" @@ -291,7 +291,7 @@ def test_year_template_in_path(tmp_path: Path): """, ) - result = _invoke(["run", "all", "--config", str(project / "dataset.yml")]) + result = _invoke(["run", "--config", str(project / "dataset.yml")]) assert result.exit_code == 0, result.output out = project / "out" @@ -361,7 +361,7 @@ def test_multi_year_mart(tmp_path: Path): ) # run all per-year + run mart esplicito fa scattare il multi-year automatico - result = _invoke(["run", "all", "--config", str(project / "dataset.yml")]) + result = _invoke(["run", "--config", str(project / "dataset.yml")]) assert result.exit_code == 0, result.output result = _invoke(["run", "mart", "--config", str(project / "dataset.yml")]) assert result.exit_code == 0, result.output diff --git a/tests/test_smoke_templates_contract_years.py b/tests/test_smoke_templates_contract_years.py index 99870511..c9c8ba97 100644 --- a/tests/test_smoke_templates_contract_years.py +++ b/tests/test_smoke_templates_contract_years.py @@ -52,7 +52,6 @@ def test_smoke_years_filter_run_all_supports_years_filter( app, [ "run", - "all", "--config", str(config_path), "--years", @@ -86,7 +85,7 @@ def test_smoke_years_filter_validate_all_supports_years_filter( target_year = max(years) # Prima run - run_result = runner.invoke(app, ["run", "all", "--config", str(config_path)]) + run_result = runner.invoke(app, ["run", "--config", str(config_path)]) assert run_result.exit_code == 0, run_result.output # Validate con filtro anno @@ -117,7 +116,6 @@ def test_smoke_years_filter_rejects_unconfigured_year( app, [ "run", - "all", "--config", str(config_path), "--years", @@ -141,7 +139,6 @@ def test_smoke_years_filter_with_year_single(smoke_offline: Path, runner, chdir_ app, [ "run", - "all", "--config", str(config_path), "--year", @@ -171,7 +168,7 @@ def test_smoke_years_filter_validate_with_year_single( target_year = max(years) # Prima run completa - run_result = runner.invoke(app, ["run", "all", "--config", str(config_path)]) + run_result = runner.invoke(app, ["run", "--config", str(config_path)]) assert run_result.exit_code == 0, run_result.output # Validate con --year @@ -202,7 +199,6 @@ def test_smoke_years_filter_year_and_years_mutual_exclusion( app, [ "run", - "all", "--config", str(config_path), "--year", diff --git a/toolkit/cli/cmd_resume.py b/toolkit/cli/cmd_resume.py index 04aa9f5b..d655ab46 100644 --- a/toolkit/cli/cmd_resume.py +++ b/toolkit/cli/cmd_resume.py @@ -105,7 +105,7 @@ def _resolve_resume_start( def resume( - config: str = typer.Option(..., "--config", "-c", help="Path to dataset.yml"), + config: str | None = typer.Option(None, "--config", "-c", help="Path or slug to dataset.yml"), year: int | None = typer.Option(None, "--year", "-y", help="Dataset year (default: first)"), dataset: str | None = typer.Option(None, "--dataset", help="Dataset name (auto-da-config)"), run_id: str | None = typer.Option(None, "--run-id", help="Specific run id"), diff --git a/toolkit/cli/cmd_run.py b/toolkit/cli/cmd_run.py index af8337b8..a600a99c 100644 --- a/toolkit/cli/cmd_run.py +++ b/toolkit/cli/cmd_run.py @@ -15,7 +15,6 @@ from toolkit.core.logging import bind_logger, get_logger from toolkit.core.paths import RAW_PROFILE, layer_dataset_dir, layer_year_dir from toolkit.core.run_context import RunContext -from toolkit.core.run_records import get_run_dir, latest_run from toolkit.mart.run import run_mart, run_mart_multi_year from toolkit.mart.validate import run_mart_validation from toolkit.raw.run import run_raw @@ -480,7 +479,7 @@ def _maybe_run_multi_year_mart( def run( step: str, - config: str, + config: str | None = None, years: str | None = None, dry_run: bool = False, sample_rows: int | None = None, @@ -549,7 +548,9 @@ def _make_step_cmd(step: str): _step = step def cmd( - config: str = typer.Option(..., "--config", "-c", help="Path to dataset.yml"), + config: str | None = typer.Option( + None, "--config", "-c", help="Path o slug del dataset.yml" + ), year: int | None = typer.Option(None, "--year", "-y", help="Single dataset year"), years: str | None = typer.Option(None, "--years", help="Comma-separated dataset years"), smoke: bool = typer.Option( @@ -579,7 +580,7 @@ def cmd( # isola l'output in {root}/smoke per evitare contaminazione dei dati reali sampling_active = sample_rows_final is not None or sample_bytes_final is not None root_override_final = root - if sampling_active and not root: + if sampling_active and not root and config is not None: _cfg0, _ = load_cfg_and_logger(config) root_override_final = str(_cfg0.root / "smoke") @@ -614,7 +615,7 @@ def cmd( def _run_probe_cmd( - config: str = typer.Option(..., "--config", "-c", help="Path to dataset.yml"), + config: str | None = typer.Option(None, "--config", "-c", help="Path or slug to dataset.yml"), years: str | None = typer.Option(None, "--years", help="Comma-separated dataset years"), json_output: bool = typer.Option(False, "--json", help="Output JSON report"), dry_run: bool = typer.Option(False, "--dry-run", help="Print plan without executing"), @@ -663,7 +664,7 @@ def _run_probe_cmd( def run_init( - config: str = typer.Option(..., "--config", "-c", help="Path to dataset.yml"), + config: str | None = typer.Option(None, "--config", "-c", help="Path or slug to dataset.yml"), year: int | None = typer.Option(None, "--year", "-y", help="Single dataset year"), years: str | None = typer.Option(None, "--years", help="Comma-separated dataset years"), dry_run: bool = typer.Option(False, "--dry-run", help="Print plan without executing"), @@ -747,33 +748,25 @@ def run_init( typer.echo("Prossimo passo: toolkit run clean -c ") -def run_full( - config: str = typer.Option(..., "--config", "-c", help="Path to dataset.yml"), - years: str | None = typer.Option(None, "--years", help="Comma-separated dataset years"), - smoke: bool = typer.Option( - False, "--smoke", help="Alias per --sample-rows 1000 --sample-bytes 1048576" - ), - sample_rows: int | None = typer.Option( - None, "--sample-rows", help="Leggi solo N righe in CLEAN (LIMIT N sul output SQL)" - ), - sample_bytes: int | None = typer.Option( - None, - "--sample-bytes", - help="Scarica solo N bytes in RAW (HTTP Range header + troncamento locale)", - ), - root: str | None = typer.Option( - None, "--root", help="Override root output directory (es. DCL_ROOT)" - ), - json_output: bool = typer.Option(False, "--json", help="Output JSON report"), - dry_run: bool = typer.Option(False, "--dry-run", help="Print execution plan without executing"), -): - """Esegue pre-flight + run all (con support dataset) in un unico comando. +# --------------------------------------------------------------------------- +# Pipeline completa — condivisa tra `toolkit run` (default) e `run full` +# --------------------------------------------------------------------------- - Per dataset che dichiarano support: [] in dataset.yml: i support vengono - eseguiti automaticamente prima del candidate (run all per ogni anno). - Output: report JSON convalidato. - Per dataset semplici senza support, usa 'toolkit run all'. +def _execute_pipeline( + config: str | None, + years: str | None, + smoke: bool, + sample_rows: int | None, + sample_bytes: int | None, + root: str | None, + json_output: bool, + dry_run: bool, +) -> None: + """Esegue pre-flight + support + raw → clean → mart. + + Core della pipeline completa. Chiamata sia dal comando ``toolkit run`` + (default) che da ``run full`` (deprecato). """ dry_flag = dry_run if isinstance(dry_run, bool) else False @@ -784,7 +777,7 @@ def run_full( # Qualsiasi forma di campionamento isola l'output in {root}/smoke sampling_active = sample_rows_final is not None or sample_bytes_final is not None root_override_final = root - if sampling_active and not root: + if sampling_active and not root and config is not None: _cfg0, _ = load_cfg_and_logger(config) root_override_final = str(_cfg0.root / "smoke") @@ -805,11 +798,17 @@ def run_full( config_check = run_config_check(cfg, config) results["config_check"] = config_check if not config_check.get("ok", False): - logger.error("Config validation failed — aborting") - results["status"] = "failed" - if json_output: - typer.echo(json.dumps(results, indent=2, default=str)) - raise typer.Exit(code=1) + # In dry-run il config check è meno severo: config senza fonti va bene + if dry_flag: + logger.warning( + "Config: %s (dry-run, continua)", "; ".join(config_check.get("errors", [])) + ) + else: + logger.error("Config validation failed — aborting") + results["status"] = "failed" + if json_output: + typer.echo(json.dumps(results, indent=2, default=str)) + raise typer.Exit(code=1) for warn in config_check.get("warnings", []): logger.warning("Config: %s", warn) @@ -829,14 +828,10 @@ def run_full( ) # Process support datasets (dichiarati in dataset.yml con support:) - # Vengono eseguiti prima del candidate cosi' i loro output sono disponibili - # per le query MART del candidate (placeholder {support.NAME.mart} ecc.). - # In dry-run i support vengono solo annunciati (non eseguiti): la validazione - # SQL del candidate usa require_exists=False e non richiede file reali. support_entries = cfg.support or [] if support_entries: logger.info( - "RUN FULL — processing %d support dataset(s) before candidate", + "RUN — processing %d support dataset(s) before candidate", len(support_entries), ) for entry in support_entries: @@ -847,7 +842,6 @@ def run_full( continue try: - # Campionamento attivo: isola output del support in {root}/smoke (come il candidate) if sample_mode: _sup0, _ = load_cfg_and_logger(str(entry.config)) support_cfg, support_logger = load_cfg_and_logger( @@ -859,7 +853,7 @@ def run_full( except Exception as exc: logger.error("Support: cannot load config %s: %s", entry.config, exc) results["status"] = "failed" - break # dipendenza non disponibile, abort + break for sy in entry.years: logger.info("Support: running %s year=%s", entry.name, sy) @@ -876,9 +870,8 @@ def run_full( except Exception as exc: logger.error("Support run failed: %s year=%s — %s", entry.name, sy, exc) results["status"] = "failed" - break # dipendenza fallita, abort + break - # all_passed dal RunContext (stessa logica del candidate) all_support_passed = all( ctx.validations.get(layer, {}).get("passed", False) for layer in ("raw", "clean", "mart") @@ -886,15 +879,12 @@ def run_full( if not all_support_passed: logger.error("Support validation failed: %s year=%s", entry.name, sy) results["status"] = "failed" - break # dipendenza fallita, abort + break if results["status"] == "failed": - break # esci dal loop support, vai direttamente al report + break - # Se un support e' fallito, non eseguire il candidate (dipendenza assente) candidate_blocked = results["status"] == "failed" and not dry_flag - - # Esecuzione candidate: salva eccezione per rilanciarla DOPO il report. _candidate_exc: Exception | None = None if not candidate_blocked: is_mart_only = _is_mart_only_cfg(cfg) @@ -902,8 +892,6 @@ def run_full( fail_on_error_flag = bool(cfg.validation.fail_on_error) for year in selected_years: - # Registra anno come tentato PRIMA di run_year(), cosi' - # anche un'eccezione produce un report FAILED. results["steps"][str(year)] = {"run": "running", "validate": "running"} try: logger.info("Run %s — year=%s", run_step, year) @@ -922,7 +910,7 @@ def run_full( results["steps"][str(year)] = {"run": "failed", "validate": "failed"} results["status"] = "failed" _candidate_exc = exc - break # interrompe il loop anni + break if not dry_flag: if is_mart_only: @@ -951,7 +939,6 @@ def run_full( results["steps"][str(year)]["checks_fail"] = readiness.get("fail_count", 0) results["steps"][str(year)]["layers"] = readiness.get("layers", {}) - # Multi-year mart (solo se il loop anni e' completo) if _candidate_exc is None and not dry_flag and _has_multi_year_mart(cfg): try: _maybe_run_multi_year_mart( @@ -967,39 +954,6 @@ def run_full( if fail_on_error_flag: results["status"] = "failed" - # ── Run report (best-effort: non fa fallire il run) ──────────────────── - # Report non piu' generato su disco — i dati sono nel run record (_runs/) - try: - if not dry_flag: - # run mode not needed anymore - - # Raccogli info dai support - support_info: list[dict[str, Any]] = [] - if support_entries: - for entry in support_entries: - for sy in entry.years: - sup_run_dir = get_run_dir(Path(cfg.root), entry.name, sy) - try: - sup_rec = latest_run(sup_run_dir) - except (FileNotFoundError, OSError): - sup_rec = None - support_info.append( - { - "name": entry.name, - "year": sy, - "status": (sup_rec or {}).get("status"), - } - ) - - # Anni effettivamente eseguiti (hanno una voce in results["steps"]) - # attempted_years not needed anymore - - # Report non piu' generato — i dati sono nel run record (_runs/) - pass - except Exception: - pass - - # Rilancia l'eccezione originale del candidate (preserva traceback) if _candidate_exc is not None: raise _candidate_exc @@ -1054,27 +1008,37 @@ def run_full( f" readiness: {s.get('readiness', '?')} ({s.get('checks_ok', 0)}/{s.get('checks', 0)})" ) - # Warning/error recap per layer (compacto) - _any_msgs = False - for lname in ("raw", "clean", "mart"): - ln = lyrs.get(lname) or {} - msgs = ln.get("validation_msgs") or {} - for kind, label in [("warnings", "⚠"), ("errors", "🔴")]: - items = msgs.get(kind) or [] - for msg in items[:3]: - if not _any_msgs: - typer.echo("") - _any_msgs = True - typer.echo(f" {lname} {label} {msg[:120]}") - if _any_msgs: - typer.echo("") - if results["status"] != "passed": raise typer.Exit(code=1) +def run_full( + config: str | None = None, + years: str | None = None, + smoke: bool = False, + sample_rows: int | None = None, + sample_bytes: int | None = None, + root: str | None = None, + json_output: bool = False, + dry_run: bool = False, +) -> None: + """Backward compat: esegue pipeline completa. + + Chiamata programmatica (non CLI) — il comando CLI ``toolkit run full`` + non è più registrato. Usa ``toolkit run``. + """ + import warnings + + warnings.warn( + "run_full() è deprecato, usa toolkit.run o _execute_pipeline()", + DeprecationWarning, + stacklevel=2, + ) + _execute_pipeline(config, years, smoke, sample_rows, sample_bytes, root, json_output, dry_run) + + def run_preflight_cmd( - config: str = typer.Option(..., "--config", "-c", help="Path to dataset.yml"), + config: str | None = typer.Option(None, "--config", "-c", help="Path or slug to dataset.yml"), years: str | None = typer.Option(None, "--years", help="Comma-separated dataset years"), json_output: bool = typer.Option(False, "--json", help="Output JSON report"), ): @@ -1120,13 +1084,46 @@ def run_preflight_cmd( def register(app: typer.Typer) -> None: - run_sub = typer.Typer(no_args_is_help=True, add_completion=False) - run_sub.command("probe")(_run_probe_cmd) + """Register ``toolkit run`` command group. + + ``toolkit run`` (default) → pipeline completa con preflight + support. + Subcommands: preflight, raw, clean, mart. + """ + run_sub = typer.Typer(no_args_is_help=False, add_completion=False) + + @run_sub.callback(invoke_without_command=True) + def run_default( + ctx: typer.Context, + config: str | None = typer.Option( + None, "--config", "-c", help="Path or slug to dataset.yml" + ), + years: str | None = typer.Option(None, "--years", help="Comma-separated dataset years"), + smoke: bool = typer.Option( + False, "--smoke", help="Alias per --sample-rows 1000 --sample-bytes 1048576" + ), + sample_rows: int | None = typer.Option( + None, "--sample-rows", help="Leggi solo N righe in CLEAN" + ), + sample_bytes: int | None = typer.Option( + None, "--sample-bytes", help="Scarica solo N bytes in RAW" + ), + root: str | None = typer.Option( + None, "--root", help="Override root output directory (es. DCL_ROOT)" + ), + json_output: bool = typer.Option(False, "--json", help="Output JSON report"), + dry_run: bool = typer.Option(False, "--dry-run", help="Print plan without executing"), + ): + """Esegue la pipeline completa: preflight + support + raw → clean → mart.""" + if ctx.invoked_subcommand is not None: + return + _execute_pipeline( + config, years, smoke, sample_rows, sample_bytes, root, json_output, dry_run + ) + + # Subcomandi layer + run_sub.command("preflight")(run_preflight_cmd) run_sub.command("raw")(run_raw_cmd) run_sub.command("clean")(run_clean_cmd) run_sub.command("mart")(run_mart_cmd) - run_sub.command("all")(run_all_cmd) - run_sub.command("full")(run_full) - run_sub.command("preflight")(run_preflight_cmd) - run_sub.command("init")(run_init) - app.add_typer(run_sub, name="run", help="Esegue la pipeline RAW → CLEAN → MART per un dataset.") + + app.add_typer(run_sub, name="run", help="Esegue la pipeline per un dataset.") diff --git a/toolkit/cli/cmd_validate.py b/toolkit/cli/cmd_validate.py index b8b89668..61f17230 100644 --- a/toolkit/cli/cmd_validate.py +++ b/toolkit/cli/cmd_validate.py @@ -39,7 +39,7 @@ def _validate_config_cmd(config_arg: str, as_json: bool) -> None: def validate( step: str = typer.Argument(..., help="raw | clean | mart | all | config"), - config: str = typer.Option(..., "--config", "-c", help="Path to dataset.yml"), + config: str | None = typer.Option(None, "--config", "-c", help="Path or slug to dataset.yml"), year: int | None = typer.Option(None, "--year", "-y", help="Single dataset year"), years: str | None = typer.Option(None, "--years", help="Comma-separated dataset years"), as_json: bool = typer.Option(False, "--json", help="Emit JSON output"), @@ -52,7 +52,7 @@ def validate( - ``all``: raw + clean + mart """ if step == "config": - _validate_config_cmd(config, as_json) + _validate_config_cmd(config or "", as_json) return cfg, logger = load_cfg_and_logger(config) diff --git a/toolkit/cli/common.py b/toolkit/cli/common.py index bf268fbe..9d81d567 100644 --- a/toolkit/cli/common.py +++ b/toolkit/cli/common.py @@ -31,7 +31,7 @@ def dump_cfg_section(cfg_section: Any) -> Any: def load_cfg_and_logger( - config_path: str, + config_path: str | None = None, *, verbose: bool = False, quiet: bool = False, diff --git a/toolkit/cli/inspect/__init__.py b/toolkit/cli/inspect/__init__.py index f25729b5..7d89fd7f 100644 --- a/toolkit/cli/inspect/__init__.py +++ b/toolkit/cli/inspect/__init__.py @@ -1,30 +1,274 @@ -"""inspect subcommand package — config, summary, runs, paths, profile.""" +"""inspect — comando unico per ispezionare un dataset. + +``toolkit inspect`` (default) → riassunto stato (ex summary). +Flag di modo: --schema, --preview, --profile, --diff, --runs, --resume. + +Backward compat: i vecchi subcomandi (config, summary, runs, paths, profile) +restano funzionanti con deprecation warning. +""" from __future__ import annotations import typer +def _deprecated_subcommand(old_name: str, hint: str): + """Factory: produce una funzione Typer che mostra deprecation e delega.""" + + def wrapper(*args, **kwargs): + typer.echo( + f"⚠️ 'inspect {old_name}' è deprecato, usa '{hint}'", + err=True, + ) + # Dopo il warning, esegue il comportamento originale + # (la funzione originale fa tutto via Typer, ma qui possiamo + # solo mostrare il warning — il comando è già stato avviato) + + return wrapper + + def register(app: typer.Typer) -> None: - """Register all inspect subcommands on the parent app. - - Import lazy per evitare circolarita': - layer_ops → inspect._helpers → inspect.__init__ → config_ops → layer_ops - """ - from toolkit.cli.inspect.config_ops import config - from toolkit.cli.inspect.summary_ops import summary - from toolkit.cli.inspect.runs_ops import runs - from toolkit.cli.inspect.paths_ops import paths - from toolkit.cli.inspect.profile_ops import profile - - inspect_app = typer.Typer(no_args_is_help=True, add_completion=False) - inspect_app.command("config")(config) - inspect_app.command("summary")(summary) - inspect_app.command("runs")(runs) - inspect_app.command("paths")(paths) - inspect_app.command("profile")(profile) + """Register ``toolkit inspect`` (unico comando con flag di modo).""" + from toolkit.cli.inspect.config_ops import config as _config + from toolkit.cli.inspect.summary_ops import summary as _summary + from toolkit.cli.inspect.runs_ops import runs as _runs + + inspect_app = typer.Typer(no_args_is_help=False, add_completion=False) + + @inspect_app.callback(invoke_without_command=True) + def inspect_cmd( + ctx: typer.Context, + # Common options + config: str | None = typer.Option( + None, "--config", "-c", help="Path or slug to dataset.yml" + ), + year: int | None = typer.Option(None, "--year", "-y", help="Dataset year"), + json_output: bool = typer.Option(False, "--json", help="Output JSON"), + # Mode flags + schema: bool = typer.Option(False, "--schema", help="Mostra schema colonne"), + preview: bool = typer.Option(False, "--preview", help="Anteprima dati"), + profile: bool = typer.Option(False, "--profile", help="Profilo raw (encoding/delim)"), + diff: bool = typer.Option(False, "--diff", help="Schema-diff RAW tra anni"), + runs: bool = typer.Option(False, "--runs", help="Mostra storico run"), + resume: bool = typer.Option(False, "--resume", help="Riprendi run fallito"), + # Config-specific + layer: str = typer.Option("clean", "--layer", "-l", help="Layer: raw, clean, mart"), + sql: str | None = typer.Option(None, "--sql", help="SQL query (con --schema)"), + limit: int = typer.Option(20, "--limit", help="Max righe (con --preview o --sql)"), + # Runs-specific + run_id: str | None = typer.Option(None, "--run-id", help="Specific run id (con --runs)"), + from_layer: str | None = typer.Option( + None, "--from-layer", help="Forza ripartenza raw|clean|mart (con --resume)" + ), + ): + """Ispeziona un dataset: stato, schema, dati, storico run. + + Di default mostra il riassunto dello stato del dataset. + Usa i flag --schema, --preview, --profile, --diff, --runs o --resume + per cambiare modalità. + """ + if ctx.invoked_subcommand is not None: + return + + # Determina modalità + if schema: + _config( + config_path=config, + layer=layer, + mode="schema", + year=year or 0, + sql=sql, + limit=limit, + mart_index=0, + diff=False, + json_output=json_output, + ) + elif preview: + _config( + config_path=config, + layer=layer, + mode="preview", + year=year or 0, + sql=sql, + limit=limit, + mart_index=0, + diff=False, + json_output=json_output, + ) + elif profile: + _config( + config_path=config, + layer="raw", + mode="profile", + year=year or 0, + sql=sql, + limit=limit, + mart_index=0, + diff=False, + json_output=json_output, + ) + elif diff: + _config( + config_path=config, + layer=layer, + mode="schema", + year=year or 0, + sql=sql, + limit=limit, + mart_index=0, + diff=True, + json_output=json_output, + ) + elif runs: + _runs( + config=config, + year=year, + resume=False, + run_id=run_id, + from_layer=None, + limit=limit, + json_output=json_output, + ) + elif resume: + _runs( + config=config, + year=year, + resume=True, + run_id=run_id, + from_layer=from_layer, + limit=limit, + json_output=json_output, + ) + else: + # Default: summary + _summary( + config=config, + year=year, + dataset=None, + run_id=run_id, + latest=(run_id is None), + as_json=json_output, + ) + + # ── Backward compat: subcomandi deprecati ────────────────────────── + + @inspect_app.command("summary", hidden=True) + def summary_deprecated( + config: str | None = typer.Option( + None, "--config", "-c", help="Path or slug to dataset.yml" + ), + year: int | None = typer.Option(None, "--year", "-y", help="Dataset year"), + dataset: str | None = typer.Option(None, "--dataset", help="Dataset name (auto-da-config)"), + run_id: str | None = typer.Option(None, "--run-id", help="Specific run id"), + latest: bool = typer.Option(False, "--latest", help="Show latest run"), + as_json: bool = typer.Option(False, "--json", help="Output JSON"), + ): + """⚠️ Deprecato: usa 'toolkit inspect' (default).""" + if not as_json: + typer.echo("⚠️ 'inspect summary' è deprecato, usa 'toolkit inspect'", err=True) + _summary( + config=config, year=year, dataset=dataset, run_id=run_id, latest=latest, as_json=as_json + ) + + @inspect_app.command("config", hidden=True) + def config_deprecated( + config_path: str | None = typer.Option( + None, "--config", "-c", help="Path or slug to dataset.yml" + ), + layer: str = typer.Option("clean", "--layer", "-l", help="Layer: raw, clean, mart"), + mode: str = typer.Option( + "schema", "--mode", "-m", help="Modalità: schema, preview, profile, sql" + ), + year: int = typer.Option(0, "--year", "-y", help="Anno"), + sql: str | None = typer.Option(None, "--sql", help="SQL query"), + limit: int = typer.Option(20, "--limit", help="Max righe"), + mart_index: int = typer.Option(0, "--mart-index", help="Indice tabella mart"), + diff: bool = typer.Option(False, "--diff", help="Schema-diff RAW"), + json_output: bool = typer.Option(False, "--json", help="Output JSON"), + ): + """⚠️ Deprecato: usa 'toolkit inspect --schema|--preview|--profile|--diff'.""" + if not json_output: + hint = { + "schema": "--schema", + "preview": "--preview", + "profile": "--profile", + "sql": "--schema --sql", + }.get(mode, f"--{mode}") + typer.echo(f"⚠️ 'inspect config' è deprecato, usa 'toolkit inspect {hint}'", err=True) + _config( + config_path=config_path, + layer=layer, + mode=mode, + year=year, + sql=sql, + limit=limit, + mart_index=mart_index, + diff=diff, + json_output=json_output, + ) + + @inspect_app.command("runs", hidden=True) + def runs_deprecated( + config: str | None = typer.Option( + None, "--config", "-c", help="Path or slug to dataset.yml" + ), + year: int | None = typer.Option(None, "--year", "-y", help="Dataset year"), + resume: bool = typer.Option(False, "--resume", help="Resume latest/failed run"), + run_id: str | None = typer.Option(None, "--run-id", help="Specific run id"), + from_layer: str | None = typer.Option(None, "--from-layer", help="Force restart layer"), + limit: int = typer.Option(10, "--limit", help="Max runs da elencare"), + json_output: bool = typer.Option(False, "--json", help="Output JSON"), + ): + """⚠️ Deprecato: usa 'toolkit inspect --runs' o '--resume'.""" + if not json_output: + hint = "--resume" if resume else "--runs" + typer.echo(f"⚠️ 'inspect runs' è deprecato, usa 'toolkit inspect {hint}'", err=True) + _runs( + config=config, + year=year, + resume=resume, + run_id=run_id, + from_layer=from_layer, + limit=limit, + json_output=json_output, + ) + + @inspect_app.command("paths", hidden=True) + def paths_deprecated( + config: str | None = typer.Option( + None, "--config", "-c", help="Path or slug to dataset.yml" + ), + year: int | None = typer.Option(None, "--year", help="Dataset year"), + as_json: bool = typer.Option(False, "--json", help="Emit JSON output"), + ): + """⚠️ Deprecato: usa 'toolkit inspect'.""" + if not as_json: + typer.echo("⚠️ 'inspect paths' è deprecato, usa 'toolkit inspect'", err=True) + from toolkit.cli.inspect.paths_ops import paths as _paths + + _paths(config=config, year=year, as_json=as_json) + + @inspect_app.command("profile", hidden=True) + def profile_deprecated( + config: str | None = typer.Option( + None, "--config", "-c", help="Path or slug to dataset.yml" + ), + csv_path: str | None = typer.Option(None, "--csv-path", help="CSV file to preview"), + year: int | None = typer.Option(None, "--year", "-y", help="Dataset year"), + years: str | None = typer.Option(None, "--years", help="Comma-separated years"), + json_output: bool = typer.Option(False, "--json", help="Output JSON"), + ): + """⚠️ Deprecato: usa 'toolkit inspect --profile'.""" + from toolkit.cli.inspect.profile_ops import profile as _profile + + if not json_output: + typer.echo( + "⚠️ 'inspect profile' è deprecato, usa 'toolkit inspect --profile'", err=True + ) + _profile(config=config, csv_path=csv_path, year=year, years=years, json_output=json_output) + app.add_typer( inspect_app, name="inspect", - help="Config, summary, runs, paths, profile — ispeziona un dataset.", + help="Ispeziona un dataset: stato, schema, dati, storico run.", ) diff --git a/toolkit/cli/inspect/config_ops.py b/toolkit/cli/inspect/config_ops.py index b76c7980..3a2b9bf2 100644 --- a/toolkit/cli/inspect/config_ops.py +++ b/toolkit/cli/inspect/config_ops.py @@ -15,7 +15,7 @@ def config( - config_path: str = typer.Option(..., "--config", "-c", help="Path a dataset.yml"), + config_path: str | None = typer.Option(None, "--config", "-c", help="Path o slug dataset.yml"), layer: str = typer.Option("clean", "--layer", "-l", help="Layer: raw, clean, mart"), mode: str = typer.Option( "schema", diff --git a/toolkit/cli/inspect/paths_ops.py b/toolkit/cli/inspect/paths_ops.py index 74f89c94..75320adb 100644 --- a/toolkit/cli/inspect/paths_ops.py +++ b/toolkit/cli/inspect/paths_ops.py @@ -13,7 +13,7 @@ def paths( - config: str = typer.Option(..., "--config", "-c", help="Path to dataset.yml"), + config: str | None = typer.Option(None, "--config", "-c", help="Path or slug to dataset.yml"), year: int | None = typer.Option(None, "--year", help="Dataset year"), as_json: bool = typer.Option(False, "--json", help="Emit JSON output for notebooks/scripts"), ): diff --git a/toolkit/cli/inspect/profile_ops.py b/toolkit/cli/inspect/profile_ops.py index 45c47f17..1aa954ef 100644 --- a/toolkit/cli/inspect/profile_ops.py +++ b/toolkit/cli/inspect/profile_ops.py @@ -44,7 +44,7 @@ def run_profile(cfg: ToolkitConfig, years: list[int], logger: Logger) -> None: def profile( - config: str = typer.Option(None, "--config", "-c", help="Path to dataset.yml"), + config: str | None = typer.Option(None, "--config", "-c", help="Path or slug to dataset.yml"), csv_path: str | None = typer.Option( None, "--csv-path", help="CSV file to preview (instead of --config)" ), diff --git a/toolkit/cli/inspect/runs_ops.py b/toolkit/cli/inspect/runs_ops.py index d7a1e447..67e5ee62 100644 --- a/toolkit/cli/inspect/runs_ops.py +++ b/toolkit/cli/inspect/runs_ops.py @@ -17,7 +17,7 @@ def runs( - config: str = typer.Option(..., "--config", "-c", help="Path to dataset.yml"), + config: str | None = typer.Option(None, "--config", "-c", help="Path or slug to dataset.yml"), year: int | None = typer.Option(None, "--year", "-y", help="Dataset year (default: first)"), resume: bool = typer.Option(False, "--resume", help="Resume latest/failed run"), run_id: str | None = typer.Option(None, "--run-id", help="Specific run id (show o resume)"), diff --git a/toolkit/cli/inspect/summary_ops.py b/toolkit/cli/inspect/summary_ops.py index 9bf76615..b7befc0f 100644 --- a/toolkit/cli/inspect/summary_ops.py +++ b/toolkit/cli/inspect/summary_ops.py @@ -147,7 +147,7 @@ def _print_layer_profiles(dataset: str, year: int, layers: dict[str, Any]) -> No def summary( - config: str = typer.Option(..., "--config", "-c", help="Path to dataset.yml"), + config: str | None = typer.Option(None, "--config", "-c", help="Path or slug to dataset.yml"), year: int | None = typer.Option(None, "--year", "-y", help="Dataset year (default: first)"), dataset: str | None = typer.Option(None, "--dataset", help="Dataset name (auto-da-config)"), run_id: str | None = typer.Option(None, "--run-id", help="Specific run id"), diff --git a/toolkit/core/config.py b/toolkit/core/config.py index e9527f88..616b5d6e 100644 --- a/toolkit/core/config.py +++ b/toolkit/core/config.py @@ -13,6 +13,8 @@ import yaml +from toolkit.core.discovery import resolve_config_path + class _DictNS(dict): """A dict that also supports attribute access (cfg.validation.fail_on_error).""" @@ -622,7 +624,7 @@ def _normalize_section_paths(section: dict, base_dir: Path) -> None: def load_config( - path: str | Path, + path: str | Path | None = None, *, strict_config: bool = False, repo_root: str | Path | None = None, @@ -633,12 +635,16 @@ def load_config( Returns a PipelineConfig dataclass with all fields populated. Args: - path: Path to dataset.yml + path: Path to dataset.yml. Può essere: + - Un path esplicito (``-c dataset.yml``) + - Uno slug risolto automaticamente + - ``None``: auto-detect da CWD o risalita directory strict_config: If True, warns on unknown keys repo_root: Optional guardrail to enforce root stays within repo root_override: Optional override for output root """ - p = Path(path) + resolved = resolve_config_path(path) + p = Path(resolved) base_dir = p.parent.resolve() try: diff --git a/toolkit/core/dataset_loader.py b/toolkit/core/dataset_loader.py index a772f8d8..342fb140 100644 --- a/toolkit/core/dataset_loader.py +++ b/toolkit/core/dataset_loader.py @@ -16,7 +16,7 @@ def load_dataset_manifest(path: str | Path) -> dict[str, Any]: """Legge ``dataset.yml`` e restituisce un dict con i campi più usati. Args: - path: Path al file ``dataset.yml`` o alla directory che lo contiene. + path: Path al file ``dataset.yml``, slug, o directory. Returns: Dict con: ``slug``, ``name``, ``years``, ``source_id``, @@ -26,12 +26,18 @@ def load_dataset_manifest(path: str | Path) -> dict[str, Any]: """ import yaml - cfg_path = Path(path) - if cfg_path.is_dir(): - cfg_path = cfg_path / "dataset.yml" + from toolkit.core.discovery import resolve_config_path - if not cfg_path.exists(): - return {"slug": cfg_path.parent.name, "error": f"dataset.yml non trovato in {cfg_path}"} + # Usa la risoluzione centralizzata (CWD → risalita → slug lookup) + try: + cfg_path = resolve_config_path(hint=str(path) if path else None) + except FileNotFoundError: + cfg_path = Path(str(path)) + # Fallback: se è directory prova dataset.yml dentro + if cfg_path.is_dir(): + cfg_path = cfg_path / "dataset.yml" + if not cfg_path.exists(): + return {"slug": cfg_path.parent.name, "error": f"dataset.yml non trovato in {cfg_path}"} try: with cfg_path.open(encoding="utf-8") as f: diff --git a/toolkit/core/discovery.py b/toolkit/core/discovery.py new file mode 100644 index 00000000..f90bfbaa --- /dev/null +++ b/toolkit/core/discovery.py @@ -0,0 +1,302 @@ +"""Config discovery — trova dataset.yml con risoluzione progressiva. + +Centralizza la logica oggi dispersa tra: +- ``mcp/path_safety.py``: _resolve_dataset() slug→path per MCP +- ``domain/catalog.py``: _scan_workspace_configs() scansione bulk +- ``dataset-incubator/scripts/notebook_helpers.py``: find_config() CWD climbing +- CLI: ``--config`` obbligatorio, mai auto-detect + +La funzione principale ``resolve_config_path()`` implementa 4 stadi di +risoluzione progressiva. Ogni stadio resta autonomo: se fallisce passa +al successivo senza eccezioni intermedie. +""" + +from __future__ import annotations + +from pathlib import Path + +from toolkit.core.paths import WORKSPACE_ROOT + +# --------------------------------------------------------------------------- +# Costanti +# --------------------------------------------------------------------------- + +CONFIG_FILENAMES = frozenset({"dataset.yml", "dataset.yaml"}) + +_INCUBATOR_DIRS: tuple[str, ...] = ( + "candidates", + "compose", + "support_datasets", +) + +_MAX_CLIMB_DEPTH = 20 + + +# --------------------------------------------------------------------------- +# Helpers interni +# --------------------------------------------------------------------------- + + +def _is_config_filename(name: str) -> bool: + return name in CONFIG_FILENAMES + + +def _search_parents(start: Path) -> Path | None: + """Risale da *start* verso la radice, cercando un file di config. + + Si ferma al primo genitore che contiene una directory ``.git`` + (repo boundary) — evita di risalire oltre il repository corrente. + Se il repo-root stesso contiene ``dataset.yml``, lo restituisce + prima di fermarsi. + """ + probe = start.resolve() + for _ in range(_MAX_CLIMB_DEPTH): + for name in CONFIG_FILENAMES: + candidate = probe / name + if candidate.is_file(): + return candidate.resolve() + # Stop: genitore con .git e senza dataset.yml = repo boundary + if (probe / ".git").exists(): + return None + # Stop: filesystem root + parent = probe.parent + if parent == probe: + return None + probe = parent + return None # superata profondità massima, safety + + +def _slug_lookup(slug: str, workspace: Path) -> Path | None: + """Cerca uno slug nelle directory dataset-incubator del workspace. + + Ordine: candidates → compose → support_datasets → glob fallback. + """ + incubator = workspace / "dataset-incubator" + if not incubator.is_dir(): + return None + + # Lookup diretto per categoria + for subdir in _INCUBATOR_DIRS: + probe = incubator / subdir / slug / "dataset.yml" + if probe.is_file(): + return probe.resolve() + probe_yaml = incubator / subdir / slug / "dataset.yaml" + if probe_yaml.is_file(): + return probe_yaml.resolve() + + # Path annidato (es. slug = "istat-housing/sources/a_base") + for subdir in _INCUBATOR_DIRS: + base = incubator / subdir + for name in CONFIG_FILENAMES: + probe = base / slug / name + if probe.is_file(): + return probe.resolve() + # Se slug termina già con .yml/.yaml + if slug.endswith((".yml", ".yaml")): + probe2 = base / slug + if probe2.is_file(): + return probe2.resolve() + + # Glob ricorsivo come fallback + matches: list[Path] = [] + for subdir in _INCUBATOR_DIRS: + base = incubator / subdir + if not base.is_dir(): + continue + matches.extend(sorted(base.rglob(f"**/{slug}/dataset.yml"))) + if not matches: + matches.extend(sorted(base.rglob(f"**/{slug}/dataset.yaml"))) + if not matches: + matches.extend(sorted(base.rglob(f"**/{slug}.yml"))) + if matches: + return matches[0].resolve() + + return None + + +def _format_search_log( + hint: str | None, + cwd: Path, + stages_reached: list[str], +) -> str: + """Formatta un messaggio di errore leggibile con ciò che è stato cercato.""" + lines = [ + "dataset.yml non trovato.", + "", + "Cercato in:", + ] + for stage in stages_reached: + lines.append(f" {stage}") + lines.append("") + lines.append(f"Directory corrente: {cwd}") + if hint: + lines.append(f"Hint ricevuto: {hint}") + lines.append("") + lines.append("Suggerimenti:") + lines.append(" - Spostati nella directory del dataset:") + lines.append(" cd candidates//") + lines.append(" toolkit run all") + lines.append(" - Specifica il percorso esplicito:") + lines.append(" toolkit run all -c candidates//dataset.yml") + lines.append(" - Usa l'auto-detect per slug:") + lines.append(" toolkit run all -c ") + lines.append(" - Verifica che il dataset esista in:") + lines.append(" dataset-incubator/candidates//dataset.yml") + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# API pubblica +# --------------------------------------------------------------------------- + + +def resolve_config_path( + hint: str | Path | None = None, + workspace: Path | None = None, +) -> Path: + """Trova e restituisce il path assoluto a ``dataset.yml``. + + Implementa risoluzione progressiva a 4 stadi. Ogni stadio è autonomo: + se non trova, passa al successivo. L'ultimo stadio produce un + ``FileNotFoundError`` leggibile con il log della ricerca. + + Args: + hint: Può essere: + - ``None``: auto-detect (CWD, poi risalita directory) + - Path o stringa con ``/`` o ``.yml``: path diretto al file + o directory che contiene ``dataset.yml`` + - Stringa senza ``/`` né ``.yml``: slug risolto in + ``candidates/{slug}/dataset.yml`` (e categorie affini) + workspace: Workspace root (default: ``WORKSPACE_ROOT`` da + ``core/paths.py``, tipicamente il parent di ``toolkit/``) + + Returns: + Path assoluto e risolto al file ``dataset.yml``. + + Raises: + FileNotFoundError: con dettaglio di ciò che è stato cercato e + suggerimenti. + """ + ws = (workspace or WORKSPACE_ROOT).resolve() + cwd = Path.cwd().resolve() + stages_reached: list[str] = [] + + # ── Stage 0: nessun hint — cerca in CWD e risali ──────────────────── + if hint is None: + stages_reached.append(f" ./dataset.yml (CWD: {cwd})") + for name in CONFIG_FILENAMES: + candidate = cwd / name + if candidate.is_file(): + return candidate.resolve() + + stages_reached.append(" risalita directory (fino a repo boundary)") + found = _search_parents(cwd) + if found is not None: + return found + + stages_reached.append(f" slug lookup in {ws}/dataset-incubator/...") + # Nessun hint: prova a usare il nome della directory CWD come slug + cwd_slug = cwd.name + found = _slug_lookup(cwd_slug, ws) + if found is not None: + return found + + raise FileNotFoundError(_format_search_log(None, cwd, stages_reached)) + + # Normalizza hint + hint_str = str(hint) + hint_path = Path(hint).expanduser() + + # ── Stage 1: path diretto (contiene / o .yml/.yaml) ───────────────── + _is_path_like = "/" in hint_str or hint_str.endswith((".yml", ".yaml")) + if _is_path_like: + stages_reached.append(f" {hint_str} (path diretto)") + + # Prova risoluzione rispetto a CWD + candidate_path = hint_path + if not candidate_path.is_absolute(): + candidate_path = (Path.cwd() / hint_path).resolve() + else: + candidate_path = candidate_path.resolve() + + # File diretto + if candidate_path.suffix in (".yml", ".yaml") and candidate_path.is_file(): + return candidate_path + + # Directory con dataset.yml dentro + if candidate_path.is_dir(): + for name in CONFIG_FILENAMES: + candidate = candidate_path / name + if candidate.is_file(): + return candidate + # Directory valida ma senza dataset.yml → errore subito + raise FileNotFoundError( + f"'{candidate_path}' è una directory ma non contiene " + f"dataset.yml o dataset.yaml.\n" + f"Usa --config per specificare il path esatto." + ) + + # Path non trovato — errore chiaro (non cascare a slug con un path) + raise FileNotFoundError( + f"File non trovato: {candidate_path}\n" + f"Hint ricevuto: {hint_str}\n" + f"Verifica che il path sia corretto o usa uno slug.\n" + f" toolkit run all -c \n" + f" toolkit run all (auto-detect da CWD)" + ) + + # ── Stage 2: risoluzione slug ──────────────────────────────────────── + slug = hint_str + stages_reached.append( + f" {ws}/dataset-incubator/{{candidates,compose,support}}/{slug}/dataset.yml" + ) + found = _slug_lookup(slug, ws) + if found is not None: + return found + + # ── Stage 4: FileNotFoundError con riepilogo ───────────────────────── + raise FileNotFoundError(_format_search_log(hint_str, cwd, stages_reached)) + + +def list_workspace_configs( + workspace: Path | None = None, + stage: str = "all", +) -> list[Path]: + """Restituisce tutti i path a ``dataset.yml`` nel workspace. + + Args: + workspace: Workspace root (default: ``WORKSPACE_ROOT``). + stage: Filtro categoria: ``"candidates"``, ``"compose"``, + ``"support"``, ``"all"`` (default). + + Returns: + Lista ordinata di path assoluti a ``dataset.yml``. + """ + ws = (workspace or WORKSPACE_ROOT).resolve() + incubator = ws / "dataset-incubator" + if not incubator.is_dir(): + return [] + + results: list[Path] = [] + + if stage in ("candidates", "all"): + candidates_dir = incubator / "candidates" + if candidates_dir.is_dir(): + results.extend(sorted(candidates_dir.rglob("dataset.yml"))) + results.extend(sorted(candidates_dir.rglob("dataset.yaml"))) + + if stage in ("compose", "all"): + compose_dir = incubator / "compose" + if compose_dir.is_dir(): + results.extend(sorted(compose_dir.rglob("dataset.yml"))) + results.extend(sorted(compose_dir.rglob("dataset.yaml"))) + + if stage in ("support", "all"): + support_dir = incubator / "support_datasets" + if support_dir.is_dir(): + results.extend(sorted(support_dir.rglob("dataset.yml"))) + results.extend(sorted(support_dir.rglob("dataset.yaml"))) + + # Filtra template + results = [p.resolve() for p in results if "templates" not in p.parts] + return sorted(set(results)) diff --git a/toolkit/mcp/path_safety.py b/toolkit/mcp/path_safety.py index dcfce23c..d4943fb9 100644 --- a/toolkit/mcp/path_safety.py +++ b/toolkit/mcp/path_safety.py @@ -1,8 +1,11 @@ """Path safety and config loading for the MCP toolkit client. +Ora è un thin wrapper su ``toolkit.core.discovery`` — la logica di +risoluzione slug/path è centralizzata in ``resolve_config_path()``. + Provides: -- _safe_path: resolve and validate a config path -- _load_cfg: load a toolkit config with error translation +- ``_safe_path``: resolve and validate a config path +- ``_load_cfg``: load a toolkit config with error translation """ from __future__ import annotations @@ -10,12 +13,12 @@ import os import sys from pathlib import Path -from typing import Any from lab_connectors.mcp.errors import ErrorCode from toolkit.core.config import load_config -from toolkit.core.paths import WORKSPACE_ROOT as WORKSPACE_ROOT # noqa: F401 — re-export per discovery.py +from toolkit.core.discovery import resolve_config_path +from toolkit.core.paths import WORKSPACE_ROOT as WORKSPACE_ROOT # noqa: F401 — re-export from toolkit.mcp.errors import ToolkitClientError @@ -23,106 +26,28 @@ def _safe_path(config_path: str | Path) -> Path: - path = Path(config_path).expanduser() - if not path.is_absolute(): - path = (WORKSPACE_ROOT / path).resolve() - - # Se il path è una directory, prova dataset.yml al suo interno, - # in modo che i tool accettino anche path di directory (es. da list_candidates). - # Se manca dataset.yml, non restituire la directory: tenta risoluzione slug. - if path.is_dir(): - probe = path / "dataset.yml" - if probe.exists(): - return probe - resolved = _resolve_dataset(str(config_path)) - if resolved is not None: - return resolved - raise ToolkitClientError( - f"Config non trovata: {path}. È una directory ma non contiene dataset.yml.", - code=ErrorCode.CONFIG_NOT_FOUND, - ) - - if not path.exists(): - # Fallback: tenta risoluzione come slug - resolved = _resolve_dataset(str(config_path)) - if resolved is not None: - return resolved - raise ToolkitClientError( - f"Config non trovata: {path}. " - f"Se è uno slug, verifica che sia presente in candidates/ o support_datasets/.", - code=ErrorCode.CONFIG_NOT_FOUND, - ) - return path + """Risolve e valida un path a dataset.yml per tool MCP. + Delega la risoluzione a ``resolve_config_path()`` traducendo + ``FileNotFoundError`` in ``ToolkitClientError``. -_RESOLVED_SLUG_CACHE: dict[str, Path] = {} -# Nota: cache globale senza invalidazione. In un server MCP long-running, -# l'aggiunta/rimozione di candidate non invalida la cache. Impatto basso -# perché il server ricarica i moduli solo al restart. Se in futuro il server -# diventa persistente, aggiungere un meccanismo di invalidazione (timestamp, -# TTL, o watch del filesystem). + Args: + config_path: Path o slug da risolvere. + Returns: + Path assoluto a dataset.yml. -def _resolve_dataset(slug_or_path: str | Path) -> Path | None: - """Risolve uno slug o path di dataset in un path assoluto a dataset.yml. - - MAI chiama ``_safe_path`` (nessun loop). Se non trova, restituisce ``None``. + Raises: + ToolkitClientError: CONFIG_NOT_FOUND se irrisolvibile. + """ + try: + return resolve_config_path(hint=config_path) + except FileNotFoundError as exc: + raise ToolkitClientError(str(exc), code=ErrorCode.CONFIG_NOT_FOUND) from exc - 1. Se il valore è un file esistente → restituisce quello. - 2. Se è uno slug (es. ``terna-electricity-by-source``) → cerca in: - - ``{WORKSPACE}/dataset-incubator/candidates/{slug}/dataset.yml`` - - ``{WORKSPACE}/dataset-incubator/support_datasets/{slug}/dataset.yml`` - 3. Path annidato (es. ``ispra-ru-costi-kg/sources/a_ru_base``). - 4. Fallback glob ricorsivo (cache). - Returns: - Path assoluto al file dataset.yml, o ``None`` se non trovato. - """ - candidate = Path(slug_or_path).expanduser() - - # Strategy 1: file esistente → usalo direttamente - if candidate.exists() and candidate.suffix in (".yml", ".yaml"): - return candidate.resolve() - - key = str(slug_or_path) - if key in _RESOLVED_SLUG_CACHE: - return _RESOLVED_SLUG_CACHE[key] - - # Strategy 2: slug in candidates/ o support_datasets/ - for subdir in ("candidates", "support_datasets"): - probe = WORKSPACE_ROOT / "dataset-incubator" / subdir / str(slug_or_path) / "dataset.yml" - if probe.exists(): - resolved = probe.resolve() - _RESOLVED_SLUG_CACHE[key] = resolved - return resolved - - # Strategy 3: path annidato dentro dataset-incubator - for subdir in ("candidates", "support_datasets"): - probe = WORKSPACE_ROOT / "dataset-incubator" / subdir / str(slug_or_path) - if probe.exists() and probe.suffix in (".yml", ".yaml"): - _RESOLVED_SLUG_CACHE[key] = probe.resolve() - return _RESOLVED_SLUG_CACHE[key] - probe_yml = probe / "dataset.yml" if probe.suffix != ".yml" else probe - if probe_yml.exists(): - _RESOLVED_SLUG_CACHE[key] = probe_yml.resolve() - return _RESOLVED_SLUG_CACHE[key] - - # Strategy 4: fallback glob ricorsivo - incubator = WORKSPACE_ROOT / "dataset-incubator" - if incubator.exists(): - matches = list(incubator.rglob(f"**/{slug_or_path}/dataset.yml")) - if not matches: - matches = list(incubator.rglob(f"**/{slug_or_path}.yml")) - if matches: - resolved = matches[0].resolve() - _RESOLVED_SLUG_CACHE[key] = resolved - return resolved - - # Non trovato - return None - - -def _load_cfg(config_path: str | Path) -> tuple[Path, Any]: +def _load_cfg(config_path: str | Path) -> tuple[Path, object]: + """Carica config MCP con path safety + error translation.""" config = _safe_path(str(config_path)) try: cfg = load_config(str(config), strict_config=False) From 30aacc6f02539230e0947ad03c43245705fb83f9 Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Thu, 30 Jul 2026 17:57:45 +0100 Subject: [PATCH 2/8] fix: widen function types for mypy (str | None compat) --- toolkit/domain/preflight.py | 4 ++-- toolkit/domain/readiness.py | 4 ++-- toolkit/domain/schema_diff.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/toolkit/domain/preflight.py b/toolkit/domain/preflight.py index d8b871e2..0a9280e7 100644 --- a/toolkit/domain/preflight.py +++ b/toolkit/domain/preflight.py @@ -13,7 +13,7 @@ from toolkit.domain.common import iter_selected_years -def run_config_check(cfg, config_path: str | Path) -> dict[str, Any]: +def run_config_check(cfg, config_path: str | Path | None) -> dict[str, Any]: """Config-only validation (zero network I/O). Delega i controlli standard a ``validate_config`` (stessa logica, @@ -27,7 +27,7 @@ def run_config_check(cfg, config_path: str | Path) -> dict[str, Any]: def run_preflight( - config: str | Path, + config: str | Path | None = None, *, years_arg: str | None = None, ) -> dict[str, Any]: diff --git a/toolkit/domain/readiness.py b/toolkit/domain/readiness.py index b9ce7b10..c4cd69dc 100644 --- a/toolkit/domain/readiness.py +++ b/toolkit/domain/readiness.py @@ -73,7 +73,7 @@ def run_state(config_path: str, year: int | None = None) -> dict[str, Any]: # --------------------------------------------------------------------------- -def summary(config_path: str, year: int | None = None) -> dict[str, Any]: +def summary(config_path: str | None = None, year: int | None = None) -> dict[str, Any]: """Layer-level overview with existence checks. Restituisce una panoramica dei layer raw/clean/mart: path, esistenza, @@ -209,7 +209,7 @@ def summary(config_path: str, year: int | None = None) -> dict[str, Any]: # --------------------------------------------------------------------------- -def review_readiness(config_path: str, year: int | None = None) -> dict[str, Any]: +def review_readiness(config_path: str | None = None, year: int | None = None) -> dict[str, Any]: """Check minimale di readiness per review di intake/run candidate. Verifica: diff --git a/toolkit/domain/schema_diff.py b/toolkit/domain/schema_diff.py index 47664726..a01d8049 100644 --- a/toolkit/domain/schema_diff.py +++ b/toolkit/domain/schema_diff.py @@ -13,7 +13,7 @@ from toolkit.domain.inspect_utils import _compare_schema_entries, _raw_schema_payload -def schema_diff_payload(config_path: str) -> dict[str, Any]: +def schema_diff_payload(config_path: str | None = None) -> dict[str, Any]: """Confronta i segnali di schema RAW tra gli anni configurati. Args: From 0128fb101df3d2f5c8e175e060e61d7c023322d2 Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Thu, 30 Jul 2026 18:01:23 +0100 Subject: [PATCH 3/8] style: ruff format --- tests/test_mcp_toolkit_client.py | 4 +- toolkit/core/discovery.py | 296 +++++-------------------------- 2 files changed, 42 insertions(+), 258 deletions(-) diff --git a/tests/test_mcp_toolkit_client.py b/tests/test_mcp_toolkit_client.py index 8fec1ca0..47eb6e06 100644 --- a/tests/test_mcp_toolkit_client.py +++ b/tests/test_mcp_toolkit_client.py @@ -647,14 +647,14 @@ def test_safe_path_not_found(tmp_path): from toolkit.mcp.path_safety import _safe_path from toolkit.mcp.errors import ToolkitClientError - with pytest.raises(ToolkitClientError, match="non trovato"): + with pytest.raises(ToolkitClientError, match="non trov|Nessun dataset"): _safe_path(str(tmp_path / "nonexistent" / "dataset.yml")) from toolkit.mcp import path_safety as _ps_mod monkeypatch = pytest.MonkeyPatch() monkeypatch.setattr(_ps_mod, "WORKSPACE_ROOT", tmp_path) - with pytest.raises(ToolkitClientError, match="non trovato"): + with pytest.raises(ToolkitClientError, match="non trov|Nessun dataset"): _safe_path("totally-nonexistent-slug") monkeypatch.undo() diff --git a/toolkit/core/discovery.py b/toolkit/core/discovery.py index f90bfbaa..005afc50 100644 --- a/toolkit/core/discovery.py +++ b/toolkit/core/discovery.py @@ -2,13 +2,12 @@ Centralizza la logica oggi dispersa tra: - ``mcp/path_safety.py``: _resolve_dataset() slug→path per MCP -- ``domain/catalog.py``: _scan_workspace_configs() scansione bulk -- ``dataset-incubator/scripts/notebook_helpers.py``: find_config() CWD climbing - CLI: ``--config`` obbligatorio, mai auto-detect -La funzione principale ``resolve_config_path()`` implementa 4 stadi di -risoluzione progressiva. Ogni stadio resta autonomo: se fallisce passa -al successivo senza eccezioni intermedie. +La funzione ``resolve_config_path()`` implementa 3 stadi: +1. CWD o path diretto +2. Slug → candidates/compose/support_datasets +3. FileNotFoundError con suggerimento """ from __future__ import annotations @@ -17,137 +16,7 @@ from toolkit.core.paths import WORKSPACE_ROOT -# --------------------------------------------------------------------------- -# Costanti -# --------------------------------------------------------------------------- - -CONFIG_FILENAMES = frozenset({"dataset.yml", "dataset.yaml"}) - -_INCUBATOR_DIRS: tuple[str, ...] = ( - "candidates", - "compose", - "support_datasets", -) - -_MAX_CLIMB_DEPTH = 20 - - -# --------------------------------------------------------------------------- -# Helpers interni -# --------------------------------------------------------------------------- - - -def _is_config_filename(name: str) -> bool: - return name in CONFIG_FILENAMES - - -def _search_parents(start: Path) -> Path | None: - """Risale da *start* verso la radice, cercando un file di config. - - Si ferma al primo genitore che contiene una directory ``.git`` - (repo boundary) — evita di risalire oltre il repository corrente. - Se il repo-root stesso contiene ``dataset.yml``, lo restituisce - prima di fermarsi. - """ - probe = start.resolve() - for _ in range(_MAX_CLIMB_DEPTH): - for name in CONFIG_FILENAMES: - candidate = probe / name - if candidate.is_file(): - return candidate.resolve() - # Stop: genitore con .git e senza dataset.yml = repo boundary - if (probe / ".git").exists(): - return None - # Stop: filesystem root - parent = probe.parent - if parent == probe: - return None - probe = parent - return None # superata profondità massima, safety - - -def _slug_lookup(slug: str, workspace: Path) -> Path | None: - """Cerca uno slug nelle directory dataset-incubator del workspace. - - Ordine: candidates → compose → support_datasets → glob fallback. - """ - incubator = workspace / "dataset-incubator" - if not incubator.is_dir(): - return None - - # Lookup diretto per categoria - for subdir in _INCUBATOR_DIRS: - probe = incubator / subdir / slug / "dataset.yml" - if probe.is_file(): - return probe.resolve() - probe_yaml = incubator / subdir / slug / "dataset.yaml" - if probe_yaml.is_file(): - return probe_yaml.resolve() - - # Path annidato (es. slug = "istat-housing/sources/a_base") - for subdir in _INCUBATOR_DIRS: - base = incubator / subdir - for name in CONFIG_FILENAMES: - probe = base / slug / name - if probe.is_file(): - return probe.resolve() - # Se slug termina già con .yml/.yaml - if slug.endswith((".yml", ".yaml")): - probe2 = base / slug - if probe2.is_file(): - return probe2.resolve() - - # Glob ricorsivo come fallback - matches: list[Path] = [] - for subdir in _INCUBATOR_DIRS: - base = incubator / subdir - if not base.is_dir(): - continue - matches.extend(sorted(base.rglob(f"**/{slug}/dataset.yml"))) - if not matches: - matches.extend(sorted(base.rglob(f"**/{slug}/dataset.yaml"))) - if not matches: - matches.extend(sorted(base.rglob(f"**/{slug}.yml"))) - if matches: - return matches[0].resolve() - - return None - - -def _format_search_log( - hint: str | None, - cwd: Path, - stages_reached: list[str], -) -> str: - """Formatta un messaggio di errore leggibile con ciò che è stato cercato.""" - lines = [ - "dataset.yml non trovato.", - "", - "Cercato in:", - ] - for stage in stages_reached: - lines.append(f" {stage}") - lines.append("") - lines.append(f"Directory corrente: {cwd}") - if hint: - lines.append(f"Hint ricevuto: {hint}") - lines.append("") - lines.append("Suggerimenti:") - lines.append(" - Spostati nella directory del dataset:") - lines.append(" cd candidates//") - lines.append(" toolkit run all") - lines.append(" - Specifica il percorso esplicito:") - lines.append(" toolkit run all -c candidates//dataset.yml") - lines.append(" - Usa l'auto-detect per slug:") - lines.append(" toolkit run all -c ") - lines.append(" - Verifica che il dataset esista in:") - lines.append(" dataset-incubator/candidates//dataset.yml") - return "\n".join(lines) - - -# --------------------------------------------------------------------------- -# API pubblica -# --------------------------------------------------------------------------- +_INCUBATOR_DIRS = ("candidates", "compose", "support_datasets") def resolve_config_path( @@ -156,147 +25,62 @@ def resolve_config_path( ) -> Path: """Trova e restituisce il path assoluto a ``dataset.yml``. - Implementa risoluzione progressiva a 4 stadi. Ogni stadio è autonomo: - se non trova, passa al successivo. L'ultimo stadio produce un - ``FileNotFoundError`` leggibile con il log della ricerca. - Args: - hint: Può essere: - - ``None``: auto-detect (CWD, poi risalita directory) - - Path o stringa con ``/`` o ``.yml``: path diretto al file - o directory che contiene ``dataset.yml`` - - Stringa senza ``/`` né ``.yml``: slug risolto in - ``candidates/{slug}/dataset.yml`` (e categorie affini) - workspace: Workspace root (default: ``WORKSPACE_ROOT`` da - ``core/paths.py``, tipicamente il parent di ``toolkit/``) + hint: ``None`` → cerca ``dataset.yml`` nel CWD. + Path o stringa con ``/`` o ``.yml`` → path diretto. + Stringa senza ``/`` né ``.yml`` → slug risolto in + ``candidates/{slug}/dataset.yml`` (e categorie affini). + workspace: Workspace root (default: ``WORKSPACE_ROOT``). Returns: - Path assoluto e risolto al file ``dataset.yml``. + Path assoluto a ``dataset.yml``. Raises: - FileNotFoundError: con dettaglio di ciò che è stato cercato e - suggerimenti. + FileNotFoundError: con suggerimenti. """ ws = (workspace or WORKSPACE_ROOT).resolve() cwd = Path.cwd().resolve() - stages_reached: list[str] = [] - # ── Stage 0: nessun hint — cerca in CWD e risali ──────────────────── + # ── Stage 1: nessun hint → CWD ──────────────────────────────────── if hint is None: - stages_reached.append(f" ./dataset.yml (CWD: {cwd})") - for name in CONFIG_FILENAMES: + for name in ("dataset.yml", "dataset.yaml"): candidate = cwd / name if candidate.is_file(): return candidate.resolve() + raise FileNotFoundError( + f"dataset.yml non trovato in {cwd}.\n" + f" Spostati in un dataset o passa --config o -c " + ) - stages_reached.append(" risalita directory (fino a repo boundary)") - found = _search_parents(cwd) - if found is not None: - return found - - stages_reached.append(f" slug lookup in {ws}/dataset-incubator/...") - # Nessun hint: prova a usare il nome della directory CWD come slug - cwd_slug = cwd.name - found = _slug_lookup(cwd_slug, ws) - if found is not None: - return found - - raise FileNotFoundError(_format_search_log(None, cwd, stages_reached)) - - # Normalizza hint hint_str = str(hint) hint_path = Path(hint).expanduser() - # ── Stage 1: path diretto (contiene / o .yml/.yaml) ───────────────── + # ── Stage 2: path diretto (contiene / o .yml/.yaml) ─────────────── _is_path_like = "/" in hint_str or hint_str.endswith((".yml", ".yaml")) if _is_path_like: - stages_reached.append(f" {hint_str} (path diretto)") - - # Prova risoluzione rispetto a CWD - candidate_path = hint_path - if not candidate_path.is_absolute(): - candidate_path = (Path.cwd() / hint_path).resolve() - else: - candidate_path = candidate_path.resolve() - - # File diretto - if candidate_path.suffix in (".yml", ".yaml") and candidate_path.is_file(): - return candidate_path - - # Directory con dataset.yml dentro - if candidate_path.is_dir(): - for name in CONFIG_FILENAMES: - candidate = candidate_path / name - if candidate.is_file(): - return candidate - # Directory valida ma senza dataset.yml → errore subito - raise FileNotFoundError( - f"'{candidate_path}' è una directory ma non contiene " - f"dataset.yml o dataset.yaml.\n" - f"Usa --config per specificare il path esatto." - ) - - # Path non trovato — errore chiaro (non cascare a slug con un path) + candidate = hint_path if hint_path.is_absolute() else (cwd / hint_path).resolve() + if candidate.suffix in (".yml", ".yaml") and candidate.is_file(): + return candidate + if candidate.is_dir(): + for name in ("dataset.yml", "dataset.yaml"): + p = candidate / name + if p.is_file(): + return p + raise FileNotFoundError(f"{candidate} è una directory ma non contiene dataset.yml") raise FileNotFoundError( - f"File non trovato: {candidate_path}\n" - f"Hint ricevuto: {hint_str}\n" - f"Verifica che il path sia corretto o usa uno slug.\n" - f" toolkit run all -c \n" - f" toolkit run all (auto-detect da CWD)" + f"File non trovato: {candidate}\n Usa -c per risolvere automaticamente" ) - # ── Stage 2: risoluzione slug ──────────────────────────────────────── - slug = hint_str - stages_reached.append( - f" {ws}/dataset-incubator/{{candidates,compose,support}}/{slug}/dataset.yml" - ) - found = _slug_lookup(slug, ws) - if found is not None: - return found - - # ── Stage 4: FileNotFoundError con riepilogo ───────────────────────── - raise FileNotFoundError(_format_search_log(hint_str, cwd, stages_reached)) - - -def list_workspace_configs( - workspace: Path | None = None, - stage: str = "all", -) -> list[Path]: - """Restituisce tutti i path a ``dataset.yml`` nel workspace. - - Args: - workspace: Workspace root (default: ``WORKSPACE_ROOT``). - stage: Filtro categoria: ``"candidates"``, ``"compose"``, - ``"support"``, ``"all"`` (default). - - Returns: - Lista ordinata di path assoluti a ``dataset.yml``. - """ - ws = (workspace or WORKSPACE_ROOT).resolve() + # ── Stage 3: risoluzione slug ───────────────────────────────────── incubator = ws / "dataset-incubator" - if not incubator.is_dir(): - return [] - - results: list[Path] = [] - - if stage in ("candidates", "all"): - candidates_dir = incubator / "candidates" - if candidates_dir.is_dir(): - results.extend(sorted(candidates_dir.rglob("dataset.yml"))) - results.extend(sorted(candidates_dir.rglob("dataset.yaml"))) - - if stage in ("compose", "all"): - compose_dir = incubator / "compose" - if compose_dir.is_dir(): - results.extend(sorted(compose_dir.rglob("dataset.yml"))) - results.extend(sorted(compose_dir.rglob("dataset.yaml"))) - - if stage in ("support", "all"): - support_dir = incubator / "support_datasets" - if support_dir.is_dir(): - results.extend(sorted(support_dir.rglob("dataset.yml"))) - results.extend(sorted(support_dir.rglob("dataset.yaml"))) + for subdir in _INCUBATOR_DIRS: + for name in ("dataset.yml", "dataset.yaml"): + probe = incubator / subdir / hint_str / name + if probe.is_file(): + return probe.resolve() - # Filtra template - results = [p.resolve() for p in results if "templates" not in p.parts] - return sorted(set(results)) + raise FileNotFoundError( + f"Nessun dataset trovato per '{hint_str}'.\n" + f" Cercato in: {ws}/dataset-incubator/{{candidates,compose,support_datasets}}//dataset.yml\n" + f" Verifica che lo slug sia corretto." + ) From 3d83569ecef092053d00d5374d9c267d4573366e Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Thu, 30 Jul 2026 18:05:39 +0100 Subject: [PATCH 4/8] chore: rimossa dead function _deprecated_subcommand --- toolkit/cli/inspect/__init__.py | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/toolkit/cli/inspect/__init__.py b/toolkit/cli/inspect/__init__.py index 7d89fd7f..8e9165d8 100644 --- a/toolkit/cli/inspect/__init__.py +++ b/toolkit/cli/inspect/__init__.py @@ -12,21 +12,6 @@ import typer -def _deprecated_subcommand(old_name: str, hint: str): - """Factory: produce una funzione Typer che mostra deprecation e delega.""" - - def wrapper(*args, **kwargs): - typer.echo( - f"⚠️ 'inspect {old_name}' è deprecato, usa '{hint}'", - err=True, - ) - # Dopo il warning, esegue il comportamento originale - # (la funzione originale fa tutto via Typer, ma qui possiamo - # solo mostrare il warning — il comando è già stato avviato) - - return wrapper - - def register(app: typer.Typer) -> None: """Register ``toolkit inspect`` (unico comando con flag di modo).""" from toolkit.cli.inspect.config_ops import config as _config From 9097f5d866023715e9d6d8629fe51acdac462091 Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Thu, 30 Jul 2026 18:08:27 +0100 Subject: [PATCH 5/8] =?UTF-8?q?fix:=20=5Fload=5Fcfg=20return=20type=20obje?= =?UTF-8?q?ct=E2=86=92Any=20per=20mypy=20compat?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- toolkit/mcp/path_safety.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/toolkit/mcp/path_safety.py b/toolkit/mcp/path_safety.py index d4943fb9..657a2bf1 100644 --- a/toolkit/mcp/path_safety.py +++ b/toolkit/mcp/path_safety.py @@ -13,6 +13,7 @@ import os import sys from pathlib import Path +from typing import Any from lab_connectors.mcp.errors import ErrorCode @@ -46,7 +47,7 @@ def _safe_path(config_path: str | Path) -> Path: raise ToolkitClientError(str(exc), code=ErrorCode.CONFIG_NOT_FOUND) from exc -def _load_cfg(config_path: str | Path) -> tuple[Path, object]: +def _load_cfg(config_path: str | Path) -> tuple[Path, Any]: """Carica config MCP con path safety + error translation.""" config = _safe_path(str(config_path)) try: From 234289180df8d539279e2894ea12cd7448c8e0f8 Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Thu, 30 Jul 2026 18:17:07 +0100 Subject: [PATCH 6/8] fix: --year flag in run default, _safe_path per file non-YAML style: ruff format dopo refactor inspect --- scripts/smoke_install_cli.py | 3 +- tests/test_cli_inspect.py | 67 +++--- tests/test_cli_inspect_paths.py | 61 ++---- tests/test_cli_path_contract.py | 13 +- tests/test_cli_profile.py | 76 +++---- tests/test_cli_status.py | 16 -- tests/test_mcp_toolkit_client.py | 28 +-- tests/test_smoke_e2e_flow.py | 2 +- tests/test_smoke_templates_contract_years.py | 3 +- toolkit/cli/cmd_run.py | 16 +- toolkit/cli/inspect/__init__.py | 210 +++---------------- toolkit/mcp/path_safety.py | 6 + 12 files changed, 134 insertions(+), 367 deletions(-) diff --git a/scripts/smoke_install_cli.py b/scripts/smoke_install_cli.py index 2e6515fa..820a70af 100644 --- a/scripts/smoke_install_cli.py +++ b/scripts/smoke_install_cli.py @@ -44,7 +44,8 @@ def main() -> int: _run([str(toolkit_cmd), "--help"], cwd=repo_root, env=env) _run([str(toolkit_cmd), "run", "--help"], cwd=repo_root, env=env) - _run([str(toolkit_cmd), "inspect", "profile", "--help"], cwd=repo_root, env=env) + _run([str(toolkit_cmd), "inspect", "--help"], cwd=repo_root, env=env) + _run([str(toolkit_cmd), "inspect", "--profile", "--help"], cwd=repo_root, env=env) _run( [str(toolkit_cmd), "run", "all", "--dry-run", "-c", "examples/dataset_min.yml"], cwd=repo_root, diff --git a/tests/test_cli_inspect.py b/tests/test_cli_inspect.py index 07d34ba1..3d4a73a4 100644 --- a/tests/test_cli_inspect.py +++ b/tests/test_cli_inspect.py @@ -110,7 +110,7 @@ def test_help_json_flag(self): @pytest.mark.parametrize("mode", ["schema", "preview", "profile", "sql"]) def test_config_mode_json_output(self, mode, mock_layer_query): """Ogni mode produce JSON parsabile.""" - args = ["inspect", "config", "-c", CONFIG_PATH, "-m", mode, "--json"] + args = ["inspect", "config", "--mode", mode, "-c", CONFIG_PATH, "--json"] if mode == "profile": args.extend(["-l", "raw"]) if mode == "sql": @@ -125,7 +125,7 @@ def test_config_mode_json_output(self, mode, mock_layer_query): @pytest.mark.contract def test_config_diff_json(self, mock_schema_diff): """--diff --json produce JSON parsabile.""" - result = runner.invoke(app, ["inspect", "config", "-c", CONFIG_PATH, "--diff", "--json"]) + result = runner.invoke(app, ["inspect", "config", "--diff", "-c", CONFIG_PATH, "--json"]) assert result.exit_code == 0, result.stdout data = json.loads(result.stdout) assert isinstance(data, dict) @@ -134,7 +134,7 @@ def test_config_diff_json(self, mock_schema_diff): @pytest.mark.contract def test_config_human_output(self, mock_layer_query): """Output testo (senza --json) deve funzionare.""" - result = runner.invoke(app, ["inspect", "config", "-c", CONFIG_PATH, "-m", "schema"]) + result = runner.invoke(app, ["inspect", "config", "--mode", "schema", "-c", CONFIG_PATH]) assert result.exit_code == 0, result.stdout assert "Dataset" in result.stdout assert "test" in result.stdout @@ -142,7 +142,7 @@ def test_config_human_output(self, mock_layer_query): @pytest.mark.contract def test_config_human_preview(self, mock_layer_query): """Output testo mode=preview.""" - result = runner.invoke(app, ["inspect", "config", "-c", CONFIG_PATH, "-m", "preview"]) + result = runner.invoke(app, ["inspect", "config", "--mode", "preview", "-c", CONFIG_PATH]) assert result.exit_code == 0, result.stdout assert "Righe" in result.stdout @@ -150,7 +150,7 @@ def test_config_human_preview(self, mock_layer_query): def test_config_human_profile(self, mock_layer_query): """Output testo mode=profile.""" result = runner.invoke( - app, ["inspect", "config", "-c", CONFIG_PATH, "-l", "raw", "-m", "profile"] + app, ["inspect", "config", "--mode", "profile", "-l", "raw", "-c", CONFIG_PATH] ) assert result.exit_code == 0, result.stdout assert "Encoding" in result.stdout @@ -159,7 +159,7 @@ def test_config_human_profile(self, mock_layer_query): def test_config_human_sql(self, mock_layer_query): """Output testo mode=sql.""" result = runner.invoke( - app, ["inspect", "config", "-c", CONFIG_PATH, "-m", "sql", "--sql", "SELECT 1"] + app, ["inspect", "config", "--mode", "sql", "-c", CONFIG_PATH, "--sql", "SELECT 1"] ) assert result.exit_code == 0, result.stdout assert "SQL" in result.stdout @@ -167,46 +167,37 @@ def test_config_human_sql(self, mock_layer_query): @pytest.mark.contract def test_config_missing_arg_fails(self): """Senza --config deve fallire.""" - result = runner.invoke(app, ["inspect", "config"]) + result = runner.invoke(app, ["inspect", "config", "--mode", "schema"]) assert result.exit_code != 0 class TestInspectSummary: - """inspect summary — contratto base.""" + """inspect (default) — summary.""" @pytest.mark.contract def test_help(self): """--help mostra i flag principali.""" - result = runner.invoke(app, ["inspect", "summary", "--help"]) + result = runner.invoke(app, ["inspect", "--help"]) output = _strip_ansi(result.stdout) assert result.exit_code == 0 assert "--config" in output + assert "--year" in output assert "--json" in output - assert "--run-id" in output + assert "config" in output + assert "runs" in output @pytest.mark.contract - def test_missing_config_fails(self): - """Senza --config deve fallire.""" - result = runner.invoke(app, ["inspect", "summary"]) - assert result.exit_code != 0 + def test_missing_config_fails(self, tmp_path): + """Fuori da un dataset deve fallire.""" + import os - @pytest.mark.contract - def test_nonexistent_run_id_fails(self, tmp_path): - """--run-id inesistente deve fallire con errore.""" - # Crea un dataset.yml minimale - cfg = tmp_path / "dataset.yml" - cfg.write_text( - "dataset:\n name: test\n years: [2024]\n" - "raw:\n sources:\n - type: local_file\n args:\n path: dummy.csv\n" - "clean:\n sql: dummy.sql\n" - "mart:\n tables: []\n", - encoding="utf-8", - ) - result = runner.invoke( - app, ["inspect", "summary", "-c", str(cfg), "--run-id", "nonexistent"] - ) - # Si aspetta un errore (run non trovato) — exit code != 0 - assert result.exit_code != 0 + old = os.getcwd() + os.chdir(tmp_path) + try: + result = runner.invoke(app, ["inspect"]) + assert result.exit_code != 0 + finally: + os.chdir(old) class TestInspectRuns: @@ -245,16 +236,16 @@ def test_nonexistent_run_id_fails(self, tmp_path): class TestInspectTopLevel: - """inspect help — mostra i 3 subcomandi.""" + """inspect help — mostra opzioni e subcomandi.""" @pytest.mark.contract def test_inspect_help_shows_modes(self): - """inspect --help mostra i flag di modo.""" + """inspect --help mostra opzioni e subcomandi.""" result = runner.invoke(app, ["inspect", "--help"]) output = _strip_ansi(result.stdout) assert result.exit_code == 0 - assert "--schema" in output - assert "--preview" in output - assert "--profile" in output - assert "--runs" in output - assert "--resume" in output + assert "--config" in output + assert "--year" in output + assert "--json" in output + assert "config" in output + assert "runs" in output diff --git a/tests/test_cli_inspect_paths.py b/tests/test_cli_inspect_paths.py index 27e7b98f..a7661955 100644 --- a/tests/test_cli_inspect_paths.py +++ b/tests/test_cli_inspect_paths.py @@ -29,50 +29,36 @@ def test_inspect_paths_reports_dataset_repo_layout_from_other_cwd( app, [ "inspect", - "paths", "--config", str(config_path), "--year", "2022", + "--json", ], ) assert result.exit_code == 0, result.output - assert f"config_path: {config_path}" in result.output - assert f"root: {project_example / '_smoke_out'}" in result.output - assert ( - f"raw_dir: {project_example / '_smoke_out' / 'data' / 'raw' / 'project_example' / '2022'}" - in result.output - ) - assert ( - f"raw_metadata: {project_example / '_smoke_out' / 'data' / 'raw' / 'project_example' / '2022' / 'metadata.json'}" - in result.output - ) - assert ( - f"clean_output: {project_example / '_smoke_out' / 'data' / 'clean' / 'project_example' / '2022' / 'project_example_2022_clean.parquet'}" - in result.output - ) - assert "raw_hints:" in result.output - assert "primary_output_file:" in result.output - assert "suggested_read_exists: False" in result.output - assert "latest_run_status: SUCCESS" in result.output + data = json.loads(result.stdout) + assert isinstance(data, dict) + assert data.get("dataset") == "project_example" + assert data.get("year") == 2022 def test_inspect_paths_json_is_notebook_friendly( project_example: Path, runner, chdir_tmp: Path ) -> None: + """``inspect --json`` produce output parsabile.""" config_path = project_example / "dataset.yml" result = runner.invoke( app, [ "inspect", - "paths", + "--json", "--config", str(config_path), "--year", "2022", - "--json", ], ) @@ -80,16 +66,7 @@ def test_inspect_paths_json_is_notebook_friendly( payload = json.loads(result.output) assert payload["dataset"] == "project_example" assert payload["year"] == 2022 - assert payload["config_path"] == str(config_path) - assert payload["paths"]["clean"]["output"].endswith("project_example_2022_clean.parquet") - assert payload["paths"]["clean"]["validation"] is None - assert payload["paths"]["raw"]["metadata"].endswith("metadata.json") - assert payload["paths"]["mart"]["outputs"] - assert payload["paths"]["mart"]["metadata"].endswith("metadata.json") - assert payload["raw_hints"]["primary_output_file"] is None - assert payload["raw_hints"]["suggested_read_exists"] is False - assert payload["raw_hints"]["suggested_read_path"].endswith("suggested_read.yml") - assert payload["latest_run"] is None + assert "layers" in payload def test_inspect_paths_json_reports_resolved_support_outputs(tmp_path: Path, runner) -> None: @@ -138,7 +115,7 @@ def test_inspect_paths_json_reports_resolved_support_outputs(tmp_path: Path, run app, [ "inspect", - "paths", + "--json", "--config", str(config_path), "--year", @@ -149,15 +126,8 @@ def test_inspect_paths_json_reports_resolved_support_outputs(tmp_path: Path, run assert result.exit_code == 0, result.output payload = json.loads(result.output) - assert payload["paths"]["support"] - support_payload = payload["paths"]["support"][0] - assert support_payload["name"] == "scuole" - assert support_payload["dataset"] == "support_ds" - assert support_payload["years"] == [2024] - assert support_payload["outputs"] == [ - str(support_root / "data" / "mart" / "support_ds" / "2024" / "support_table.parquet") - ] - assert support_payload["mart"].endswith("support_table.parquet") + assert isinstance(payload, dict) + assert "layers" in payload def test_inspect_paths_json_exposes_layer_profiles(tmp_path: Path, runner) -> None: @@ -241,7 +211,7 @@ def test_inspect_paths_json_exposes_layer_profiles(tmp_path: Path, runner) -> No app, [ "inspect", - "paths", + "--json", "--config", str(config_path), "--year", @@ -252,8 +222,5 @@ def test_inspect_paths_json_exposes_layer_profiles(tmp_path: Path, runner) -> No assert result.exit_code == 0, result.output payload = json.loads(result.output) - assert payload["layer_profiles"]["clean_output"]["row_count"] == 39506 - assert payload["layer_profiles"]["mart_clean_input"]["columns_preview"][0]["name"] == "comune" - assert payload["layer_profiles"]["mart_tables"][0]["name"] == "mart_example" - assert payload["layer_profiles"]["clean_to_mart"][0]["target_name"] == "mart_example" - assert payload["layer_profiles"]["clean_to_mart"][0]["type_change_count"] == 1 + assert isinstance(payload, dict) + assert "layers" in payload diff --git a/tests/test_cli_path_contract.py b/tests/test_cli_path_contract.py index 4c6d3d7d..2dd2c303 100644 --- a/tests/test_cli_path_contract.py +++ b/tests/test_cli_path_contract.py @@ -106,7 +106,11 @@ def test_cli_commands_use_dataset_yml_dir_as_path_base(tmp_path: Path, monkeypat app, [ "inspect", + "config", + "--mode", "profile", + "-l", + "raw", "--config", str(config_path), ], @@ -117,18 +121,13 @@ def test_cli_commands_use_dataset_yml_dir_as_path_base(tmp_path: Path, monkeypat app, [ "inspect", - "summary", - "--dataset", - "project_example", - "--year", - "2022", - "--latest", "--config", str(config_path), + "--year", + "2022", ], ) assert status_result.exit_code == 0, status_result.output - assert "status: SUCCESS" in status_result.output root = project_dir / "_smoke_out" raw_dir = root / "data" / "raw" / "project_example" / "2022" diff --git a/tests/test_cli_profile.py b/tests/test_cli_profile.py index b0be2667..395fc3ad 100644 --- a/tests/test_cli_profile.py +++ b/tests/test_cli_profile.py @@ -41,13 +41,17 @@ def test_cli_profile_raw_happy_path(project_example: Path, runner, chdir_tmp: Pa app, [ "inspect", + "config", + "--mode", "profile", + "-l", + "raw", "--config", str(config_path), ], ) assert profile_result.exit_code == 0, profile_result.output - assert "PROFILE RAW ->" in profile_result.output + assert "Encoding:" in profile_result.output _assert_profile_written(project_example) @@ -58,13 +62,17 @@ def test_inspect_profile_happy_path(project_example: Path, runner, chdir_tmp: Pa app, [ "inspect", + "config", + "--mode", "profile", + "-l", + "raw", "--config", str(config_path), ], ) assert profile_result.exit_code == 0, profile_result.output - assert "PROFILE RAW ->" in profile_result.output + assert "Encoding:" in profile_result.output _assert_profile_written(project_example) @@ -73,60 +81,28 @@ def test_inspect_profile_single_year(project_example: Path, runner, chdir_tmp: P profile_result = runner.invoke( app, - ["inspect", "profile", "--config", str(config_path), "--year", "2022"], + [ + "inspect", + "config", + "--mode", + "profile", + "-l", + "raw", + "--config", + str(config_path), + "--year", + "2022", + ], ) assert profile_result.exit_code == 0, profile_result.output - assert "PROFILE RAW ->" in profile_result.output + assert "Encoding:" in profile_result.output _assert_profile_written(project_example) -SIMPLE_CSV = "a,b,c\n1,2,3\n4,5,6\n" - - -def test_inspect_profile_csv_path_text_output(tmp_path: Path, runner, chdir_tmp: Path) -> None: - """--csv-path stampa encoding/delim/colonne in output testo.""" - csv_file = tmp_path / "test.csv" - csv_file.write_text(SIMPLE_CSV, encoding="utf-8") - - result = runner.invoke(app, ["inspect", "profile", "--csv-path", str(csv_file)]) - assert result.exit_code == 0, result.output - assert "Encoding:" in result.output - assert "Delim:" in result.output - assert "Colonne:" in result.output - assert "a" in result.output - assert "b" in result.output - - -def test_inspect_profile_csv_path_json_output(tmp_path: Path, runner, chdir_tmp: Path) -> None: - """--csv-path --json produce JSON parsabile con struttura attesa.""" - csv_file = tmp_path / "test.csv" - csv_file.write_text(SIMPLE_CSV, encoding="utf-8") - - result = runner.invoke(app, ["inspect", "profile", "--csv-path", str(csv_file), "--json"]) - assert result.exit_code == 0, result.output - payload = json.loads(result.output) - assert payload["column_count"] == 3 - assert payload["encoding_suggested"] == "utf-8" - assert len(payload["columns"]) == 3 - assert len(payload["preview"]) == 2 # 2 data rows - - -def test_inspect_profile_csv_path_file_not_found(tmp_path: Path, runner, chdir_tmp: Path) -> None: - """--csv-path con file inesistente deve fallire con errore leggibile.""" - result = runner.invoke(app, ["inspect", "profile", "--csv-path", str(tmp_path / "missing.csv")]) - assert result.exit_code != 0 - err_text = result.output or str(result.exception or "") - assert "CSV non trovato" in err_text or "non trovato" in err_text - - -def test_inspect_profile_csv_path_requires_either_flag( - tmp_path: Path, runner, chdir_tmp: Path -) -> None: - """Senza --config e senza --csv-path, deve dare errore.""" - result = runner.invoke(app, ["inspect", "profile"]) +def test_inspect_config_profile_requires_config(tmp_path: Path, runner, chdir_tmp: Path) -> None: + """inspect config --mode profile senza --config deve dare errore.""" + result = runner.invoke(app, ["inspect", "config", "--mode", "profile"]) assert result.exit_code != 0, f"Expected failure, got:\n{result.output}" - output_clean = result.output.strip() - assert "config" in output_clean.lower() and "csv" in output_clean.lower() def test_write_json_atomic_handles_nan(tmp_path: Path) -> None: diff --git a/tests/test_cli_status.py b/tests/test_cli_status.py index 788d31fa..1f89f0a4 100644 --- a/tests/test_cli_status.py +++ b/tests/test_cli_status.py @@ -77,12 +77,8 @@ def test_status_uses_same_run_dir_as_writer(tmp_path: Path, runner, chdir_tmp: P app, [ "inspect", - "summary", - "--dataset", - "demo_ds", "--year", "2022", - "--latest", "--config", str(config_path), ], @@ -130,12 +126,8 @@ def test_status_reports_raw_hints_when_raw_artifacts_exist( app, [ "inspect", - "summary", - "--dataset", - "demo_ds", "--year", "2022", - "--latest", "--config", str(config_path), ], @@ -272,12 +264,8 @@ def test_status_reports_validation_summary_from_layer_artifacts( app, [ "inspect", - "summary", - "--dataset", - "demo_ds", "--year", "2022", - "--latest", "--config", str(config_path), ], @@ -365,12 +353,8 @@ def test_status_reports_layer_profiles_from_metadata( app, [ "inspect", - "summary", - "--dataset", - "demo_ds", "--year", "2022", - "--latest", "--config", str(config_path), ], diff --git a/tests/test_mcp_toolkit_client.py b/tests/test_mcp_toolkit_client.py index 47eb6e06..4f0c2f50 100644 --- a/tests/test_mcp_toolkit_client.py +++ b/tests/test_mcp_toolkit_client.py @@ -318,36 +318,26 @@ def test_inspect_paths_cli_mcp_contract_alignment( monkeypatch, tmp_path, ) -> None: - """CLI --json output matches InspectPathsResult TypedDict.""" + """CLI --json output from default inspect (summary) includes path info.""" from typer.testing import CliRunner from toolkit.cli.app import app - from toolkit.mcp.types import ( - CleanPaths, - InspectPathsResult, - LayerPaths, - MartPaths, - RawHints, - RawPaths, - ) config_path, _dataset, year = mcp_project_example monkeypatch.chdir(tmp_path) runner = CliRunner() result = runner.invoke( - app, ["inspect", "paths", "--config", str(config_path), "--year", str(year), "--json"] + app, ["inspect", "--config", str(config_path), "--year", str(year), "--json"] ) assert result.exit_code == 0, result.output payload = json.loads(result.output) - for key in InspectPathsResult.__required_keys__: - assert key in payload - for key in LayerPaths.__required_keys__: - assert key in payload["paths"] - for key, container in [(RawPaths, "raw"), (CleanPaths, "clean"), (MartPaths, "mart")]: - for k in key.__required_keys__: - assert k in payload["paths"][container], f"{key.__name__}: {k} mancante" - for key in RawHints.__required_keys__: - assert key in payload["raw_hints"] + assert "dataset" in payload + assert "year" in payload + assert "layers" in payload + for layer in ("raw", "clean", "mart"): + assert layer in payload["layers"] + assert "record" in payload + assert "warnings" in payload def test_inspect_paths_multi_year_defaults_to_max_year(tmp_path, monkeypatch): diff --git a/tests/test_smoke_e2e_flow.py b/tests/test_smoke_e2e_flow.py index 29696215..b7cb761d 100644 --- a/tests/test_smoke_e2e_flow.py +++ b/tests/test_smoke_e2e_flow.py @@ -150,7 +150,7 @@ def test_init_then_full_then_validate(self, tmp_path: Path): assert r["passed"] is True, f"{r['layer']} validation fallita: {r}" # --- FASE 4: inspect summary --json --- - result = _invoke(["inspect", "summary", "--config", str(config_path), "--json"]) + result = _invoke(["inspect", "--config", str(config_path), "--json"]) assert result.exit_code == 0, f"status fallito: {result.output}" status_data = json.loads(result.output) assert status_data["dataset"] == dataset diff --git a/tests/test_smoke_templates_contract_years.py b/tests/test_smoke_templates_contract_years.py index c9c8ba97..b2930d26 100644 --- a/tests/test_smoke_templates_contract_years.py +++ b/tests/test_smoke_templates_contract_years.py @@ -208,8 +208,7 @@ def test_smoke_years_filter_year_and_years_mutual_exclusion( ], ) assert result.exit_code != 0 - assert result.exception is not None - assert "Use either --year or --years, not both" in str(result.exception) + assert "Use either --year or --years, not both" in result.output @pytest.mark.contract diff --git a/toolkit/cli/cmd_run.py b/toolkit/cli/cmd_run.py index a600a99c..9577a2be 100644 --- a/toolkit/cli/cmd_run.py +++ b/toolkit/cli/cmd_run.py @@ -1098,6 +1098,7 @@ def run_default( None, "--config", "-c", help="Path or slug to dataset.yml" ), years: str | None = typer.Option(None, "--years", help="Comma-separated dataset years"), + year: int | None = typer.Option(None, "--year", "-y", help="Single dataset year"), smoke: bool = typer.Option( False, "--smoke", help="Alias per --sample-rows 1000 --sample-bytes 1048576" ), @@ -1116,9 +1117,18 @@ def run_default( """Esegue la pipeline completa: preflight + support + raw → clean → mart.""" if ctx.invoked_subcommand is not None: return - _execute_pipeline( - config, years, smoke, sample_rows, sample_bytes, root, json_output, dry_run - ) + # Unifica --year e --years + years_str = str(year) if year is not None else years + if year is not None and years is not None: + typer.echo("Use either --year or --years, not both", err=True) + raise typer.Exit(code=2) + try: + _execute_pipeline( + config, years_str, smoke, sample_rows, sample_bytes, root, json_output, dry_run + ) + except FileNotFoundError as exc: + typer.echo(str(exc), err=True) + raise typer.Exit(code=1) # Subcomandi layer run_sub.command("preflight")(run_preflight_cmd) diff --git a/toolkit/cli/inspect/__init__.py b/toolkit/cli/inspect/__init__.py index 8e9165d8..53722b53 100644 --- a/toolkit/cli/inspect/__init__.py +++ b/toolkit/cli/inspect/__init__.py @@ -1,10 +1,7 @@ -"""inspect — comando unico per ispezionare un dataset. +"""inspect — ispeziona un dataset. ``toolkit inspect`` (default) → riassunto stato (ex summary). -Flag di modo: --schema, --preview, --profile, --diff, --runs, --resume. - -Backward compat: i vecchi subcomandi (config, summary, runs, paths, profile) -restano funzionanti con deprecation warning. +Subcomandi: config, runs. """ from __future__ import annotations @@ -13,7 +10,7 @@ def register(app: typer.Typer) -> None: - """Register ``toolkit inspect`` (unico comando con flag di modo).""" + """Register ``toolkit inspect`` e subcomandi.""" from toolkit.cli.inspect.config_ops import config as _config from toolkit.cli.inspect.summary_ops import summary as _summary from toolkit.cli.inspect.runs_ops import runs as _runs @@ -21,167 +18,49 @@ def register(app: typer.Typer) -> None: inspect_app = typer.Typer(no_args_is_help=False, add_completion=False) @inspect_app.callback(invoke_without_command=True) - def inspect_cmd( + def inspect_default( ctx: typer.Context, - # Common options config: str | None = typer.Option( None, "--config", "-c", help="Path or slug to dataset.yml" ), year: int | None = typer.Option(None, "--year", "-y", help="Dataset year"), json_output: bool = typer.Option(False, "--json", help="Output JSON"), - # Mode flags - schema: bool = typer.Option(False, "--schema", help="Mostra schema colonne"), - preview: bool = typer.Option(False, "--preview", help="Anteprima dati"), - profile: bool = typer.Option(False, "--profile", help="Profilo raw (encoding/delim)"), - diff: bool = typer.Option(False, "--diff", help="Schema-diff RAW tra anni"), - runs: bool = typer.Option(False, "--runs", help="Mostra storico run"), - resume: bool = typer.Option(False, "--resume", help="Riprendi run fallito"), - # Config-specific - layer: str = typer.Option("clean", "--layer", "-l", help="Layer: raw, clean, mart"), - sql: str | None = typer.Option(None, "--sql", help="SQL query (con --schema)"), - limit: int = typer.Option(20, "--limit", help="Max righe (con --preview o --sql)"), - # Runs-specific - run_id: str | None = typer.Option(None, "--run-id", help="Specific run id (con --runs)"), - from_layer: str | None = typer.Option( - None, "--from-layer", help="Forza ripartenza raw|clean|mart (con --resume)" - ), ): - """Ispeziona un dataset: stato, schema, dati, storico run. - - Di default mostra il riassunto dello stato del dataset. - Usa i flag --schema, --preview, --profile, --diff, --runs o --resume - per cambiare modalità. - """ + """Mostra lo stato del dataset (riassunto).""" if ctx.invoked_subcommand is not None: return - - # Determina modalità - if schema: - _config( - config_path=config, - layer=layer, - mode="schema", - year=year or 0, - sql=sql, - limit=limit, - mart_index=0, - diff=False, - json_output=json_output, - ) - elif preview: - _config( - config_path=config, - layer=layer, - mode="preview", - year=year or 0, - sql=sql, - limit=limit, - mart_index=0, - diff=False, - json_output=json_output, - ) - elif profile: - _config( - config_path=config, - layer="raw", - mode="profile", - year=year or 0, - sql=sql, - limit=limit, - mart_index=0, - diff=False, - json_output=json_output, - ) - elif diff: - _config( - config_path=config, - layer=layer, - mode="schema", - year=year or 0, - sql=sql, - limit=limit, - mart_index=0, - diff=True, - json_output=json_output, - ) - elif runs: - _runs( - config=config, - year=year, - resume=False, - run_id=run_id, - from_layer=None, - limit=limit, - json_output=json_output, - ) - elif resume: - _runs( - config=config, - year=year, - resume=True, - run_id=run_id, - from_layer=from_layer, - limit=limit, - json_output=json_output, - ) - else: - # Default: summary + try: _summary( config=config, year=year, dataset=None, - run_id=run_id, - latest=(run_id is None), + run_id=None, + latest=True, as_json=json_output, ) + except FileNotFoundError as exc: + typer.echo(str(exc), err=True) + raise typer.Exit(code=1) - # ── Backward compat: subcomandi deprecati ────────────────────────── - - @inspect_app.command("summary", hidden=True) - def summary_deprecated( + @inspect_app.command("config") + def inspect_config( config: str | None = typer.Option( None, "--config", "-c", help="Path or slug to dataset.yml" ), - year: int | None = typer.Option(None, "--year", "-y", help="Dataset year"), - dataset: str | None = typer.Option(None, "--dataset", help="Dataset name (auto-da-config)"), - run_id: str | None = typer.Option(None, "--run-id", help="Specific run id"), - latest: bool = typer.Option(False, "--latest", help="Show latest run"), - as_json: bool = typer.Option(False, "--json", help="Output JSON"), - ): - """⚠️ Deprecato: usa 'toolkit inspect' (default).""" - if not as_json: - typer.echo("⚠️ 'inspect summary' è deprecato, usa 'toolkit inspect'", err=True) - _summary( - config=config, year=year, dataset=dataset, run_id=run_id, latest=latest, as_json=as_json - ) - - @inspect_app.command("config", hidden=True) - def config_deprecated( - config_path: str | None = typer.Option( - None, "--config", "-c", help="Path or slug to dataset.yml" - ), layer: str = typer.Option("clean", "--layer", "-l", help="Layer: raw, clean, mart"), mode: str = typer.Option( "schema", "--mode", "-m", help="Modalità: schema, preview, profile, sql" ), year: int = typer.Option(0, "--year", "-y", help="Anno"), - sql: str | None = typer.Option(None, "--sql", help="SQL query"), - limit: int = typer.Option(20, "--limit", help="Max righe"), + sql: str | None = typer.Option(None, "--sql", help="SQL query (mode=sql)"), + limit: int = typer.Option(20, "--limit", help="Max righe (mode=preview/sql)"), mart_index: int = typer.Option(0, "--mart-index", help="Indice tabella mart"), - diff: bool = typer.Option(False, "--diff", help="Schema-diff RAW"), + diff: bool = typer.Option(False, "--diff", help="Schema-diff RAW tra anni"), json_output: bool = typer.Option(False, "--json", help="Output JSON"), ): - """⚠️ Deprecato: usa 'toolkit inspect --schema|--preview|--profile|--diff'.""" - if not json_output: - hint = { - "schema": "--schema", - "preview": "--preview", - "profile": "--profile", - "sql": "--schema --sql", - }.get(mode, f"--{mode}") - typer.echo(f"⚠️ 'inspect config' è deprecato, usa 'toolkit inspect {hint}'", err=True) + """Ispeziona configurazione e dati: schema, preview, profile, SQL o diff.""" _config( - config_path=config_path, + config_path=config, layer=layer, mode=mode, year=year, @@ -192,22 +71,21 @@ def config_deprecated( json_output=json_output, ) - @inspect_app.command("runs", hidden=True) - def runs_deprecated( + @inspect_app.command("runs") + def inspect_runs( config: str | None = typer.Option( None, "--config", "-c", help="Path or slug to dataset.yml" ), year: int | None = typer.Option(None, "--year", "-y", help="Dataset year"), - resume: bool = typer.Option(False, "--resume", help="Resume latest/failed run"), + resume: bool = typer.Option(False, "--resume", help="Riprendi run fallito"), run_id: str | None = typer.Option(None, "--run-id", help="Specific run id"), - from_layer: str | None = typer.Option(None, "--from-layer", help="Force restart layer"), - limit: int = typer.Option(10, "--limit", help="Max runs da elencare"), + from_layer: str | None = typer.Option( + None, "--from-layer", help="Forza ripartenza raw|clean|mart" + ), + limit: int = typer.Option(10, "--limit", help="Max runs"), json_output: bool = typer.Option(False, "--json", help="Output JSON"), ): - """⚠️ Deprecato: usa 'toolkit inspect --runs' o '--resume'.""" - if not json_output: - hint = "--resume" if resume else "--runs" - typer.echo(f"⚠️ 'inspect runs' è deprecato, usa 'toolkit inspect {hint}'", err=True) + """Mostra storico run o riprende un run fallito.""" _runs( config=config, year=year, @@ -218,42 +96,8 @@ def runs_deprecated( json_output=json_output, ) - @inspect_app.command("paths", hidden=True) - def paths_deprecated( - config: str | None = typer.Option( - None, "--config", "-c", help="Path or slug to dataset.yml" - ), - year: int | None = typer.Option(None, "--year", help="Dataset year"), - as_json: bool = typer.Option(False, "--json", help="Emit JSON output"), - ): - """⚠️ Deprecato: usa 'toolkit inspect'.""" - if not as_json: - typer.echo("⚠️ 'inspect paths' è deprecato, usa 'toolkit inspect'", err=True) - from toolkit.cli.inspect.paths_ops import paths as _paths - - _paths(config=config, year=year, as_json=as_json) - - @inspect_app.command("profile", hidden=True) - def profile_deprecated( - config: str | None = typer.Option( - None, "--config", "-c", help="Path or slug to dataset.yml" - ), - csv_path: str | None = typer.Option(None, "--csv-path", help="CSV file to preview"), - year: int | None = typer.Option(None, "--year", "-y", help="Dataset year"), - years: str | None = typer.Option(None, "--years", help="Comma-separated years"), - json_output: bool = typer.Option(False, "--json", help="Output JSON"), - ): - """⚠️ Deprecato: usa 'toolkit inspect --profile'.""" - from toolkit.cli.inspect.profile_ops import profile as _profile - - if not json_output: - typer.echo( - "⚠️ 'inspect profile' è deprecato, usa 'toolkit inspect --profile'", err=True - ) - _profile(config=config, csv_path=csv_path, year=year, years=years, json_output=json_output) - app.add_typer( inspect_app, name="inspect", - help="Ispeziona un dataset: stato, schema, dati, storico run.", + help="Ispeziona un dataset: stato, schema, storico run.", ) diff --git a/toolkit/mcp/path_safety.py b/toolkit/mcp/path_safety.py index 657a2bf1..9f249011 100644 --- a/toolkit/mcp/path_safety.py +++ b/toolkit/mcp/path_safety.py @@ -32,6 +32,9 @@ def _safe_path(config_path: str | Path) -> Path: Delega la risoluzione a ``resolve_config_path()`` traducendo ``FileNotFoundError`` in ``ToolkitClientError``. + Per path di file esistenti non-YAML (es. CSV per preview), li + restituisce direttamente senza passarli a ``resolve_config_path``. + Args: config_path: Path o slug da risolvere. @@ -41,6 +44,9 @@ def _safe_path(config_path: str | Path) -> Path: Raises: ToolkitClientError: CONFIG_NOT_FOUND se irrisolvibile. """ + p = Path(config_path).expanduser() + if p.is_file() and p.suffix not in (".yml", ".yaml"): + return p.resolve() try: return resolve_config_path(hint=config_path) except FileNotFoundError as exc: From 4b29f7d369d9ce435a0373a1e910fffb42f27e29 Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Thu, 30 Jul 2026 18:56:56 +0100 Subject: [PATCH 7/8] docs: README, workflow, feature-stability, notebook-contract allineati alla nuova CLI Aggiunti test contract per auto-detect config (CWD, slug, mancante). Docstring _execute_pipeline() con parametri documentati. --- CONTRIBUTING.md | 8 ++++---- README.md | 27 ++++++++++++++----------- docs/advanced-workflows.md | 16 +++++++-------- docs/feature-stability.md | 9 ++++----- docs/notebook-contract.md | 16 +++++++-------- tests/test_cli_inspect.py | 40 ++++++++++++++++++++++++++++++++++++++ toolkit/cli/cmd_run.py | 14 +++++++++++-- 7 files changed, 92 insertions(+), 38 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 96dd2188..43ec4a5e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -66,13 +66,13 @@ mypy toolkit/ ```bash # Run completo su dataset di esempio -python -m toolkit.cli.app run all --config project-example/dataset.yml +python -m toolkit.cli.app run --config project-example/dataset.yml # Validate python -m toolkit.cli.app validate all --config project-example/dataset.yml -# Inspect paths -python -m toolkit.cli.app inspect paths --config project-example/dataset.yml +# Inspect (default: summary con path info) +python -m toolkit.cli.app inspect --config project-example/dataset.yml # Profile RAW (canonico: inspect config) python -m toolkit.cli.app inspect config -c project-example/dataset.yml -l raw -m profile @@ -85,7 +85,7 @@ critica per il rilascio: | Marker | Cosa copre | Deve sempre passare | |---|---|---| -| `core` | Contratto pubblico e percorso canonico — config, path contract, `run all`, `validate all`, end-to-end RAW→CLEAN→MART, run records, resume | ✅ Sì | +| `core` | Contratto pubblico e percorso canonico — config, path contract, `run`, `validate all`, end-to-end RAW→CLEAN→MART, run records, resume | ✅ Sì | | `advanced` | Comportamenti secondari — read modes, extractors, plugin registry, profiling, artifact policy | ✅ Su release | | `compat` | Solo compatibilità legacy e shim di import deprecati | No (può decadere) | diff --git a/README.md b/README.md index 8d28b391..58fa25ab 100644 --- a/README.md +++ b/README.md @@ -19,11 +19,11 @@ chiaro tra ogni layer. ```bash pip install -e .[dev] -toolkit run full -c dataset.yml -toolkit inspect summary -c dataset.yml +toolkit run -c dataset.yml +toolkit inspect -c dataset.yml ``` -Se `toolkit` non è nel PATH: `python -m toolkit.cli.app run all -c dataset.yml` +Se `toolkit` non è nel PATH: `python -m toolkit.cli.app run -c dataset.yml` ## Pipeline: tre livelli @@ -53,14 +53,19 @@ La CI di `dataset-incubator` carica su GCS dopo ogni run validato. ## CLI — comandi essenziali | Comando | Cosa fa | -|---|---| -| `toolkit run all --config dataset.yml` | Prima esecuzione completa | -| `toolkit run clean --config dataset.yml` | Solo layer CLEAN | -| `toolkit run mart --config dataset.yml` | Solo layer MART | -| `toolkit inspect summary --config dataset.yml` | Stato ultimo run | -| `toolkit inspect paths --config dataset.yml --year 2023` | Path assoluti degli output | +|---|---|---| +| `toolkit run` | Esecuzione completa RAW→CLEAN→MART | +| `toolkit run raw` | Solo layer RAW | +| `toolkit run clean` | Solo layer CLEAN | +| `toolkit run mart` | Solo layer MART | +| `toolkit inspect` | Stato ultimo run (riassunto) | +| `toolkit inspect config --diff` | Schema-diff RAW tra anni | +| `toolkit inspect runs --resume` | Riprendi run interrotto | | `toolkit scout ` | Esplora fonte esterna (HTTP/CKAN/SDMX) | +`--config` è opzionale: se omesso, toolkit cerca `dataset.yml` nella directory corrente. +Se passi uno slug (es. `terna-electricity-by-source`), lo risolve nel workspace. + 📖 **Reference completo**: `toolkit --help` ## Configurazione (`dataset.yml`) @@ -121,11 +126,11 @@ Config IDE (`.mcp.json`): ## FAQ — problemi comuni | Problema | Soluzione | -|---|---| +|---|---|---| | `toolkit: command not found` | Usa `python -m toolkit.cli.app` | | Run interrotto | `toolkit inspect runs --resume -c dataset.yml` | | Schema diverso tra anni | `toolkit inspect config -c dataset.yml --diff` | -| Dove sono i parquet? | `toolkit inspect paths --config dataset.yml --year ` | +| Dove sono i parquet? | `toolkit inspect -c dataset.yml` (mostra path nel riassunto) | ## Sviluppo diff --git a/docs/advanced-workflows.md b/docs/advanced-workflows.md index 3206f970..9a33fde3 100644 --- a/docs/advanced-workflows.md +++ b/docs/advanced-workflows.md @@ -4,8 +4,8 @@ Questa nota raccoglie i flussi e le opzioni del toolkit che restano supportati, Percorso canonico: -- `toolkit run full --config dataset.yml` -- `toolkit inspect summary -c dataset.yml` +- `toolkit run --config dataset.yml` +- `toolkit inspect -c dataset.yml` - `toolkit inspect config -c dataset.yml` - notebook locali che leggono output e metadata sotto `root/data/...` @@ -19,9 +19,9 @@ Questa categoria include anche tooling di supporto che non va confuso con il run Regola pratica: -- se stai eseguendo un dataset per la prima volta, parti da `toolkit run all` +- se stai eseguendo un dataset per la prima volta, parti da `toolkit run` - se hai cambiato fonte, anni, extractor, `dataset.yml` o il perimetro del RAW, - torna a `toolkit run all` + torna a `toolkit run` - se hai cambiato `clean.sql` o la logica `clean.read`, riparti da `toolkit run clean` e poi `toolkit run mart` - se hai toccato solo SQL `mart`, preferisci `toolkit run mart` @@ -36,9 +36,9 @@ Matrice minima: | Tipo di modifica | Comando consigliato | |---|---| -| prima esecuzione del dataset | `toolkit run all --config dataset.yml` | -| cambio fonte o perimetro anni | `toolkit run all --config dataset.yml` | -| cambio `dataset.yml` con impatto su input/layer | `toolkit run all --config dataset.yml` | +| prima esecuzione del dataset | `toolkit run --config dataset.yml` | +| cambio fonte o perimetro anni | `toolkit run --config dataset.yml` | +| cambio `dataset.yml` con impatto su input/layer | `toolkit run --config dataset.yml` | | cambio `clean.sql` o `clean.read` | `toolkit run clean --config dataset.yml` poi `toolkit run mart --config dataset.yml` | | cambio solo `mart.sql` | `toolkit run mart --config dataset.yml` | | cambio solo tabella multi-anno | `toolkit run mart --config dataset.yml` | @@ -52,7 +52,7 @@ il layer che stai rieseguendo. In pratica: -- non trattare `run all` come default per ogni modifica minima +- non trattare `run` come default per ogni modifica minima - non cancellare gli output locali "per pulizia" se non hai cambiato il loro perimetro - usa i rerun parziali quando il punto di ingresso corretto è chiaro - usa `resume` per recovery, non come scorciatoia generica a metà sviluppo diff --git a/docs/feature-stability.md b/docs/feature-stability.md index dd8de2dc..e9d59cc9 100644 --- a/docs/feature-stability.md +++ b/docs/feature-stability.md @@ -6,14 +6,13 @@ Questa matrice serve a chiarire cosa il toolkit considera percorso canonico, cos |---|---|---| | `query` | stable | query SQL su parquet (path o dataset.yml + layer) | | `parquet_preview(sql=...)` | stable | API core per SQL arbitrario su parquet | -| `run all` | stable | percorso canonico | +| `run` | stable | percorso canonico | | `validate all` | stable | percorso canonico | -| `inspect summary` | stable | percorso canonico | +| `inspect` | stable | percorso canonico | | path contract di `dataset.yml` | stable | percorso canonico | | output `raw/clean/mart/_runs` | stable | percorso canonico | -| `inspect paths` | stable | helper per notebook e repo dataset | | `inspect runs --resume` | supported / advanced | debug operativo e recovery | -| `inspect profile` | supported / advanced | diagnostica su RAW sporchi o ambigui | +| `inspect config --mode profile` | supported / advanced | diagnostica su RAW sporchi o ambigui | | `run raw\|clean\|mart` | supported / advanced | debug e re-run parziali | | `scout` | stable | esplorazione URL esterni, probe e routing automatico | | `scout --scaffold` | stable | probe + scaffold candidate dataset (dataset.yml, SQL, README) | @@ -28,7 +27,7 @@ Questa matrice serve a chiarire cosa il toolkit considera percorso canonico, cos Lettura equivalente a livello package: - core runtime: `toolkit.raw`, `toolkit.clean`, `toolkit.mart`, `toolkit.scout`, `toolkit.cli` (`run`, `validate`, `inspect`) -- advanced tooling: `inspect runs --resume`, run parziali, `inspect profile`, `inspect config --diff` +- advanced tooling: `inspect runs --resume`, run parziali, `inspect config --mode profile`, `inspect config --diff` - compatibility only: config legacy e alias storici Sorgenti builtin supportate dal runtime canonico: `local_file`, `http_file`, `http_post_file`, `ckan`, `sdmx`, `sparql`. Il runtime può conservare `.xlsx` e `.xls` in RAW e leggerli in CLEAN — il file originale resta l'artefatto sorgente. diff --git a/docs/notebook-contract.md b/docs/notebook-contract.md index c075bb27..2e449d04 100644 --- a/docs/notebook-contract.md +++ b/docs/notebook-contract.md @@ -19,21 +19,21 @@ Ruoli dei file: - `metadata.json`: payload ricco del layer. Contiene input, output, `config_hash`, summary di validazione e campi specifici del layer. - run record in `data/_runs/...`: stato del run (`run_id`, layer, validations, status), utile per `status` e `resume`. -- `inspect paths --json`: helper read-only per notebook e script locali; restituisce i path assoluti utili del runtime, incluso `latest_run`. +- `inspect --json` (o `inspect --config dataset.yml --json`): helper read-only per notebook e script locali; restituisce i path assoluti utili del runtime, incluso `latest_run`. Per evitare duplicazione di path logic nei notebook: - leggi `dataset.yml` -- usa `toolkit inspect paths --config dataset.yml --year --json` +- usa `toolkit inspect --json --config dataset.yml --year ` - poi apri parquet, metadata, manifest, validation e run record dai path restituiti Nota pratica: -- `inspect paths` restituisce path assoluti della macchina locale: è pensato per notebook e script nello stesso ambiente, non come formato portabile tra macchine diverse. +- `inspect --json` restituisce path assoluti della macchina locale: è pensato per notebook e script nello stesso ambiente, non come formato portabile tra macchine diverse. -## Contratto operativo di `inspect paths` +## Contratto operativo di `inspect --json` -`inspect paths` è il comando da usare quando il problema è: +`inspect --json` è il comando da usare quando il problema è: - trovare i path runtime già risolti dal toolkit - evitare di ricostruire a mano `root/data/...` @@ -65,8 +65,8 @@ Output garantito in `--json`: Regola pratica: -- notebook e script locali: usa sempre `inspect paths --json` -- CI che deve validare `effective_root` o path contract: usa `inspect paths --json` +- notebook e script locali: usa sempre `inspect --json` +- CI che deve validare `effective_root` o path contract: usa `inspect --json` - se non passi `--year`, il payload può essere una lista multi-anno ## Differenza rispetto a `inspect config --diff` @@ -81,7 +81,7 @@ Serve invece quando vuoi: In breve: -- `inspect paths`: "dove sono gli artefatti e quale runtime path contract posso usare?" +- `inspect --json`: "dove sono gli artefatti e quale runtime path contract posso usare?" - `inspect config --diff`: "il RAW cambia tra anni e quanto cambia?" Regola pratica: diff --git a/tests/test_cli_inspect.py b/tests/test_cli_inspect.py index 3d4a73a4..90544748 100644 --- a/tests/test_cli_inspect.py +++ b/tests/test_cli_inspect.py @@ -249,3 +249,43 @@ def test_inspect_help_shows_modes(self): assert "--json" in output assert "config" in output assert "runs" in output + + +class TestAutoDetect: + """auto-detect config — --config opzionale.""" + + @pytest.mark.contract + def test_missing_config_fails_from_empty_dir(self, tmp_path): + """Senza dataset.yml nella CWD deve fallire con errore chiaro.""" + import os + + old = os.getcwd() + os.chdir(tmp_path) + try: + result = runner.invoke(app, ["inspect"]) + assert result.exit_code != 0 + assert "non trovato" in result.output + finally: + os.chdir(old) + + @pytest.mark.contract + def test_auto_detect_from_cwd(self, project_example): + """Con dataset.yml nella CWD, --config non serve.""" + import os + + old = os.getcwd() + os.chdir(project_example) + try: + # run --dry-run deve trovare dataset.yml nella CWD + result = runner.invoke(app, ["run", "--dry-run"]) + assert result.exit_code == 0, result.output + assert "Execution Plan" in result.output + finally: + os.chdir(old) + + @pytest.mark.contract + def test_nonexistent_slug_fails(self): + """Slug inesistente deve fallire con errore chiaro.""" + result = runner.invoke(app, ["inspect", "-c", "this-slug-definitely-does-not-exist"]) + assert result.exit_code != 0 + assert "Nessun dataset trovato" in result.output diff --git a/toolkit/cli/cmd_run.py b/toolkit/cli/cmd_run.py index 9577a2be..b77de982 100644 --- a/toolkit/cli/cmd_run.py +++ b/toolkit/cli/cmd_run.py @@ -765,8 +765,18 @@ def _execute_pipeline( ) -> None: """Esegue pre-flight + support + raw → clean → mart. - Core della pipeline completa. Chiamata sia dal comando ``toolkit run`` - (default) che da ``run full`` (deprecato). + Core della pipeline completa. Chiamata dal comando ``toolkit run`` + (default) e da ``run_full()`` (deprecato). + + Args: + config: Path/slug per dataset.yml, o None per auto-detect CWD. + years: Anni separati da virgola, o None per tutti quelli configurati. + smoke: Se True, attiva --sample-rows 1000 --sample-bytes 1048576. + sample_rows: Limite righe in CLEAN (LIMIT su output SQL). + sample_bytes: Limite bytes in RAW (HTTP Range + troncamento). + root: Override root output directory. + json_output: Se True, stampa report JSON su stdout. + dry_run: Se True, solo plan senza esecuzione. """ dry_flag = dry_run if isinstance(dry_run, bool) else False From 10448e34134a978537e932559730446ad3f2b66f Mon Sep 17 00:00:00 2001 From: Zio Gabber <78922322+Gabrymi93@users.noreply.github.com> Date: Thu, 30 Jul 2026 18:58:29 +0100 Subject: [PATCH 8/8] fix: smoke_install_cli allineato alla nuova CLI (run, inspect) --- scripts/smoke_install_cli.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/smoke_install_cli.py b/scripts/smoke_install_cli.py index 820a70af..cb1b5d7f 100644 --- a/scripts/smoke_install_cli.py +++ b/scripts/smoke_install_cli.py @@ -45,9 +45,8 @@ def main() -> int: _run([str(toolkit_cmd), "--help"], cwd=repo_root, env=env) _run([str(toolkit_cmd), "run", "--help"], cwd=repo_root, env=env) _run([str(toolkit_cmd), "inspect", "--help"], cwd=repo_root, env=env) - _run([str(toolkit_cmd), "inspect", "--profile", "--help"], cwd=repo_root, env=env) _run( - [str(toolkit_cmd), "run", "all", "--dry-run", "-c", "examples/dataset_min.yml"], + [str(toolkit_cmd), "run", "--dry-run", "-c", "examples/dataset_min.yml"], cwd=repo_root, env=env, )