Skip to content

[caliper] Progress with the parser and plotter #57

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kpouget wants to merge 16 commits into
base: main
Choose a base branch
from
Open

[caliper] Progress with the parser and plotter #57

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
7a3f61e
[caliper] cli: main: remove alternative flag names
kpouget May 7, 2026
7a0dbe9
[caliper] cli: main: add 'caliper artifacts import' command
kpouget May 12, 2026
d40b69a
[caliper] cli: main: better error handling
kpouget May 12, 2026
9351e03
[caliper] postprocess: guidellm: new post-processing helper
kpouget May 12, 2026
9c65fe3
[caliper] engine: file_export: runner: don't indent everything under …
kpouget May 12, 2026
376bcbb
[core] Show a helpful message on 'command not found'
kpouget May 12, 2026
4e2e5d1
[core] library: replot: add the replot library
kpouget May 12, 2026
c745d7f
[skeleton] postprocess: refactor the default plugin
kpouget May 7, 2026
9ff0975
[skeleton] orchestration: config: add caliper.replot
kpouget May 13, 2026
427a3a5
[fournos_launcher] sanitize the job name before submitting it
kpouget May 12, 2026
5b56f98
[fournos_launcher] orchestration: pr_args: add /replot.url parsing
kpouget May 12, 2026
2f2af26
[llm_d_legacy] orchestration: ci: don't request GPUs
kpouget May 12, 2026
e449b9d
[llm_d_legacy] orchestration: ci: add the replot entrypoint
kpouget May 12, 2026
676bb5f
[llm_d_legacy] testing: config: add the caliper post-configuration
kpouget May 13, 2026
da0cfaf
[llm_d_legacy] orchestration: ci: init in main
kpouget May 13, 2026
ed039a3
[caliper] orchestration: replot: new binding
kpouget May 13, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
100 changes: 84 additions & 16 deletions projects/caliper/cli/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ def _require_artifacts_dir(ctx: click.Context) -> Path:
_exit_with_help(
ctx,
"This command requires the test artifact tree root: "
"`--artifacts-dir DIR` or `--base-dir DIR` "
"`--artifacts-dir DIR` "
"(before or after the subcommand).",
code=1,
)
Expand Down Expand Up @@ -89,7 +89,6 @@ def _workspace_cli_options(cmd: Any) -> Any:
opts = (
click.option(
"--artifacts-dir",
"--base-dir",
"artifacts_dir",
type=click.Path(path_type=Path, exists=True),
default=None,
Expand All @@ -105,12 +104,11 @@ def _workspace_cli_options(cmd: Any) -> Any:
help=_POSTPROCESS_CONFIG_HELP + " Overrides the global option when set here.",
),
click.option(
"--plugin-module",
"--plugin",
"plugin_module_override",
metavar="MODULE",
default=None,
help="Plugin import path; same as global --plugin-module / --plugin.",
help="Plugin import path; same as global --plugin.",
),
)
for opt in reversed(opts):
Expand Down Expand Up @@ -141,7 +139,6 @@ def _plugin_tuple(ctx: click.Context) -> tuple[str, Any]:
@click.group(context_settings={"help_option_names": ["-h", "--help"]})
@click.option(
"--artifacts-dir",
"--base-dir",
"artifacts_dir",
type=click.Path(path_type=Path, exists=True),
default=None,
Expand All @@ -154,7 +151,6 @@ def _plugin_tuple(ctx: click.Context) -> tuple[str, Any]:
help=_POSTPROCESS_CONFIG_HELP,
)
@click.option(
"--plugin-module",
"--plugin",
"plugin_module",
metavar="MODULE",
Expand Down Expand Up @@ -360,7 +356,7 @@ def kpi_analyze(
@main.group("artifacts")
@click.pass_context
def artifacts_group(ctx: click.Context) -> None:
"""File artifact export."""
"""File artifact export and import."""


@artifacts_group.command("export")
Expand Down Expand Up @@ -521,14 +517,87 @@ def ai_eval_export(
plugin=plugin,
output=output,
use_cache=True,
cache_path=None,
)
except Exception as e: # noqa: BLE001
click.echo(f"ai-eval-export failed: {e}", err=True)
sys.exit(2)
click.echo(f"Wrote {output}")


@artifacts_group.command("import")
@click.option("--from-mlflow", "mlflow_run_id", help="MLflow run ID to download artifacts from.")
@click.option(
"--output-dir",
type=click.Path(path_type=Path),
required=True,
help="Local directory to download artifacts to.",
)
@click.option(
"--mlflow-tracking-uri",
envvar="MLFLOW_TRACKING_URI",
help="MLflow tracking server URI (can be set via MLFLOW_TRACKING_URI).",
)
@click.option(
"--artifact-path",
default="",
help="Specific artifact path to download (default: download all artifacts).",
)
@click.pass_context
def import_command(
ctx: click.Context,
mlflow_run_id: str | None,
output_dir: Path,
mlflow_tracking_uri: str | None,
artifact_path: str,
) -> None:
"""Download artifact files from MLflow."""
if mlflow_run_id:
if not mlflow_tracking_uri:
click.echo(
"Error: MLflow tracking URI required. Set --mlflow-tracking-uri or MLFLOW_TRACKING_URI.",
err=True,
)
sys.exit(1)

try:
import mlflow
from mlflow.tracking import MlflowClient

# Set tracking URI
mlflow.set_tracking_uri(mlflow_tracking_uri)
client = MlflowClient()

# Download artifacts
output_dir.mkdir(parents=True, exist_ok=True)

# Download artifacts to the output directory
downloaded_path = client.download_artifacts(
run_id=mlflow_run_id, path=artifact_path, dst_path=str(output_dir)
)

# Count downloaded files
if Path(downloaded_path).is_file():
downloaded_files = [Path(downloaded_path)]
else:
downloaded_files = list(Path(downloaded_path).rglob("*"))
downloaded_files = [f for f in downloaded_files if f.is_file()]

click.echo(f"Downloaded {len(downloaded_files)} files to {output_dir}")
if downloaded_files:
click.echo("Downloaded files:")
for file in downloaded_files[:10]: # Show first 10
click.echo(f" {file.relative_to(output_dir)}")
if len(downloaded_files) > 10:
click.echo(f" ... and {len(downloaded_files) - 10} more")

except Exception as e: # noqa: BLE001
click.echo(f"artifacts import failed: {e}", err=True)
sys.exit(2)
else:
click.echo("Error: Specify source backend: --from-mlflow RUN_ID", err=True)
sys.exit(1)
Comment thread
coderabbitai[bot] marked this conversation as resolved.


def run_cli() -> None:
"""Invoke CLI; on missing required options, print subcommand help."""
try:
Expand All @@ -537,14 +606,13 @@ def run_cli() -> None:
rv = main.main(standalone_mode=False, prog_name="caliper")
if isinstance(rv, int) and rv != 0:
sys.exit(rv)
except click.MissingParameter as exc:
msg = exc.format_message()
sub = getattr(exc, "ctx", None)
click.echo(f"Error: {msg}\n", err=True)
if sub is not None:
click.echo(sub.get_help(), err=True)
ec = getattr(exc, "exit_code", 2)
sys.exit(2 if ec is None else int(ec))
except click.ClickException as exc:
# Handle click exceptions including NoArgsIsHelpError and MissingParameter
if hasattr(exc, "ctx") and exc.ctx:
click.echo(exc.ctx.get_help(), err=True)
else:
exc.show(sys.stderr)
sys.exit(2)
Comment on lines +609 to +615
Copy link
Copy Markdown

@coderabbitai coderabbitai Bot May 13, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟡 Minor | ⚡ Quick win

🧩 Analysis chain

🌐 Web query:

In Click 8.1.0, does click.Context.get_help()include the original exception text forMissingParameterandNoSuchOption, or is exc.show(...) required to preserve it?

💡 Result:

In Click 8.1.0, click.Context.get_help does NOT include the original exception text for MissingParameter or NoSuchOption. In other words, exc.show(...) is required if you want the exception’s message (e.g., the “Missing parameter …” or “No such option …” text) to be preserved and displayed. Reasoning grounded in the code: - Context.get_help just returns the formatted help page generated from the command (it calls command.get_help/format_help) and has no access to any active exception or its message. [1] - ClickException.show is the method responsible for rendering the exception message to stderr/file (including adding the “Try '… --help' for help” hint and then echoing “Error: {message}”). [2] - MissingParameter and NoSuchOption both implement their own message formatting via ClickException.format_message / str logic (so the message lives on the exception object, not on the help text). [2] So: if you call ctx.get_help (or command.get_help(ctx)) in an exception handler without also showing the exception, you will get only usage/help text, not the original exception message; to include it you must call exc.show(...).

Citations:

Preserve error messages when showing help for Click exceptions.

When exc.ctx is present, the current code shows only the help text and loses the actual error message (e.g., "Missing parameter …" or "No such option …"). Click's Context.get_help() returns only the formatted help page; exc.show() is required to display the exception message. Call exc.show(sys.stderr) first, then optionally show help:

Proposed fix 
     except click.ClickException as exc:
-        # Handle click exceptions including NoArgsIsHelpError and MissingParameter
-        if hasattr(exc, "ctx") and exc.ctx:
-            click.echo(exc.ctx.get_help(), err=True)
-        else:
-            exc.show(sys.stderr)
+        # Preserve Click's error text, then add command help when available.
+        exc.show(sys.stderr)
+        if getattr(exc, "ctx", None):
+            click.echo("", err=True)
+            click.echo(exc.ctx.get_help(), err=True)
         sys.exit(2)
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
except click.ClickException as exc:
# Handle click exceptions including NoArgsIsHelpError and MissingParameter
if hasattr(exc, "ctx") and exc.ctx:
click.echo(exc.ctx.get_help(), err=True)
else:
exc.show(sys.stderr)
sys.exit(2)
except click.ClickException as exc:
# Preserve Click's error text, then add command help when available.
exc.show(sys.stderr)
if getattr(exc, "ctx", None):
click.echo("", err=True)
click.echo(exc.ctx.get_help(), err=True)
sys.exit(2)
🤖 Prompt for AI Agents 
Verify each finding against current code. Fix only still-valid issues, skip the
rest with a brief reason, keep changes minimal, and validate.

In `@projects/caliper/cli/main.py` around lines 609 - 615, The exception handling
for click.ClickException currently prints only the help text when exc.ctx exists
and drops the actual error message; update the handler so it calls
exc.show(sys.stderr) first to print the exception message (e.g., from
MissingParameter or NoArgsIsHelpError) and then, if exc.ctx is present, call
click.echo(exc.ctx.get_help(), err=True) to also display the help page; keep the
existing sys.exit(2) behavior.

except SystemExit:
raise

Expand Down
44 changes: 22 additions & 22 deletions projects/caliper/engine/file_export/runner.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,29 +85,29 @@ def run_file_export(
try:
if verbose:
print(f"caliper: starting backend {b!r} …", file=sys.stderr)
detail, ml_meta = mlflow_backend.log_artifacts(
artifact_root=source,
paths=paths,
tracking_uri=mlflow_tracking_uri,
experiment=mlflow_experiment,
run_id=mlflow_run_id,
run_name=mlflow_run_name,
insecure_tls=mlflow_insecure_tls,
connection=mlflow_connection,
verbose=verbose,
upload_workers=upload_workers,
run_metadata=mlflow_run_metadata,
)
results.append(
FileExportBackendResult(
backend="mlflow",
status="success",
detail=detail,
metadata=ml_meta,
)
detail, ml_meta = mlflow_backend.log_artifacts(
artifact_root=source,
paths=paths,
tracking_uri=mlflow_tracking_uri,
experiment=mlflow_experiment,
run_id=mlflow_run_id,
run_name=mlflow_run_name,
insecure_tls=mlflow_insecure_tls,
connection=mlflow_connection,
verbose=verbose,
upload_workers=upload_workers,
run_metadata=mlflow_run_metadata,
)
results.append(
FileExportBackendResult(
backend="mlflow",
status="success",
detail=detail,
metadata=ml_meta,
)
if verbose:
print(f"caliper: backend {b!r} finished ({detail})", file=sys.stderr)
)
if verbose:
print(f"caliper: backend {b!r} finished ({detail})", file=sys.stderr)
except Exception as e: # noqa: BLE001
# Full chain on stderr; stdout line stays a short message for automation.
traceback.print_exception(e, file=sys.stderr)
Expand Down
Loading
Loading