From de2920bf45b3c5e063d1d2a1cd40a5323d580d89 Mon Sep 17 00:00:00 2001 From: Narendran Raghavan Date: Tue, 15 Sep 2026 21:12:18 -0700 Subject: [PATCH 1/3] feat(report): add LLM analysis provenance Signed-off-by: Narendran Raghavan --- src/skillspector/inference_usage.py | 4 + src/skillspector/llm_provenance.py | 494 ++++++++++++++++++++++ src/skillspector/nodes/build_context.py | 5 +- src/skillspector/nodes/report.py | 14 +- src/skillspector/providers/chat_models.py | 49 ++- src/skillspector/state.py | 3 + tests/nodes/test_report.py | 87 ++++ tests/unit/test_llm_provenance.py | 436 +++++++++++++++++++ tests/unit/test_providers.py | 25 ++ 9 files changed, 1101 insertions(+), 16 deletions(-) create mode 100644 src/skillspector/llm_provenance.py create mode 100644 tests/unit/test_llm_provenance.py diff --git a/src/skillspector/inference_usage.py b/src/skillspector/inference_usage.py index 6726fd9b4..f47db9c1b 100644 --- a/src/skillspector/inference_usage.py +++ b/src/skillspector/inference_usage.py @@ -110,14 +110,18 @@ def _model_label(value: object, fallback: str = "unknown") -> str: def provider_name(provider: object) -> str: """Return a stable provider label without endpoint or credential data.""" names = { + "AntigravityCLIProvider": "antigravity_cli", "AnthropicProvider": "anthropic", "AnthropicProxyProvider": "anthropic_proxy", + "AzureOpenAIProvider": "azure_openai", "BedrockProvider": "bedrock", "ClaudeCLIProvider": "claude_cli", "CodexCLIProvider": "codex_cli", "GeminiCLIProvider": "gemini_cli", "NvBuildProvider": "nv_build", "NvInferenceProvider": "nv_inference", + "OllamaProvider": "ollama", + "OpenAICompatibleProvider": "openai_compatible", "OpenAIProvider": "openai", } return names.get(type(provider).__name__, _label(type(provider).__name__.lower())) diff --git a/src/skillspector/llm_provenance.py b/src/skillspector/llm_provenance.py new file mode 100644 index 000000000..d7d28025b --- /dev/null +++ b/src/skillspector/llm_provenance.py @@ -0,0 +1,494 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Sanitized, scan-level provenance for LLM-backed analysis configuration.""" + +from __future__ import annotations + +import math +import os +import re +from collections.abc import Mapping, Sequence +from importlib.metadata import version + +from skillspector.inference_usage import provider_name +from skillspector.providers import get_active_provider, get_model_config_provider +from skillspector.providers.chat_models import ( + MAX_SAMPLING_SEED, + MIN_SAMPLING_SEED, + resolve_reasoning_effort, + resolve_seed, + resolve_temperature, +) + +LLM_PROVENANCE_SCHEMA_VERSION = 1 +LLM_ANALYZER_SLOTS = ( + "mcp_tool_poisoning", + "semantic_developer_intent", + "semantic_quality_policy", + "semantic_security_discovery", + "meta_analyzer", +) + +_TEMPERATURE_ADAPTERS = frozenset( + { + "anthropic", + "anthropic_proxy", + "azure_openai", + "bedrock", + "nv_build", + "ollama", + "openai", + "openai_compatible", + } +) +_SEED_ADAPTERS = frozenset({"azure_openai", "nv_build", "ollama", "openai", "openai_compatible"}) +_REASONING_EFFORT_ADAPTERS = frozenset( + { + "anthropic", + "anthropic_proxy", + "nv_build", + "ollama", + "openai", + "openai_compatible", + } +) +_SAFE_LABEL = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/+\-]{0,255}") +_SAFE_SETTING = re.compile(r"[A-Za-z0-9][A-Za-z0-9 ._:/+\-]{0,255}") +_SECRET_PREFIXES = ("sk-", "nvapi-", "ghp_", "glpat-", "bearer-") +_MAX_SAFE_SEED = MAX_SAMPLING_SEED +_MIN_SAFE_SEED = MIN_SAMPLING_SEED +_CONTROL_SOURCES = frozenset( + {"environment", "provider_default", "unset", "invalid_environment", "out_of_range", "unknown"} +) +_DEPLOYMENT_SOURCES = frozenset({"environment", "resolved_model", "not_applicable", "unknown"}) +_API_VERSION_SOURCES = frozenset({"environment", "provider_default", "not_applicable", "unknown"}) +_AZURE_OPENAI_DEFAULT_API_VERSION = "2024-06-01" + + +def _safe_label(value: object, fallback: str = "unknown") -> str: + candidate = value if isinstance(value, str) else "" + candidate = candidate.strip() + lowered = candidate.lower() + if ( + _SAFE_LABEL.fullmatch(candidate) + and "://" not in candidate + and "@" not in candidate + and not lowered.startswith(_SECRET_PREFIXES) + ): + return candidate + return fallback + + +def _safe_setting(value: object, fallback: str = "unknown") -> str: + """Return a bounded printable setting while allowing provider-specific spaces.""" + candidate = value if isinstance(value, str) else "" + candidate = candidate.strip() + lowered = candidate.lower() + if ( + _SAFE_SETTING.fullmatch(candidate) + and "://" not in candidate + and "@" not in candidate + and not lowered.startswith(_SECRET_PREFIXES) + ): + return candidate + return fallback + + +def _safe_optional_label(value: object) -> str | None: + """Return a safe label or ``None`` without inventing a placeholder.""" + label = _safe_label(value, fallback="") + return label or None + + +def _capture_provider_routing(resolved_adapter: str) -> dict[str, object]: + """Capture non-secret provider routing inputs used by client construction.""" + if resolved_adapter != "azure_openai": + return { + "deployment_override": None, + "deployment_source": "not_applicable", + "api_version": None, + "api_version_source": "not_applicable", + } + + raw_deployment = os.environ.get("AZURE_OPENAI_DEPLOYMENT", "").strip() + deployment = _safe_optional_label(raw_deployment) + raw_api_version = os.environ.get("AZURE_OPENAI_API_VERSION", "").strip() + api_version = _safe_optional_label(raw_api_version or _AZURE_OPENAI_DEFAULT_API_VERSION) + return { + "deployment_override": deployment, + "deployment_source": ( + "environment" if deployment else "unknown" if raw_deployment else "resolved_model" + ), + "api_version": api_version, + "api_version_source": ( + "environment" + if raw_api_version and api_version + else "unknown" + if raw_api_version + else "provider_default" + ), + } + + +def _requested_temperature(raw: str) -> tuple[float | None, str]: + if not raw: + return None, "provider_default" + try: + value = resolve_temperature() + except ValueError: + return None, "invalid_environment" + if value is None: + return None, "provider_default" + return value, "environment" + + +def _requested_seed(raw: str) -> tuple[int | None, str]: + if not raw: + return None, "unset" + try: + value = resolve_seed() + except ValueError: + return None, "out_of_range" if raw.lstrip("+-").isdigit() else "invalid_environment" + if value is None: + return None, "unset" + return value, "environment" + + +def _requested_effort(raw: str) -> tuple[str | None, str]: + if not raw: + return None, "provider_default" + value = resolve_reasoning_effort() + value = _safe_setting(value, fallback="") + return (value, "environment") if value else (None, "invalid_environment") + + +def _control( + requested: float | int | str | None, + source: str, + *, + adapter_support: bool, +) -> dict[str, object]: + return { + "requested": requested, + "source": source, + # Client forwarding is confirmed later from provider-response telemetry. + # Configuration capture alone cannot prove that an analyzer made a call. + "forwarded_to_client": None, + "adapter_support": adapter_support, + # SkillSpector can prove what it passed to the client constructor, not + # whether a gateway/model honored the optional control. + "provider_support": "unknown", + } + + +def capture_llm_provenance(model_config: Mapping[str, object]) -> dict[str, object]: + """Capture resolved LLM configuration once, before analyzer execution.""" + configured_adapter = provider_name(get_active_provider()) + # This provider resolves model defaults before any client is constructed. + # Runtime response telemetry supplies the actual effective provider later. + resolved_adapter = provider_name(get_model_config_provider()) + package_version = version("skillspector") + + temperature, temperature_source = _requested_temperature( + os.environ.get("SKILLSPECTOR_TEMPERATURE", "").strip() + ) + seed, seed_source = _requested_seed(os.environ.get("SKILLSPECTOR_SEED", "").strip()) + effort, effort_source = _requested_effort( + os.environ.get("SKILLSPECTOR_REASONING_EFFORT", "").strip() + ) + + analyzers = [ + { + "analyzer_id": slot, + "model": _safe_label(model_config.get(slot), fallback="redacted"), + "model_source": "resolved_configuration", + "analyzer_revision": { + "value": package_version, + "source": "skillspector_package", + }, + } + for slot in LLM_ANALYZER_SLOTS + ] + sampling = { + "temperature": _control( + temperature, + temperature_source, + adapter_support=resolved_adapter in _TEMPERATURE_ADAPTERS, + ), + "seed": _control( + seed, + seed_source, + adapter_support=resolved_adapter in _SEED_ADAPTERS, + ), + "reasoning_effort": _control( + effort, + effort_source, + adapter_support=resolved_adapter in _REASONING_EFFORT_ADAPTERS, + ), + } + return { + "schema_version": LLM_PROVENANCE_SCHEMA_VERSION, + "provider": { + "configured_adapter": _safe_label(configured_adapter), + "resolved_adapter": _safe_label(resolved_adapter), + "routing": _capture_provider_routing(resolved_adapter), + "service": "unknown", + }, + "analyzers": analyzers, + "sampling": sampling, + } + + +def _sanitize_control( + name: str, + value: object, + *, + use_llm: bool, + effective_adapters: Sequence[str], +) -> dict[str, object]: + raw = value if isinstance(value, Mapping) else {} + source = raw.get("source") + source = source if isinstance(source, str) and source in _CONTROL_SOURCES else "unknown" + adapter_support = raw.get("adapter_support") + adapter_support = adapter_support if isinstance(adapter_support, bool) else False + if effective_adapters: + supported_adapters = ( + _TEMPERATURE_ADAPTERS + if name == "temperature" + else _SEED_ADAPTERS + if name == "seed" + else _REASONING_EFFORT_ADAPTERS + ) + adapter_support = all(adapter in supported_adapters for adapter in effective_adapters) + + raw_requested = raw.get("requested") + requested = raw_requested + if name == "temperature": + requested = ( + float(requested) + if isinstance(requested, (int, float)) + and not isinstance(requested, bool) + and math.isfinite(requested) + and 0 <= requested <= 1 + else None + ) + elif name == "seed": + requested = ( + requested + if isinstance(requested, int) + and not isinstance(requested, bool) + and _MIN_SAFE_SEED <= requested <= _MAX_SAFE_SEED + else None + ) + else: + requested = _safe_setting(requested, fallback="") or None + if source == "environment" and requested is None: + source = ( + "out_of_range" + if name == "seed" + and isinstance(raw_requested, int) + and not isinstance(raw_requested, bool) + else "invalid_environment" + ) + if source != "environment": + requested = None + + # A configured client value is only reportable after provider-response + # telemetry proves that an analyzer call reached an effective adapter. + forwarded = requested if effective_adapters and adapter_support else None + if name == "temperature": + forwarded = ( + float(forwarded) + if isinstance(forwarded, (int, float)) + and not isinstance(forwarded, bool) + and math.isfinite(forwarded) + and 0 <= forwarded <= 1 + else None + ) + elif name == "seed": + forwarded = ( + forwarded + if isinstance(forwarded, int) + and not isinstance(forwarded, bool) + and _MIN_SAFE_SEED <= forwarded <= _MAX_SAFE_SEED + else None + ) + else: + forwarded = _safe_setting(forwarded, fallback="") or None + if not use_llm or not adapter_support or source != "environment" or forwarded != requested: + forwarded = None + + return { + "requested": requested, + "source": source, + "forwarded_to_client": forwarded, + "adapter_support": adapter_support, + "provider_support": "unknown", + } + + +def _sanitize_provider_routing(value: object, *, resolved_adapter: str) -> dict[str, object]: + """Return the fixed routing contract without endpoints or credentials.""" + if resolved_adapter == "unknown": + return { + "deployment_override": None, + "deployment_source": "unknown", + "api_version": None, + "api_version_source": "unknown", + } + if resolved_adapter != "azure_openai": + return { + "deployment_override": None, + "deployment_source": "not_applicable", + "api_version": None, + "api_version_source": "not_applicable", + } + + raw = value if isinstance(value, Mapping) else {} + deployment = _safe_optional_label(raw.get("deployment_override")) + deployment_source = raw.get("deployment_source") + if not isinstance(deployment_source, str) or deployment_source not in _DEPLOYMENT_SOURCES: + deployment_source = "unknown" + if deployment_source == "environment" and deployment is None: + deployment_source = "unknown" + elif deployment_source != "environment": + deployment = None + + api_version = _safe_optional_label(raw.get("api_version")) + api_version_source = raw.get("api_version_source") + if not isinstance(api_version_source, str) or api_version_source not in _API_VERSION_SOURCES: + api_version_source = "unknown" + if api_version_source in {"environment", "provider_default"} and api_version is None: + api_version_source = "unknown" + elif api_version_source not in {"environment", "provider_default"}: + api_version = None + + return { + "deployment_override": deployment, + "deployment_source": deployment_source, + "api_version": api_version, + "api_version_source": api_version_source, + } + + +def _effective_adapters(inference_usage: object) -> list[str]: + """Return providers proven by sanitized provider-response telemetry.""" + records = inference_usage if isinstance(inference_usage, Sequence) else [] + adapters = { + adapter + for record in records + if isinstance(record, Mapping) + and record.get("usage_source") == "provider_response" + and (adapter := _safe_optional_label(record.get("provider"))) is not None + } + return sorted(adapters) + + +def sanitize_llm_provenance( + value: object, + *, + use_llm: bool, + inference_usage: object = None, +) -> dict[str, object]: + """Return the fixed public provenance projection without arbitrary state.""" + raw = value if isinstance(value, Mapping) else {} + raw_provider = raw.get("provider") + provider = raw_provider if isinstance(raw_provider, Mapping) else {} + configured_adapter = _safe_label(provider.get("configured_adapter")) + resolved_adapter = _safe_label(provider.get("resolved_adapter")) + effective_adapters = _effective_adapters(inference_usage) if use_llm else [] + effective_adapter = ( + "not_applicable" + if not use_llm + else effective_adapters[0] + if len(effective_adapters) == 1 + else "mixed" + if effective_adapters + else "unknown" + ) + + raw_analyzers = raw.get("analyzers") + by_id: dict[str, Mapping[object, object]] = {} + if isinstance(raw_analyzers, list): + for item in raw_analyzers: + if not isinstance(item, Mapping): + continue + analyzer_id = item.get("analyzer_id") + if isinstance(analyzer_id, str) and analyzer_id in LLM_ANALYZER_SLOTS: + by_id[analyzer_id] = item + analyzers: list[dict[str, object]] = [] + for slot in LLM_ANALYZER_SLOTS: + item = by_id.get(slot) + item = item if isinstance(item, Mapping) else {} + revision = item.get("analyzer_revision") + revision = revision if isinstance(revision, Mapping) else {} + analyzers.append( + { + "analyzer_id": slot, + "model": _safe_label(item.get("model"), fallback="redacted"), + "model_source": "resolved_configuration", + "analyzer_revision": { + "value": _safe_label(revision.get("value")), + "source": "skillspector_package", + }, + } + ) + + raw_sampling = raw.get("sampling") + sampling = raw_sampling if isinstance(raw_sampling, Mapping) else {} + sanitized_sampling = { + name: _sanitize_control( + name, + sampling.get(name), + use_llm=use_llm, + effective_adapters=effective_adapters, + ) + for name in ("temperature", "seed", "reasoning_effort") + } + requested_controls = [ + control for control in sanitized_sampling.values() if control["source"] == "environment" + ] + if not use_llm: + control_status = "not_applied" + elif any( + control["source"] in {"invalid_environment", "out_of_range"} + for control in sanitized_sampling.values() + ): + control_status = "invalid_configuration" + elif any(control["source"] == "unknown" for control in sanitized_sampling.values()): + control_status = "configuration_unknown" + elif not effective_adapters: + control_status = "controls_not_observed" + elif not requested_controls: + control_status = "provider_defaults" + elif all(control["forwarded_to_client"] is not None for control in requested_controls): + control_status = "best_effort_controls_forwarded" + else: + control_status = "controls_partially_forwarded" + + return { + "schema_version": LLM_PROVENANCE_SCHEMA_VERSION, + "provider": { + "configured_adapter": configured_adapter, + "resolved_adapter": resolved_adapter, + "effective_adapter": effective_adapter, + "effective_adapters": effective_adapters, + "service": "unknown", + "routing": _sanitize_provider_routing( + provider.get("routing"), resolved_adapter=resolved_adapter + ), + }, + "analyzers": analyzers, + "sampling": sanitized_sampling, + "determinism": { + "classification": "nondeterministic" if use_llm else "not_applicable", + "control_status": control_status, + "provider_guarantee": False, + "reason": ( + "Optional controls do not guarantee identical provider output." + if use_llm + else "LLM analysis was disabled for this scan." + ), + }, + } diff --git a/src/skillspector/nodes/build_context.py b/src/skillspector/nodes/build_context.py index cb377f739..41bdac7c7 100644 --- a/src/skillspector/nodes/build_context.py +++ b/src/skillspector/nodes/build_context.py @@ -60,6 +60,7 @@ LedgerRecordType, ledger_event, ) +from skillspector.llm_provenance import capture_llm_provenance from skillspector.logging_config import get_logger from skillspector.nested_artifacts import ( inspect_nested_artifacts, @@ -2217,6 +2218,7 @@ def _mark_runtime_partial(affected_paths: list[str], first_limited_path: str) -> bool(metadata.get("executable")) for metadata in nested.metadata ) + model_config = build_model_config() result: dict[str, object] = { "components": components, "llm_components": llm_components, @@ -2245,7 +2247,8 @@ def _mark_runtime_partial(affected_paths: list[str], first_limited_path: str) -> "python_ast_cache_key": python_ast_cache_key, "manifest": manifest, "previous_manifest": None, - "model_config": build_model_config(), + "model_config": model_config, + "llm_provenance": capture_llm_provenance(model_config), "component_metadata": component_metadata, "has_executable_scripts": has_executable_scripts, "workflow_resource_budget": workflow_budget, diff --git a/src/skillspector/nodes/report.py b/src/skillspector/nodes/report.py index 284cdde1e..91def82c2 100644 --- a/src/skillspector/nodes/report.py +++ b/src/skillspector/nodes/report.py @@ -38,6 +38,7 @@ from skillspector import __version__ as skillspector_version from skillspector.inference_usage import sanitize_inference_usage from skillspector.inspection_ledger import MAX_FINDING_OUTPUT_RECORDS, AnalysisCompleteness +from skillspector.llm_provenance import sanitize_llm_provenance from skillspector.llm_utils import is_llm_available from skillspector.logging_config import get_logger from skillspector.models import Finding @@ -1068,6 +1069,7 @@ def _build_metadata( use_llm: bool, llm_call_log: Sequence[Mapping[str, object]] | None = None, inference_usage: Sequence[Mapping[str, object]] | None = None, + llm_provenance: object = None, transitive_targets_scanned: int | None = None, transitive_bytes_scanned: int | None = None, transitive_truncation_reasons: Sequence[str] | None = None, @@ -1106,6 +1108,7 @@ def _build_metadata( # some coverage was lost) into one boolean. meta_analysis_applied = use_llm and provider_available and meta_analyzer_succeeded + sanitized_inference_usage = sanitize_inference_usage(inference_usage) meta: dict[str, object] = { "has_executable_scripts": has_executable_scripts, "skillspector_version": skillspector_version, @@ -1117,7 +1120,12 @@ def _build_metadata( # A list (including an empty list) makes observability explicit. Empty # means the provider/transport supplied no counters; it is never an # estimated zero-cost assertion. - "inference_usage": sanitize_inference_usage(inference_usage), + "inference_usage": sanitized_inference_usage, + "llm_provenance": sanitize_llm_provenance( + llm_provenance, + use_llm=use_llm, + inference_usage=sanitized_inference_usage, + ), } if not meta_analysis_applied: meta["filtering_mode"] = "heuristic" @@ -1159,6 +1167,7 @@ def _format_json( use_llm: bool = True, llm_call_log: Sequence[Mapping[str, object]] | None = None, inference_usage: Sequence[Mapping[str, object]] | None = None, + llm_provenance: object = None, analysis_completeness: Mapping[str, object] | None = None, suppressed: list[SuppressedFinding] | None = None, execution_successful: bool = True, @@ -1204,6 +1213,7 @@ def _format_json( use_llm, llm_call_log, inference_usage, + llm_provenance, transitive_targets_scanned, transitive_bytes_scanned, transitive_truncation_reasons, @@ -1457,6 +1467,7 @@ def report(state: SkillspectorState) -> dict[str, object]: use_llm = state.get("use_llm", True) llm_call_log = state.get("llm_call_log") or [] inference_usage = state.get("inference_usage") or [] + llm_provenance = state.get("llm_provenance") transitive_targets_scanned = state.get("transitive_targets_scanned") transitive_bytes_scanned = state.get("transitive_bytes_scanned") transitive_truncation_reasons = [ @@ -1578,6 +1589,7 @@ def report(state: SkillspectorState) -> dict[str, object]: use_llm=use_llm, llm_call_log=llm_call_log, inference_usage=inference_usage, + llm_provenance=llm_provenance, analysis_completeness=analysis_completeness, suppressed=suppressed, execution_successful=execution_successful, diff --git a/src/skillspector/providers/chat_models.py b/src/skillspector/providers/chat_models.py index aa13d44ae..eb25d3a9e 100644 --- a/src/skillspector/providers/chat_models.py +++ b/src/skillspector/providers/chat_models.py @@ -26,6 +26,8 @@ from pydantic import SecretStr logger = logging.getLogger(__name__) +MIN_SAMPLING_SEED = -(1 << 63) +MAX_SAMPLING_SEED = (1 << 63) - 1 def resolve_reasoning_effort() -> str | None: @@ -34,25 +36,44 @@ def resolve_reasoning_effort() -> str | None: return reasoning_effort or None +def resolve_temperature() -> float | None: + """Resolve the optional temperature using the provider validation contract.""" + raw_temperature = os.environ.get("SKILLSPECTOR_TEMPERATURE", "").strip() + if not raw_temperature: + return None + try: + temperature = float(raw_temperature) + except ValueError as exc: + raise ValueError("SKILLSPECTOR_TEMPERATURE must be a number between 0 and 1") from exc + if not 0 <= temperature <= 1: + raise ValueError("SKILLSPECTOR_TEMPERATURE must be between 0 and 1") + return temperature + + +def resolve_seed() -> int | None: + """Resolve the optional seed within the portable signed 64-bit contract.""" + raw_seed = os.environ.get("SKILLSPECTOR_SEED", "").strip() + if not raw_seed: + return None + try: + seed = int(raw_seed) + except ValueError as exc: + raise ValueError("SKILLSPECTOR_SEED must be an integer") from exc + if not MIN_SAMPLING_SEED <= seed <= MAX_SAMPLING_SEED: + raise ValueError("SKILLSPECTOR_SEED must be a signed 64-bit integer") + return seed + + def resolve_sampling_parameters(*, include_seed: bool = False) -> dict[str, float | int]: """Resolve optional, validated sampling controls for hosted providers.""" parameters: dict[str, float | int] = {} - raw_temperature = os.environ.get("SKILLSPECTOR_TEMPERATURE", "").strip() - if raw_temperature: - try: - temperature = float(raw_temperature) - except ValueError as exc: - raise ValueError("SKILLSPECTOR_TEMPERATURE must be a number between 0 and 1") from exc - if not 0 <= temperature <= 1: - raise ValueError("SKILLSPECTOR_TEMPERATURE must be between 0 and 1") + temperature = resolve_temperature() + if temperature is not None: parameters["temperature"] = temperature - raw_seed = os.environ.get("SKILLSPECTOR_SEED", "").strip() - if include_seed and raw_seed: - try: - parameters["seed"] = int(raw_seed) - except ValueError as exc: - raise ValueError("SKILLSPECTOR_SEED must be an integer") from exc + seed = resolve_seed() if include_seed else None + if seed is not None: + parameters["seed"] = seed return parameters diff --git a/src/skillspector/state.py b/src/skillspector/state.py index fc0f7215e..898127a1e 100644 --- a/src/skillspector/state.py +++ b/src/skillspector/state.py @@ -304,6 +304,9 @@ class SkillspectorState(TypedDict, total=False): # Model IDs per LLM-using node: e.g. {"default": "...", "meta_analyzer": "..."} model_config: dict[str, str] + # Sanitized LLM configuration captured with model resolution. The report + # projects this independently from provider token-usage telemetry. + llm_provenance: dict[str, object] # Component metadata for reporting and risk scoring (from build_context) component_metadata: list[dict[str, object]] diff --git a/tests/nodes/test_report.py b/tests/nodes/test_report.py index 59413b557..3c8e64b06 100644 --- a/tests/nodes/test_report.py +++ b/tests/nodes/test_report.py @@ -1491,6 +1491,93 @@ def test_json_report_exposes_only_sanitized_provider_usage( ] +def test_json_report_exposes_captured_llm_provenance( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("skillspector.nodes.report.is_llm_available", lambda: (True, None)) + state: SkillspectorState = { + "filtered_findings": [], + "component_metadata": [], + "has_executable_scripts": False, + "manifest": {}, + "output_format": "json", + "use_llm": True, + "llm_call_log": [], + "inference_usage": [ + { + "node": "semantic_developer_intent", + "request_kind": "structured_output", + "provider": "openai", + "model": "safe/model:1", + "model_source": "requested_model", + "usage_source": "provider_response", + "total_tokens": 1, + } + ], + "llm_provenance": { + "provider": { + "configured_adapter": "anthropic", + "resolved_adapter": "openai", + "service": "private-service-name", + }, + "analyzers": [ + { + "analyzer_id": "semantic_developer_intent", + "model": "safe/model:1", + "analyzer_revision": {"value": "2.11.2", "prompt": "must not leak"}, + } + ], + "sampling": { + "temperature": { + "requested": 0.0, + "source": "environment", + "forwarded_to_client": 0.0, + "adapter_support": True, + }, + "seed": { + "requested": 7, + "source": "environment", + "forwarded_to_client": 7, + "adapter_support": True, + }, + }, + "endpoint": "https://private.example.test", + }, + } + + meta = _meta_from_json_report(state) + provenance = meta["llm_provenance"] + + assert provenance["provider"] == { + "configured_adapter": "anthropic", + "resolved_adapter": "openai", + "effective_adapter": "openai", + "effective_adapters": ["openai"], + "service": "unknown", + "routing": { + "deployment_override": None, + "deployment_source": "not_applicable", + "api_version": None, + "api_version_source": "not_applicable", + }, + } + intent = next( + item + for item in provenance["analyzers"] + if item["analyzer_id"] == "semantic_developer_intent" + ) + assert intent["model"] == "safe/model:1" + assert intent["analyzer_revision"] == { + "value": "2.11.2", + "source": "skillspector_package", + } + assert provenance["sampling"]["temperature"]["forwarded_to_client"] == 0.0 + assert provenance["sampling"]["seed"]["forwarded_to_client"] == 7 + assert provenance["determinism"]["classification"] == "nondeterministic" + assert "private.example" not in json.dumps(meta) + assert "must not leak" not in json.dumps(meta) + + def test_report_no_llm_failures_not_counted_as_degraded(monkeypatch: pytest.MonkeyPatch) -> None: """use_llm False -> failures (if any) never mark the scan degraded.""" monkeypatch.setattr("skillspector.nodes.report.is_llm_available", lambda: (True, None)) diff --git a/tests/unit/test_llm_provenance.py b/tests/unit/test_llm_provenance.py new file mode 100644 index 000000000..9993bac34 --- /dev/null +++ b/tests/unit/test_llm_provenance.py @@ -0,0 +1,436 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for sanitized, scan-level LLM provenance.""" + +from __future__ import annotations + +import json + +import pytest + +from skillspector.inference_usage import provider_name +from skillspector.llm_provenance import ( + LLM_ANALYZER_SLOTS, + capture_llm_provenance, + sanitize_llm_provenance, +) + + +class OpenAIProvider: + pass + + +class AnthropicProvider: + pass + + +class AzureOpenAIProvider: + pass + + +def _models(value: str = "safe/model:1") -> dict[str, str]: + return dict.fromkeys(LLM_ANALYZER_SLOTS, value) + + +def _usage(provider: str) -> list[dict[str, object]]: + return [{"provider": provider, "usage_source": "provider_response"}] + + +def test_capture_records_resolved_adapters_models_and_forwarded_controls( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "skillspector.llm_provenance.get_active_provider", lambda: AnthropicProvider() + ) + monkeypatch.setattr( + "skillspector.llm_provenance.get_model_config_provider", lambda: OpenAIProvider() + ) + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0") + monkeypatch.setenv("SKILLSPECTOR_SEED", "0") + monkeypatch.setenv("SKILLSPECTOR_REASONING_EFFORT", "low") + + captured = capture_llm_provenance(_models()) + result = sanitize_llm_provenance(captured, use_llm=True, inference_usage=_usage("openai")) + + assert result["provider"] == { + "configured_adapter": "anthropic", + "resolved_adapter": "openai", + "effective_adapter": "openai", + "effective_adapters": ["openai"], + "service": "unknown", + "routing": { + "deployment_override": None, + "deployment_source": "not_applicable", + "api_version": None, + "api_version_source": "not_applicable", + }, + } + assert [item["analyzer_id"] for item in result["analyzers"]] == list(LLM_ANALYZER_SLOTS) + assert {item["model"] for item in result["analyzers"]} == {"safe/model:1"} + assert {item["analyzer_revision"]["source"] for item in result["analyzers"]} == { + "skillspector_package" + } + assert result["sampling"]["temperature"]["forwarded_to_client"] == 0.0 + assert result["sampling"]["seed"]["forwarded_to_client"] == 0 + assert result["sampling"]["reasoning_effort"]["forwarded_to_client"] == "low" + assert result["determinism"] == { + "classification": "nondeterministic", + "control_status": "best_effort_controls_forwarded", + "provider_guarantee": False, + "reason": "Optional controls do not guarantee identical provider output.", + } + + +def test_seed_is_requested_but_not_claimed_forwarded_for_anthropic( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "skillspector.llm_provenance.get_active_provider", lambda: AnthropicProvider() + ) + monkeypatch.setattr( + "skillspector.llm_provenance.get_model_config_provider", lambda: AnthropicProvider() + ) + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0.1") + monkeypatch.setenv("SKILLSPECTOR_SEED", "7") + + result = sanitize_llm_provenance( + capture_llm_provenance(_models()), + use_llm=True, + inference_usage=_usage("anthropic"), + ) + + assert result["sampling"]["temperature"]["forwarded_to_client"] == 0.1 + assert result["sampling"]["seed"] == { + "requested": 7, + "source": "environment", + "forwarded_to_client": None, + "adapter_support": False, + "provider_support": "unknown", + } + assert result["determinism"]["control_status"] == "controls_partially_forwarded" + + +def test_capture_is_stable_if_environment_changes_before_report( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("skillspector.llm_provenance.get_active_provider", lambda: OpenAIProvider()) + monkeypatch.setattr( + "skillspector.llm_provenance.get_model_config_provider", lambda: OpenAIProvider() + ) + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0.2") + monkeypatch.setenv("SKILLSPECTOR_SEED", "11") + captured = capture_llm_provenance(_models()) + + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0.9") + monkeypatch.setenv("SKILLSPECTOR_SEED", "99") + result = sanitize_llm_provenance( + captured, + use_llm=True, + inference_usage=_usage("openai"), + ) + + assert result["sampling"]["temperature"]["requested"] == 0.2 + assert result["sampling"]["seed"]["requested"] == 11 + + +def test_configured_controls_are_not_claimed_forwarded_without_response_evidence( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("skillspector.llm_provenance.get_active_provider", lambda: OpenAIProvider()) + monkeypatch.setattr( + "skillspector.llm_provenance.get_model_config_provider", lambda: OpenAIProvider() + ) + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0.2") + monkeypatch.setenv("SKILLSPECTOR_SEED", "11") + + result = sanitize_llm_provenance(capture_llm_provenance(_models()), use_llm=True) + + assert result["provider"]["effective_adapter"] == "unknown" + assert all(control["forwarded_to_client"] is None for control in result["sampling"].values()) + assert result["determinism"]["control_status"] == "controls_not_observed" + + +def test_sanitizer_drops_unverified_forwarding_without_response_evidence() -> None: + result = sanitize_llm_provenance( + { + "sampling": { + "temperature": { + "requested": 0.2, + "source": "environment", + "forwarded_to_client": 0.2, + "adapter_support": True, + }, + "seed": { + "requested": None, + "source": "unset", + "forwarded_to_client": None, + "adapter_support": True, + }, + "reasoning_effort": { + "requested": None, + "source": "provider_default", + "forwarded_to_client": None, + "adapter_support": True, + }, + } + }, + use_llm=True, + ) + + assert result["sampling"]["temperature"]["forwarded_to_client"] is None + assert result["determinism"]["control_status"] == "controls_not_observed" + + +def test_static_scan_is_explicitly_not_applicable(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("skillspector.llm_provenance.get_active_provider", lambda: OpenAIProvider()) + monkeypatch.setattr( + "skillspector.llm_provenance.get_model_config_provider", lambda: OpenAIProvider() + ) + + result = sanitize_llm_provenance(capture_llm_provenance(_models()), use_llm=False) + + assert result["determinism"]["classification"] == "not_applicable" + assert result["determinism"]["control_status"] == "not_applied" + assert result["provider"]["effective_adapter"] == "not_applicable" + assert all(control["forwarded_to_client"] is None for control in result["sampling"].values()) + + +def test_public_projection_drops_unknown_fields_and_redacts_unsafe_labels() -> None: + malicious = { + "provider": { + "configured_adapter": "https://user:secret@provider.test", + "effective_adapter": "sk-secret-value", + "endpoint": "https://private.example.test", + "routing": { + "deployment_override": "sk-secret-deployment", + "deployment_source": "environment", + "api_version": "https://private.example.test/version", + "api_version_source": "environment", + }, + }, + "analyzers": [ + { + "analyzer_id": LLM_ANALYZER_SLOTS[0], + "model": "nvapi-secret-value", + "analyzer_revision": {"value": "2.11.2", "prompt": "private prompt"}, + "credentials": "do-not-emit", + }, + {"analyzer_id": "unknown", "model": "private/model"}, + ], + "sampling": { + "temperature": { + "requested": float("nan"), + "source": "environment", + "forwarded_to_client": float("inf"), + "adapter_support": True, + "headers": {"authorization": "secret"}, + } + }, + "raw_prompt": "do-not-emit", + } + + result = sanitize_llm_provenance(malicious, use_llm=True) + serialized = json.dumps(result) + + assert result["provider"]["configured_adapter"] == "unknown" + assert result["provider"]["resolved_adapter"] == "unknown" + assert result["provider"]["effective_adapter"] == "unknown" + assert result["provider"]["routing"] == { + "deployment_override": None, + "deployment_source": "unknown", + "api_version": None, + "api_version_source": "unknown", + } + assert result["analyzers"][0]["model"] == "redacted" + assert len(result["analyzers"]) == len(LLM_ANALYZER_SLOTS) + assert "secret" not in serialized + assert "private.example" not in serialized + assert "do-not-emit" not in serialized + assert "private prompt" not in serialized + + +def test_multiple_response_providers_are_reported_as_mixed() -> None: + result = sanitize_llm_provenance( + None, + use_llm=True, + inference_usage=[*_usage("openai"), *_usage("anthropic")], + ) + + assert result["provider"]["effective_adapter"] == "mixed" + assert result["provider"]["effective_adapters"] == ["anthropic", "openai"] + + +def test_public_projection_repairs_inconsistent_control_shapes() -> None: + result = sanitize_llm_provenance( + { + "sampling": { + "temperature": { + "requested": 999, + "source": "environment", + "forwarded_to_client": 999, + "adapter_support": True, + }, + "seed": { + "requested": 1.5, + "source": "environment", + "forwarded_to_client": 1.5, + "adapter_support": True, + }, + "reasoning_effort": { + "requested": "low", + "source": "environment", + "forwarded_to_client": "high", + "adapter_support": True, + }, + } + }, + use_llm=True, + ) + + assert result["sampling"]["temperature"]["requested"] is None + assert result["sampling"]["temperature"]["source"] == "invalid_environment" + assert result["sampling"]["temperature"]["forwarded_to_client"] is None + assert result["sampling"]["seed"]["requested"] is None + assert result["sampling"]["seed"]["source"] == "invalid_environment" + assert result["sampling"]["seed"]["forwarded_to_client"] is None + assert result["sampling"]["reasoning_effort"]["requested"] == "low" + assert result["sampling"]["reasoning_effort"]["forwarded_to_client"] is None + assert result["determinism"]["control_status"] == "invalid_configuration" + + +def test_public_projection_ignores_unhashable_analyzer_ids() -> None: + result = sanitize_llm_provenance( + {"analyzers": [{"analyzer_id": {}, "model": "private/model"}]}, + use_llm=True, + ) + + assert len(result["analyzers"]) == len(LLM_ANALYZER_SLOTS) + assert {item["model"] for item in result["analyzers"]} == {"redacted"} + + +def test_provider_specific_reasoning_effort_is_recorded_exactly( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("skillspector.llm_provenance.get_active_provider", lambda: OpenAIProvider()) + monkeypatch.setattr( + "skillspector.llm_provenance.get_model_config_provider", lambda: OpenAIProvider() + ) + monkeypatch.setenv("SKILLSPECTOR_REASONING_EFFORT", "provider specific value") + + result = sanitize_llm_provenance( + capture_llm_provenance(_models()), + use_llm=True, + inference_usage=_usage("openai"), + ) + + assert result["sampling"]["reasoning_effort"]["requested"] == "provider specific value" + assert ( + result["sampling"]["reasoning_effort"]["forwarded_to_client"] == "provider specific value" + ) + + +def test_invalid_and_unknown_controls_are_not_called_provider_defaults( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("skillspector.llm_provenance.get_active_provider", lambda: OpenAIProvider()) + monkeypatch.setattr( + "skillspector.llm_provenance.get_model_config_provider", lambda: OpenAIProvider() + ) + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "warm") + + invalid = sanitize_llm_provenance(capture_llm_provenance(_models()), use_llm=True) + unknown = sanitize_llm_provenance(None, use_llm=True) + + assert invalid["determinism"]["control_status"] == "invalid_configuration" + assert unknown["determinism"]["control_status"] == "configuration_unknown" + + +def test_effective_provider_comes_from_response_not_preflight_candidate( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A Bedrock-to-OpenAI fallback must report the provider that answered.""" + bedrock_provider = type("BedrockProvider", (), {})() + monkeypatch.setattr("skillspector.llm_provenance.get_active_provider", lambda: bedrock_provider) + monkeypatch.setattr( + "skillspector.llm_provenance.get_model_config_provider", lambda: bedrock_provider + ) + monkeypatch.setenv("SKILLSPECTOR_SEED", "17") + + result = sanitize_llm_provenance( + capture_llm_provenance(_models()), + use_llm=True, + inference_usage=_usage("openai"), + ) + + assert result["provider"]["resolved_adapter"] == "bedrock" + assert result["provider"]["effective_adapter"] == "openai" + assert result["provider"]["effective_adapters"] == ["openai"] + assert result["sampling"]["seed"]["adapter_support"] is True + assert result["sampling"]["seed"]["forwarded_to_client"] == 17 + + +def test_azure_routing_records_deployment_and_api_version( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "skillspector.llm_provenance.get_active_provider", lambda: AzureOpenAIProvider() + ) + monkeypatch.setattr( + "skillspector.llm_provenance.get_model_config_provider", lambda: AzureOpenAIProvider() + ) + monkeypatch.setenv("AZURE_OPENAI_DEPLOYMENT", "production-v2") + monkeypatch.setenv("AZURE_OPENAI_API_VERSION", "2025-01-01") + + result = sanitize_llm_provenance( + capture_llm_provenance(_models("gpt-4o")), + use_llm=True, + inference_usage=_usage("azure_openai"), + ) + + assert result["analyzers"][0]["model"] == "gpt-4o" + assert result["provider"]["routing"] == { + "deployment_override": "production-v2", + "deployment_source": "environment", + "api_version": "2025-01-01", + "api_version_source": "environment", + } + + +def test_azure_routing_uses_model_and_default_api_version_without_overrides( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "skillspector.llm_provenance.get_active_provider", lambda: AzureOpenAIProvider() + ) + monkeypatch.setattr( + "skillspector.llm_provenance.get_model_config_provider", lambda: AzureOpenAIProvider() + ) + + result = sanitize_llm_provenance( + capture_llm_provenance(_models("gpt-4o")), + use_llm=True, + inference_usage=_usage("azure_openai"), + ) + + assert result["provider"]["routing"] == { + "deployment_override": None, + "deployment_source": "resolved_model", + "api_version": "2024-06-01", + "api_version_source": "provider_default", + } + + +@pytest.mark.parametrize( + ("class_name", "expected"), + [ + ("AzureOpenAIProvider", "azure_openai"), + ("OllamaProvider", "ollama"), + ("OpenAICompatibleProvider", "openai_compatible"), + ("AntigravityCLIProvider", "antigravity_cli"), + ], +) +def test_builtin_provider_names_are_canonical(class_name: str, expected: str) -> None: + provider_type = type(class_name, (), {}) + + assert provider_name(provider_type()) == expected diff --git a/tests/unit/test_providers.py b/tests/unit/test_providers.py index 6aafcff77..237d99d81 100644 --- a/tests/unit/test_providers.py +++ b/tests/unit/test_providers.py @@ -591,12 +591,37 @@ def fake_chat_openai(**kwargs: object) -> dict[str, object]: assert captured["temperature"] == 0.25 assert captured["seed"] == 42 + @pytest.mark.parametrize("seed", [-(1 << 63), (1 << 63) - 1]) + def test_signed_64_bit_seed_boundaries_are_forwarded( + self, + monkeypatch: pytest.MonkeyPatch, + seed: int, + ) -> None: + captured: dict[str, object] = {} + + def fake_chat_openai(**kwargs: object) -> dict[str, object]: + captured.update(kwargs) + return kwargs + + monkeypatch.setattr(chat_models, "ChatOpenAI", fake_chat_openai) + monkeypatch.setenv("SKILLSPECTOR_SEED", str(seed)) + + create_openai_compatible_chat_model( + model="gpt-5.4", + credentials=("sk-x", "http://localhost:1234/v1"), + max_tokens=123, + ) + + assert captured["seed"] == seed + @pytest.mark.parametrize( ("name", "value", "message"), [ ("SKILLSPECTOR_TEMPERATURE", "warm", "must be a number"), ("SKILLSPECTOR_TEMPERATURE", "1.1", "must be between 0 and 1"), ("SKILLSPECTOR_SEED", "4.2", "must be an integer"), + ("SKILLSPECTOR_SEED", str(1 << 63), "must be a signed 64-bit integer"), + ("SKILLSPECTOR_SEED", str(-(1 << 63) - 1), "must be a signed 64-bit integer"), ], ) def test_invalid_sampling_control_fails_before_model_construction( From c9f4a0c4207f0b2c7f5da5d1d33294c524995207 Mon Sep 17 00:00:00 2001 From: Narendran Raghavan Date: Thu, 17 Sep 2026 13:34:45 -0700 Subject: [PATCH 2/3] fix(report): harden provenance evidence Signed-off-by: Narendran Raghavan --- src/skillspector/inference_usage.py | 137 ++++++++++-- src/skillspector/llm_analyzer_base.py | 11 +- src/skillspector/llm_provenance.py | 185 +++++++++++----- src/skillspector/llm_utils.py | 12 +- src/skillspector/nodes/report.py | 3 +- .../providers/anthropic/provider.py | 11 +- .../providers/anthropic_proxy/provider.py | 11 +- .../providers/azure_openai/provider.py | 8 +- .../providers/bedrock/provider.py | 8 +- src/skillspector/providers/chat_models.py | 11 +- tests/nodes/test_report.py | 30 ++- tests/unit/test_inference_usage.py | 28 +++ tests/unit/test_llm_provenance.py | 207 +++++++++++++++++- 13 files changed, 569 insertions(+), 93 deletions(-) diff --git a/src/skillspector/inference_usage.py b/src/skillspector/inference_usage.py index 8a8d50f05..91dd7f92e 100644 --- a/src/skillspector/inference_usage.py +++ b/src/skillspector/inference_usage.py @@ -33,15 +33,51 @@ _MAX_SAMPLING_SEED = (1 << 63) - 1 _FORWARDED_CONTROL_NAMES = ("temperature", "seed", "reasoning_effort") _SAFE_SETTING_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9 ._:/+\-]{0,255}") -_SECRET_PREFIXES = ("sk-", "nvapi-", "ghp_", "glpat-", "bearer-") +_CREDENTIAL_PREFIXES = ( + "sk-", + "nvapi-", + "ghp_", + "gho_", + "ghu_", + "ghs_", + "ghr_", + "github_pat_", + "glpat-", + "bearer-", + "xoxb-", + "xoxp-", + "xoxa-", + "xoxr-", + "hf_", + "aiza", + "akia", + "asia", + "aws-secret-", +) +_UNPREFIXED_CREDENTIAL = re.compile(r"(?:[0-9a-fA-F]{32,64}|[A-Za-z0-9_+/=-]{40,88})\Z") _CHAT_MODEL_CONTROLS: dict[ int, - tuple[weakref.ReferenceType[object], dict[str, float | int | str | None]], + tuple[ + weakref.ReferenceType[object], + dict[str, float | int | str | None], + dict[str, float | int | str | None], + ], ] = {} _CHAT_MODEL_CONTROLS_LOCK = threading.Lock() +def looks_like_credential(value: object) -> bool: + """Return whether a printable label resembles a common secret value.""" + if not isinstance(value, str): + return False + candidate = value.strip() + lowered = candidate.lower() + return lowered.startswith(_CREDENTIAL_PREFIXES) or bool( + _UNPREFIXED_CREDENTIAL.fullmatch(candidate) + ) + + class InferenceUsageRecord(TypedDict): """One provider-reported inference request, safe to serialize.""" @@ -60,6 +96,7 @@ class InferenceUsageRecord(TypedDict): # Internal-only construction evidence. ``sanitize_inference_usage`` never # includes this field in the public token-usage projection; the provenance # sanitizer consumes it separately after a provider response is observed. + requested_controls: NotRequired[dict[str, float | int | str | None]] forwarded_controls: NotRequired[dict[str, float | int | str | None]] @@ -85,12 +122,11 @@ def _forwarded_controls(value: Mapping[str, object] | None) -> dict[str, float | controls[name] = raw elif isinstance(raw, str): setting = raw.strip() - lowered = setting.lower() if ( _SAFE_SETTING_RE.fullmatch(setting) and "://" not in setting and "@" not in setting - and not lowered.startswith(_SECRET_PREFIXES) + and not looks_like_credential(setting) ): controls[name] = setting return controls @@ -98,11 +134,14 @@ def _forwarded_controls(value: Mapping[str, object] | None) -> dict[str, float | def register_chat_model_controls( chat_model: object, - controls: Mapping[str, object], + forwarded_controls: Mapping[str, object], + *, + requested_controls: Mapping[str, object] | None = None, ) -> None: - """Associate a constructed chat model with the controls passed to its client.""" + """Associate a model with requested and normalized request controls.""" model_id = id(chat_model) - sanitized = _forwarded_controls(controls) + sanitized_requested = _forwarded_controls(requested_controls) + sanitized_forwarded = _forwarded_controls(forwarded_controls) def _discard(model_ref: weakref.ReferenceType[object]) -> None: with _CHAT_MODEL_CONTROLS_LOCK: @@ -115,11 +154,28 @@ def _discard(model_ref: weakref.ReferenceType[object]) -> None: except TypeError: return with _CHAT_MODEL_CONTROLS_LOCK: - _CHAT_MODEL_CONTROLS[model_id] = (model_ref, sanitized) + _CHAT_MODEL_CONTROLS[model_id] = ( + model_ref, + sanitized_requested, + sanitized_forwarded, + ) def chat_model_controls(chat_model: object | None) -> dict[str, float | int | str | None]: """Return detached construction evidence for *chat_model*, when recorded.""" + if chat_model is None: + return {} + with _CHAT_MODEL_CONTROLS_LOCK: + current = _CHAT_MODEL_CONTROLS.get(id(chat_model)) + if current is None or current[0]() is not chat_model: + return {} + return current[2].copy() + + +def chat_model_requested_controls( + chat_model: object | None, +) -> dict[str, float | int | str | None]: + """Return the controls resolved when *chat_model* was constructed.""" if chat_model is None: return {} with _CHAT_MODEL_CONTROLS_LOCK: @@ -129,6 +185,44 @@ def chat_model_controls(chat_model: object | None) -> dict[str, float | int | st return current[1].copy() +def retained_chat_model_controls( + chat_model: object, + names: Sequence[str], +) -> dict[str, float | int | str | None]: + """Return controls retained by the normalized provider request payload. + + LangChain may accept a constructor option and then remove it for a + provider/model combination. Provenance must describe the request that the + adapter will send, not the raw constructor arguments supplied before that + normalization. + """ + selected = [name for name in names if name in _FORWARDED_CONTROL_NAMES] + controls: dict[str, object] = dict.fromkeys(selected) + payload: Mapping[str, object] | None = None + payload_builder = getattr(chat_model, "_get_request_payload", None) + if callable(payload_builder): + try: + candidate = payload_builder("SkillSpector provenance probe") + except (TypeError, ValueError): + candidate = None + if isinstance(candidate, Mapping): + payload = candidate + + if payload is not None: + output_config = payload.get("output_config") + output_config = output_config if isinstance(output_config, Mapping) else {} + for name in selected: + if name == "reasoning_effort": + controls[name] = payload.get(name, output_config.get("effort")) + else: + controls[name] = payload.get(name) + else: + for name in selected: + attribute = "effort" if name == "reasoning_effort" else name + controls[name] = getattr(chat_model, name, getattr(chat_model, attribute, None)) + return _forwarded_controls(controls) + + def _mapping(value: object) -> Mapping[str, object]: return value if isinstance(value, Mapping) else {} @@ -180,7 +274,12 @@ def _strict_label(value: object) -> str | None: def _strict_model_label(value: object) -> str | None: """Return a model label only when it cannot encode a URL or userinfo.""" candidate = _strict_label(value) - if candidate is None or "://" in candidate or "@" in candidate: + if ( + candidate is None + or "://" in candidate + or "@" in candidate + or looks_like_credential(candidate) + ): return None return candidate @@ -383,12 +482,14 @@ def __init__( request_kind: str, provider: str, requested_model: str, + requested_controls: Mapping[str, object] | None = None, forwarded_controls: Mapping[str, object] | None = None, ) -> None: self._node = node self._request_kind = request_kind self._provider = provider self._requested_model = requested_model + self._requested_controls = _forwarded_controls(requested_controls) self._forwarded_controls = _forwarded_controls(forwarded_controls) self._records: list[InferenceUsageRecord] = [] self._response_received = False @@ -416,6 +517,8 @@ def on_llm_end(self, response: LLMResult, **kwargs: object) -> None: with self._lock: self._response_received = True if record is not None: + if self._requested_controls: + record["requested_controls"] = self._requested_controls.copy() if self._forwarded_controls: record["forwarded_controls"] = self._forwarded_controls.copy() self._records.append(record) @@ -431,6 +534,7 @@ def _response_observation(self) -> InferenceUsageRecord: "model": _model_label(self._requested_model), "model_source": "requested_model", "usage_source": "provider_response", + "requested_controls": self._requested_controls.copy(), "forwarded_controls": self._forwarded_controls.copy(), } @@ -448,11 +552,15 @@ def set_provider(self, provider: str) -> None: raise RuntimeError("cannot change inference provider after a response") self._provider = label - def set_forwarded_controls(self, controls: Mapping[str, object] | None) -> None: - """Update construction evidence before the next provider response.""" - sanitized = _forwarded_controls(controls) + def set_controls( + self, + requested: Mapping[str, object] | None, + forwarded: Mapping[str, object] | None, + ) -> None: + """Update constructor/request evidence before the next response.""" with self._lock: - self._forwarded_controls = sanitized + self._requested_controls = _forwarded_controls(requested) + self._forwarded_controls = _forwarded_controls(forwarded) @property def response_received(self) -> bool: @@ -469,6 +577,9 @@ def snapshot(self) -> list[InferenceUsageRecord]: controls = record.get("forwarded_controls") if isinstance(controls, dict): detached["forwarded_controls"] = controls.copy() + requested = record.get("requested_controls") + if isinstance(requested, dict): + detached["requested_controls"] = requested.copy() snapshot.append(detached) return snapshot diff --git a/src/skillspector/llm_analyzer_base.py b/src/skillspector/llm_analyzer_base.py index 141598f3c..ad8f4afd7 100644 --- a/src/skillspector/llm_analyzer_base.py +++ b/src/skillspector/llm_analyzer_base.py @@ -41,7 +41,11 @@ from langchain_openai import ChatOpenAI from pydantic import BaseModel, Field, ValidationError, field_validator -from skillspector.inference_usage import InferenceUsageRecord, chat_model_controls +from skillspector.inference_usage import ( + InferenceUsageRecord, + chat_model_controls, + chat_model_requested_controls, +) from skillspector.inspection_ledger import ( AnalyzerStatusEvent, InspectionLedgerEvent, @@ -728,7 +732,10 @@ def _model_for_call(self) -> tuple[object, object | None]: effective_provider = chat_model_provider_name(llm) if effective_provider is not None: self._usage_collector.set_provider(effective_provider) - self._usage_collector.set_forwarded_controls(chat_model_controls(llm)) + self._usage_collector.set_controls( + chat_model_requested_controls(llm), + chat_model_controls(llm), + ) structured = ( llm.with_structured_output(self.response_schema) if self.response_schema else None ) diff --git a/src/skillspector/llm_provenance.py b/src/skillspector/llm_provenance.py index 69696faef..e0fef908a 100644 --- a/src/skillspector/llm_provenance.py +++ b/src/skillspector/llm_provenance.py @@ -5,13 +5,14 @@ from __future__ import annotations +import json import math import os import re from collections.abc import Mapping, Sequence -from importlib.metadata import version +from importlib.metadata import distribution, version -from skillspector.inference_usage import provider_name +from skillspector.inference_usage import looks_like_credential, provider_name from skillspector.providers import get_active_provider, get_model_config_provider from skillspector.providers.chat_models import ( MAX_SAMPLING_SEED, @@ -37,17 +38,21 @@ "azure_openai", "bedrock", "nv_build", + "nv_inference", "ollama", "openai", "openai_compatible", } ) -_SEED_ADAPTERS = frozenset({"azure_openai", "nv_build", "ollama", "openai", "openai_compatible"}) +_SEED_ADAPTERS = frozenset( + {"azure_openai", "nv_build", "nv_inference", "ollama", "openai", "openai_compatible"} +) _REASONING_EFFORT_ADAPTERS = frozenset( { "anthropic", "anthropic_proxy", "nv_build", + "nv_inference", "ollama", "openai", "openai_compatible", @@ -55,7 +60,6 @@ ) _SAFE_LABEL = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/+\-]{0,255}") _SAFE_SETTING = re.compile(r"[A-Za-z0-9][A-Za-z0-9 ._:/+\-]{0,255}") -_SECRET_PREFIXES = ("sk-", "nvapi-", "ghp_", "glpat-", "bearer-") _MAX_SAFE_SEED = MAX_SAMPLING_SEED _MIN_SAFE_SEED = MIN_SAMPLING_SEED _CONTROL_SOURCES = frozenset( @@ -64,17 +68,18 @@ _DEPLOYMENT_SOURCES = frozenset({"environment", "resolved_model", "not_applicable", "unknown"}) _API_VERSION_SOURCES = frozenset({"environment", "provider_default", "not_applicable", "unknown"}) _AZURE_OPENAI_DEFAULT_API_VERSION = "2024-06-01" +_SOURCE_REVISION = re.compile(r"[0-9a-fA-F]{7,64}") +_SOURCE_REVISION_SOURCES = frozenset({"build_environment", "package_vcs_metadata", "unknown"}) def _safe_label(value: object, fallback: str = "unknown") -> str: candidate = value if isinstance(value, str) else "" candidate = candidate.strip() - lowered = candidate.lower() if ( _SAFE_LABEL.fullmatch(candidate) and "://" not in candidate and "@" not in candidate - and not lowered.startswith(_SECRET_PREFIXES) + and not looks_like_credential(candidate) ): return candidate return fallback @@ -84,12 +89,11 @@ def _safe_setting(value: object, fallback: str = "unknown") -> str: """Return a bounded printable setting while allowing provider-specific spaces.""" candidate = value if isinstance(value, str) else "" candidate = candidate.strip() - lowered = candidate.lower() if ( _SAFE_SETTING.fullmatch(candidate) and "://" not in candidate and "@" not in candidate - and not lowered.startswith(_SECRET_PREFIXES) + and not looks_like_credential(candidate) ): return candidate return fallback @@ -101,6 +105,25 @@ def _safe_optional_label(value: object) -> str | None: return label or None +def _capture_source_revision() -> tuple[str, str]: + """Return an injected/packaged VCS identity without invoking Git.""" + injected = os.environ.get("SKILLSPECTOR_BUILD_REVISION", "").strip() + if _SOURCE_REVISION.fullmatch(injected): + return injected.lower(), "build_environment" + + try: + direct_url = distribution("skillspector").read_text("direct_url.json") + metadata = json.loads(direct_url) if direct_url else {} + except (json.JSONDecodeError, OSError, TypeError): + metadata = {} + vcs_info = metadata.get("vcs_info") if isinstance(metadata, Mapping) else None + vcs_info = vcs_info if isinstance(vcs_info, Mapping) else {} + packaged = vcs_info.get("commit_id") + if isinstance(packaged, str) and _SOURCE_REVISION.fullmatch(packaged.strip()): + return packaged.strip().lower(), "package_vcs_metadata" + return "unknown", "unknown" + + def _capture_provider_routing(resolved_adapter: str) -> dict[str, object]: """Capture non-secret provider routing inputs used by client construction.""" if resolved_adapter != "azure_openai": @@ -189,6 +212,7 @@ def capture_llm_provenance(model_config: Mapping[str, object]) -> dict[str, obje # Runtime response telemetry supplies the actual effective provider later. resolved_adapter = provider_name(get_model_config_provider()) package_version = version("skillspector") + source_revision, source_revision_source = _capture_source_revision() temperature, temperature_source = _requested_temperature( os.environ.get("SKILLSPECTOR_TEMPERATURE", "").strip() @@ -206,6 +230,10 @@ def capture_llm_provenance(model_config: Mapping[str, object]) -> dict[str, obje "analyzer_revision": { "value": package_version, "source": "skillspector_package", + "source_revision": { + "value": source_revision, + "source": source_revision_source, + }, }, } for slot in LLM_ANALYZER_SLOTS @@ -240,13 +268,51 @@ def capture_llm_provenance(model_config: Mapping[str, object]) -> dict[str, obje } +def _sanitize_observed_control_values( + name: str, + values: Sequence[object], +) -> tuple[list[float | int | str | None], bool]: + observed: list[float | int | str | None] = [] + invalid = False + for candidate in values: + if candidate is None: + observed.append(None) + elif name == "temperature": + if ( + isinstance(candidate, (int, float)) + and not isinstance(candidate, bool) + and math.isfinite(candidate) + and 0 <= candidate <= 1 + ): + observed.append(float(candidate)) + else: + invalid = True + elif name == "seed": + if ( + isinstance(candidate, int) + and not isinstance(candidate, bool) + and _MIN_SAFE_SEED <= candidate <= _MAX_SAFE_SEED + ): + observed.append(candidate) + else: + invalid = True + else: + setting = _safe_setting(candidate, fallback="") + if setting: + observed.append(setting) + else: + invalid = True + return observed, invalid + + def _sanitize_control( name: str, value: object, *, use_llm: bool, effective_adapters: Sequence[str], - observed_values: Sequence[object], + observed_requested_values: Sequence[object], + observed_forwarded_values: Sequence[object], ) -> dict[str, object]: raw = value if isinstance(value, Mapping) else {} source = raw.get("source") @@ -295,47 +361,23 @@ def _sanitize_control( if source != "environment": requested = None - # Provider-response observations carry the controls recorded from the - # actual client constructor. They supersede the earlier configuration - # capture, which is necessarily provisional until a client is built. - observed: list[float | int | str | None] = [] - invalid_observation = False - for candidate in observed_values: - if candidate is None: - observed.append(None) - elif name == "temperature": - if ( - isinstance(candidate, (int, float)) - and not isinstance(candidate, bool) - and math.isfinite(candidate) - and 0 <= candidate <= 1 - ): - observed.append(float(candidate)) - else: - invalid_observation = True - elif name == "seed": - if ( - isinstance(candidate, int) - and not isinstance(candidate, bool) - and _MIN_SAFE_SEED <= candidate <= _MAX_SAFE_SEED - ): - observed.append(candidate) - else: - invalid_observation = True - else: - setting = _safe_setting(candidate, fallback="") - if setting: - observed.append(setting) - else: - invalid_observation = True - + # The constructor-time request supersedes the provisional preflight + # capture. Forwarding is derived separately from the normalized provider + # payload because a model adapter may accept and then omit a control. forwarded: float | int | str | None = None - if use_llm and adapter_support and observed_values: + if use_llm and adapter_support and observed_requested_values: + observed, invalid_observation = _sanitize_observed_control_values( + name, observed_requested_values + ) distinct = [] for candidate in observed: if candidate not in distinct: distinct.append(candidate) - if invalid_observation or len(observed) != len(observed_values) or len(distinct) != 1: + if ( + invalid_observation + or len(observed) != len(observed_requested_values) + or len(distinct) != 1 + ): requested = None source = "unknown" else: @@ -346,7 +388,22 @@ def _sanitize_control( else: requested = actual source = "environment" - forwarded = actual + + if use_llm and adapter_support and observed_forwarded_values: + observed, invalid_observation = _sanitize_observed_control_values( + name, observed_forwarded_values + ) + distinct = [] + for candidate in observed: + if candidate not in distinct: + distinct.append(candidate) + if ( + not invalid_observation + and len(observed) == len(observed_forwarded_values) + and len(distinct) == 1 + and distinct[0] is not None + ): + forwarded = distinct[0] return { "requested": requested, @@ -423,11 +480,12 @@ def _effective_adapters(records: Sequence[Mapping[object, object]]) -> list[str] def _observed_controls( records: Sequence[Mapping[object, object]], + field: str, ) -> dict[str, list[object]]: """Collect fixed-field constructor controls from successful calls.""" observed = {name: [] for name in ("temperature", "seed", "reasoning_effort")} for record in records: - controls = record.get("forwarded_controls") + controls = record.get(field) if not isinstance(controls, Mapping): continue for name in observed: @@ -463,7 +521,8 @@ def sanitize_llm_provenance( resolved_adapter = _safe_label(provider.get("resolved_adapter")) response_records = _provider_response_records(inference_usage) if use_llm else [] effective_adapters = _effective_adapters(response_records) - observed_controls = _observed_controls(response_records) + observed_requested_controls = _observed_controls(response_records, "requested_controls") + observed_forwarded_controls = _observed_controls(response_records, "forwarded_controls") effective_adapter = ( "not_applicable" if not use_llm @@ -489,6 +548,23 @@ def sanitize_llm_provenance( item = item if isinstance(item, Mapping) else {} revision = item.get("analyzer_revision") revision = revision if isinstance(revision, Mapping) else {} + raw_source_revision = revision.get("source_revision") + raw_source_revision = ( + raw_source_revision if isinstance(raw_source_revision, Mapping) else {} + ) + source_revision = raw_source_revision.get("value") + source_revision = ( + source_revision.lower() + if isinstance(source_revision, str) and _SOURCE_REVISION.fullmatch(source_revision) + else "unknown" + ) + source_revision_source = raw_source_revision.get("source") + if ( + not isinstance(source_revision_source, str) + or source_revision_source not in _SOURCE_REVISION_SOURCES + or source_revision == "unknown" + ): + source_revision_source = "unknown" analyzers.append( { "analyzer_id": slot, @@ -497,6 +573,10 @@ def sanitize_llm_provenance( "analyzer_revision": { "value": _safe_label(revision.get("value")), "source": "skillspector_package", + "source_revision": { + "value": source_revision, + "source": source_revision_source, + }, }, } ) @@ -509,14 +589,17 @@ def sanitize_llm_provenance( sampling.get(name), use_llm=use_llm, effective_adapters=effective_adapters, - observed_values=observed_controls[name], + observed_requested_values=observed_requested_controls[name], + observed_forwarded_values=observed_forwarded_controls[name], ) for name in ("temperature", "seed", "reasoning_effort") } expected_observations = _expected_observed_controls(effective_adapters) controls_observed = all( all( - isinstance(record.get("forwarded_controls"), Mapping) + isinstance(record.get("requested_controls"), Mapping) + and name in record.get("requested_controls", {}) + and isinstance(record.get("forwarded_controls"), Mapping) and name in record.get("forwarded_controls", {}) for record in response_records ) @@ -564,7 +647,7 @@ def sanitize_llm_provenance( "reason": ( "Optional controls do not guarantee identical provider output." if use_llm - else "LLM analysis was disabled for this scan." + else "LLM analysis was not executed for this scan." ), }, } diff --git a/src/skillspector/llm_utils.py b/src/skillspector/llm_utils.py index c9bcd675f..c8f4d6872 100644 --- a/src/skillspector/llm_utils.py +++ b/src/skillspector/llm_utils.py @@ -49,6 +49,7 @@ from skillspector.inference_usage import ( InferenceUsageCollector, chat_model_controls, + chat_model_requested_controls, provider_name, ) from skillspector.model_info import get_max_input_tokens, get_max_output_tokens @@ -448,6 +449,7 @@ def new_inference_usage_collector( request_kind=request_kind, provider=effective_provider, requested_model=model, + requested_controls=chat_model_requested_controls(chat_model), forwarded_controls=chat_model_controls(chat_model), ) @@ -485,9 +487,13 @@ def chat_completion( chat_model=chat_model, ) effective_provider = chat_model_provider_name(chat_model) - if usage_collector is not None and effective_provider is not None: - collector.set_provider(effective_provider) - collector.set_forwarded_controls(chat_model_controls(chat_model)) + if usage_collector is not None: + if effective_provider is not None: + collector.set_provider(effective_provider) + collector.set_controls( + chat_model_requested_controls(chat_model), + chat_model_controls(chat_model), + ) response = _invoke_with_usage(chat_model, prompt, collector) if hasattr(response, "text"): return response.text # type: ignore[union-attr] diff --git a/src/skillspector/nodes/report.py b/src/skillspector/nodes/report.py index cca65a4f4..f7af100d5 100644 --- a/src/skillspector/nodes/report.py +++ b/src/skillspector/nodes/report.py @@ -1150,6 +1150,7 @@ def _build_metadata( # some coverage was lost) into one boolean. execution_enabled = use_llm if llm_execution_enabled is None else llm_execution_enabled unavailable_before_execution = bool(use_llm and not execution_enabled) + llm_executed = bool(use_llm and execution_enabled) meta_analysis_applied = ( use_llm and execution_enabled and provider_available and meta_analyzer_succeeded ) @@ -1169,7 +1170,7 @@ def _build_metadata( "inference_usage": sanitized_inference_usage, "llm_provenance": sanitize_llm_provenance( llm_provenance, - use_llm=use_llm, + use_llm=llm_executed, # Counter-less responses and constructor controls are internal # provenance evidence. The public inference_usage projection above # intentionally omits both, so provenance must inspect the raw diff --git a/src/skillspector/providers/anthropic/provider.py b/src/skillspector/providers/anthropic/provider.py index 0e88c3487..310b5f132 100644 --- a/src/skillspector/providers/anthropic/provider.py +++ b/src/skillspector/providers/anthropic/provider.py @@ -31,7 +31,10 @@ from langchain_core.language_models.chat_models import BaseChatModel from pydantic import SecretStr -from skillspector.inference_usage import register_chat_model_controls +from skillspector.inference_usage import ( + register_chat_model_controls, + retained_chat_model_controls, +) from skillspector.providers import registry from skillspector.providers.chat_models import resolve_reasoning_effort, resolve_sampling_parameters @@ -86,7 +89,11 @@ def create_chat_model( chat_model = ChatAnthropic(**kwargs) register_chat_model_controls( chat_model, - { + retained_chat_model_controls( + chat_model, + ("temperature", "reasoning_effort"), + ), + requested_controls={ "temperature": sampling_parameters.get("temperature"), "reasoning_effort": effort, }, diff --git a/src/skillspector/providers/anthropic_proxy/provider.py b/src/skillspector/providers/anthropic_proxy/provider.py index 89fc6e880..e3f206020 100644 --- a/src/skillspector/providers/anthropic_proxy/provider.py +++ b/src/skillspector/providers/anthropic_proxy/provider.py @@ -52,7 +52,10 @@ from langchain_core.language_models.chat_models import BaseChatModel from pydantic import SecretStr -from skillspector.inference_usage import register_chat_model_controls +from skillspector.inference_usage import ( + register_chat_model_controls, + retained_chat_model_controls, +) from skillspector.providers import registry from skillspector.providers.chat_models import resolve_reasoning_effort, resolve_sampling_parameters @@ -250,7 +253,11 @@ def create_chat_model( chat_model = _ChatAnthropicProxy(**kwargs) register_chat_model_controls( chat_model, - { + retained_chat_model_controls( + chat_model, + ("temperature", "reasoning_effort"), + ), + requested_controls={ "temperature": sampling_parameters.get("temperature"), "reasoning_effort": effort, }, diff --git a/src/skillspector/providers/azure_openai/provider.py b/src/skillspector/providers/azure_openai/provider.py index 160569850..ff556bcb4 100644 --- a/src/skillspector/providers/azure_openai/provider.py +++ b/src/skillspector/providers/azure_openai/provider.py @@ -36,7 +36,10 @@ from langchain_openai import AzureChatOpenAI from pydantic import SecretStr -from skillspector.inference_usage import register_chat_model_controls +from skillspector.inference_usage import ( + register_chat_model_controls, + retained_chat_model_controls, +) from skillspector.providers import registry from skillspector.providers.chat_models import resolve_sampling_parameters @@ -86,7 +89,8 @@ def create_chat_model( chat_model = AzureChatOpenAI(**kwargs) register_chat_model_controls( chat_model, - { + retained_chat_model_controls(chat_model, ("temperature", "seed")), + requested_controls={ "temperature": sampling_parameters.get("temperature"), "seed": sampling_parameters.get("seed"), }, diff --git a/src/skillspector/providers/bedrock/provider.py b/src/skillspector/providers/bedrock/provider.py index f0ee5b597..b954279c2 100644 --- a/src/skillspector/providers/bedrock/provider.py +++ b/src/skillspector/providers/bedrock/provider.py @@ -40,7 +40,10 @@ from langchain_aws import ChatBedrockConverse from langchain_core.language_models.chat_models import BaseChatModel -from skillspector.inference_usage import register_chat_model_controls +from skillspector.inference_usage import ( + register_chat_model_controls, + retained_chat_model_controls, +) from skillspector.providers import registry from skillspector.providers.chat_models import resolve_sampling_parameters @@ -137,7 +140,8 @@ def create_chat_model( chat_model = ChatBedrockConverse(**kwargs) register_chat_model_controls( chat_model, - {"temperature": sampling_parameters.get("temperature")}, + retained_chat_model_controls(chat_model, ("temperature",)), + requested_controls={"temperature": sampling_parameters.get("temperature")}, ) return chat_model diff --git a/src/skillspector/providers/chat_models.py b/src/skillspector/providers/chat_models.py index 8ec586f6a..d16912af2 100644 --- a/src/skillspector/providers/chat_models.py +++ b/src/skillspector/providers/chat_models.py @@ -25,7 +25,10 @@ from langchain_openai import ChatOpenAI from pydantic import SecretStr -from skillspector.inference_usage import register_chat_model_controls +from skillspector.inference_usage import ( + register_chat_model_controls, + retained_chat_model_controls, +) logger = logging.getLogger(__name__) MIN_SAMPLING_SEED = -(1 << 63) @@ -132,7 +135,11 @@ def create_openai_compatible_chat_model( chat_model = ChatOpenAI(**kwargs) register_chat_model_controls( chat_model, - { + retained_chat_model_controls( + chat_model, + ("temperature", "seed", "reasoning_effort"), + ), + requested_controls={ "temperature": sampling_parameters.get("temperature"), "seed": sampling_parameters.get("seed"), "reasoning_effort": reasoning_effort, diff --git a/tests/nodes/test_report.py b/tests/nodes/test_report.py index 5efabaf56..fcdd85978 100644 --- a/tests/nodes/test_report.py +++ b/tests/nodes/test_report.py @@ -1568,6 +1568,11 @@ def test_json_report_exposes_captured_llm_provenance( "model_source": "requested_model", "usage_source": "provider_response", "total_tokens": 1, + "requested_controls": { + "temperature": 0.0, + "seed": 7, + "reasoning_effort": None, + }, "forwarded_controls": { "temperature": 0.0, "seed": 7, @@ -1631,6 +1636,7 @@ def test_json_report_exposes_captured_llm_provenance( assert intent["analyzer_revision"] == { "value": "2.11.2", "source": "skillspector_package", + "source_revision": {"value": "unknown", "source": "unknown"}, } assert provenance["sampling"]["temperature"]["forwarded_to_client"] == 0.0 assert provenance["sampling"]["seed"]["forwarded_to_client"] == 7 @@ -2306,13 +2312,35 @@ def test_preflight_unavailable_log_does_not_claim_runtime_calls_failed( "output_format": "json", "use_llm": False, "llm_requested": True, + # Stale/caller-supplied response evidence cannot turn a preflight- + # disabled scan into one that claims LLM execution. + "inference_usage": [ + { + "node": "semantic_developer_intent", + "request_kind": "structured_output", + "provider": "openai", + "model": "gpt-5.4", + "model_source": "requested_model", + "usage_source": "provider_response", + "total_tokens": 1, + "forwarded_controls": {"seed": 7}, + } + ], } with caplog.at_level(logging.WARNING, logger="skillspector.nodes.report"): - report(state) + result = report(state) assert "unavailable during preflight" in caplog.text assert "0/0" not in caplog.text + provenance = json.loads(result["report_body"])["metadata"]["llm_provenance"] + assert provenance["provider"]["effective_adapter"] == "not_applicable" + assert provenance["determinism"] == { + "classification": "not_applicable", + "control_status": "not_applied", + "provider_guarantee": False, + "reason": "LLM analysis was not executed for this scan.", + } def test_non_degraded_clean_scan_stays_safe() -> None: diff --git a/tests/unit/test_inference_usage.py b/tests/unit/test_inference_usage.py index 326e4fdf3..6a08fd615 100644 --- a/tests/unit/test_inference_usage.py +++ b/tests/unit/test_inference_usage.py @@ -13,6 +13,9 @@ from skillspector.inference_usage import ( InferenceUsageCollector, _usage_record, + chat_model_controls, + chat_model_requested_controls, + register_chat_model_controls, sanitize_inference_usage, ) @@ -75,6 +78,29 @@ def test_collector_marks_response_received_without_usage_counters() -> None: assert sanitize_inference_usage(observation) == [] +def test_constructor_control_registry_drops_credential_shaped_effort() -> None: + class _ChatModel: + pass + + model = _ChatModel() + register_chat_model_controls( + model, + { + "temperature": 0.2, + "seed": 7, + "reasoning_effort": "github_pat_fake-value", + }, + requested_controls={ + "temperature": 0.2, + "seed": 7, + "reasoning_effort": "github_pat_fake-value", + }, + ) + + assert chat_model_controls(model) == {"temperature": 0.2, "seed": 7} + assert chat_model_requested_controls(model) == {"temperature": 0.2, "seed": 7} + + def test_raw_anthropic_usage_adds_external_cache_counters_to_prompt_total() -> None: """Anthropic raw input_tokens excludes cache reads and cache creation.""" message = SimpleNamespace( @@ -367,6 +393,8 @@ def test_report_sanitizer_rejects_url_and_userinfo_model_labels() -> None: [ {**common, "model": "https://key@private-host/v1"}, {**common, "model": "key@private-host"}, + {**common, "model": "github_pat_fake-value"}, + {**common, "model": "0123456789abcdef" * 2}, ] ) == [] diff --git a/tests/unit/test_llm_provenance.py b/tests/unit/test_llm_provenance.py index 852c25185..2162dd451 100644 --- a/tests/unit/test_llm_provenance.py +++ b/tests/unit/test_llm_provenance.py @@ -31,6 +31,10 @@ class AzureOpenAIProvider: pass +class NvInferenceProvider: + pass + + def _models(value: str = "safe/model:1") -> dict[str, str]: return dict.fromkeys(LLM_ANALYZER_SLOTS, value) @@ -44,6 +48,7 @@ def _usage( "usage_source": "provider_response", } if controls is not None: + record["requested_controls"] = controls record["forwarded_controls"] = controls return [record] @@ -100,6 +105,63 @@ def test_capture_records_resolved_adapters_models_and_forwarded_controls( } +def test_editable_source_build_reports_unknown_source_revision( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class _EditableDistribution: + @staticmethod + def read_text(name: str) -> str | None: + assert name == "direct_url.json" + return json.dumps( + { + "url": "file:///workspace/SkillSpector", + "dir_info": {"editable": True}, + } + ) + + monkeypatch.delenv("SKILLSPECTOR_BUILD_REVISION", raising=False) + monkeypatch.setattr( + "skillspector.llm_provenance.distribution", lambda _name: _EditableDistribution() + ) + monkeypatch.setattr("skillspector.llm_provenance.get_active_provider", lambda: OpenAIProvider()) + monkeypatch.setattr( + "skillspector.llm_provenance.get_model_config_provider", lambda: OpenAIProvider() + ) + + result = sanitize_llm_provenance( + capture_llm_provenance(_models()), + use_llm=False, + ) + + assert result["analyzers"][0]["analyzer_revision"]["source_revision"] == { + "value": "unknown", + "source": "unknown", + } + + +def test_injected_build_revision_is_reported_separately_from_package_version( + monkeypatch: pytest.MonkeyPatch, +) -> None: + revision = "ABCDEF0123456789ABCDEF0123456789ABCDEF01" + monkeypatch.setenv("SKILLSPECTOR_BUILD_REVISION", revision) + monkeypatch.setattr("skillspector.llm_provenance.get_active_provider", lambda: OpenAIProvider()) + monkeypatch.setattr( + "skillspector.llm_provenance.get_model_config_provider", lambda: OpenAIProvider() + ) + + result = sanitize_llm_provenance( + capture_llm_provenance(_models()), + use_llm=False, + ) + analyzer_revision = result["analyzers"][0]["analyzer_revision"] + + assert analyzer_revision["source"] == "skillspector_package" + assert analyzer_revision["source_revision"] == { + "value": revision.lower(), + "source": "build_environment", + } + + def test_seed_is_requested_but_not_claimed_forwarded_for_anthropic( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -166,30 +228,21 @@ def test_constructor_observation_supersedes_stale_configuration_capture( monkeypatch.setattr("skillspector.llm_utils.get_active_provider", lambda: OpenAIProvider()) monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0.2") monkeypatch.setenv("SKILLSPECTOR_SEED", "11") - captured = capture_llm_provenance(_models()) + captured = capture_llm_provenance(_models("gpt-4o")) monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0.9") monkeypatch.setenv("SKILLSPECTOR_SEED", "99") monkeypatch.setenv("SKILLSPECTOR_REASONING_EFFORT", "high") - class _FakeChatModel: - pass - - fake_model = _FakeChatModel() - monkeypatch.setattr( - "skillspector.providers.chat_models.ChatOpenAI", - lambda **_kwargs: fake_model, - ) chat_model = create_openai_compatible_chat_model( - model="safe/model:1", + model="gpt-4o", credentials=("test-key", None), max_tokens=128, ) - assert chat_model is fake_model collector = new_inference_usage_collector( node="semantic_developer_intent", request_kind="structured_output", - model="safe/model:1", + model="gpt-4o", chat_model=chat_model, ) collector.mark_response_received() @@ -208,6 +261,49 @@ class _FakeChatModel: assert result["sampling"]["reasoning_effort"]["forwarded_to_client"] == "high" +def test_gpt_5_4_reports_only_controls_retained_by_request_payload( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("skillspector.llm_provenance.get_active_provider", lambda: OpenAIProvider()) + monkeypatch.setattr( + "skillspector.llm_provenance.get_model_config_provider", lambda: OpenAIProvider() + ) + monkeypatch.setattr("skillspector.llm_utils.get_active_provider", lambda: OpenAIProvider()) + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0.2") + monkeypatch.setenv("SKILLSPECTOR_SEED", "11") + captured = capture_llm_provenance(_models("gpt-5.4")) + + chat_model = create_openai_compatible_chat_model( + model="gpt-5.4", + credentials=("test-key", None), + max_tokens=128, + ) + collector = new_inference_usage_collector( + node="semantic_developer_intent", + request_kind="structured_output", + model="gpt-5.4", + chat_model=chat_model, + ) + collector.mark_response_received() + + result = sanitize_llm_provenance( + captured, + use_llm=True, + inference_usage=collector.snapshot(), + ) + + assert result["sampling"]["temperature"] == { + "requested": 0.2, + "source": "environment", + "forwarded_to_client": None, + "adapter_support": True, + "provider_support": "unknown", + } + assert result["sampling"]["seed"]["requested"] == 11 + assert result["sampling"]["seed"]["forwarded_to_client"] == 11 + assert result["determinism"]["control_status"] == "controls_partially_forwarded" + + def test_configured_controls_are_not_claimed_forwarded_without_response_evidence( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -324,6 +420,68 @@ def test_public_projection_drops_unknown_fields_and_redacts_unsafe_labels() -> N assert "private prompt" not in serialized +@pytest.mark.parametrize( + "credential", + [ + "AWS-secret-fake-value", + "github_pat_fake-value", + "xoxb-fake-value", + "hf_fake-value", + "AIza-fake-value", + "0123456789abcdef" * 2, + "AbCdEfGhIjKlMnOpQrSt" * 2, + ], +) +def test_capture_and_projection_do_not_emit_credential_shaped_overrides( + monkeypatch: pytest.MonkeyPatch, + credential: str, +) -> None: + provider = AzureOpenAIProvider() + monkeypatch.setattr("skillspector.llm_provenance.get_active_provider", lambda: provider) + monkeypatch.setattr("skillspector.llm_provenance.get_model_config_provider", lambda: provider) + monkeypatch.setenv("SKILLSPECTOR_REASONING_EFFORT", credential) + monkeypatch.setenv("AZURE_OPENAI_DEPLOYMENT", credential) + + result = sanitize_llm_provenance( + capture_llm_provenance(_models(credential)), + use_llm=False, + ) + serialized = json.dumps(result) + + assert credential not in serialized + assert {item["model"] for item in result["analyzers"]} == {"redacted"} + assert result["provider"]["routing"]["deployment_override"] is None + assert result["sampling"]["reasoning_effort"]["requested"] is None + + +def test_malformed_source_revision_is_not_emitted() -> None: + secret = "github_pat_fake-value" + result = sanitize_llm_provenance( + { + "analyzers": [ + { + "analyzer_id": LLM_ANALYZER_SLOTS[0], + "model": "safe/model:1", + "analyzer_revision": { + "value": "2.11.2", + "source_revision": { + "value": secret, + "source": "build_environment", + }, + }, + } + ] + }, + use_llm=False, + ) + + assert secret not in json.dumps(result) + assert result["analyzers"][0]["analyzer_revision"]["source_revision"] == { + "value": "unknown", + "source": "unknown", + } + + def test_multiple_response_providers_are_reported_as_mixed() -> None: result = sanitize_llm_provenance( None, @@ -450,6 +608,31 @@ def test_effective_provider_comes_from_response_not_preflight_candidate( assert result["sampling"]["seed"]["forwarded_to_client"] == 17 +def test_nv_inference_preserves_all_observed_controls( + monkeypatch: pytest.MonkeyPatch, +) -> None: + provider = NvInferenceProvider() + monkeypatch.setattr("skillspector.llm_provenance.get_active_provider", lambda: provider) + monkeypatch.setattr("skillspector.llm_provenance.get_model_config_provider", lambda: provider) + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0.25") + monkeypatch.setenv("SKILLSPECTOR_SEED", "23") + monkeypatch.setenv("SKILLSPECTOR_REASONING_EFFORT", "high") + + result = sanitize_llm_provenance( + capture_llm_provenance(_models()), + use_llm=True, + inference_usage=_usage( + "nv_inference", + {"temperature": 0.25, "seed": 23, "reasoning_effort": "high"}, + ), + ) + + assert result["provider"]["effective_adapter"] == "nv_inference" + assert result["sampling"]["temperature"]["forwarded_to_client"] == 0.25 + assert result["sampling"]["seed"]["forwarded_to_client"] == 23 + assert result["sampling"]["reasoning_effort"]["forwarded_to_client"] == "high" + + def test_azure_routing_records_deployment_and_api_version( monkeypatch: pytest.MonkeyPatch, ) -> None: From e0f082436b4e47dc30af47408c0cedb302607a8a Mon Sep 17 00:00:00 2001 From: Narendran Raghavan Date: Fri, 18 Sep 2026 03:11:14 -0700 Subject: [PATCH 3/3] fix(report): validate provenance execution and credential fields Signed-off-by: Narendran Raghavan --- src/skillspector/inference_usage.py | 48 +++++-- src/skillspector/llm_provenance.py | 45 +++--- src/skillspector/nodes/report.py | 9 +- tests/nodes/test_report.py | 13 ++ .../nodes/test_report_provenance_boundary.py | 133 ++++++++++++++++++ tests/unit/test_inference_usage.py | 12 +- tests/unit/test_llm_provenance.py | 13 +- 7 files changed, 228 insertions(+), 45 deletions(-) create mode 100644 tests/nodes/test_report_provenance_boundary.py diff --git a/src/skillspector/inference_usage.py b/src/skillspector/inference_usage.py index 91dd7f92e..d684e583f 100644 --- a/src/skillspector/inference_usage.py +++ b/src/skillspector/inference_usage.py @@ -10,6 +10,8 @@ from __future__ import annotations +import base64 +import json import re import threading import weakref @@ -32,7 +34,9 @@ _MIN_SAMPLING_SEED = -(1 << 63) _MAX_SAMPLING_SEED = (1 << 63) - 1 _FORWARDED_CONTROL_NAMES = ("temperature", "seed", "reasoning_effort") -_SAFE_SETTING_RE = re.compile(r"[A-Za-z0-9][A-Za-z0-9 ._:/+\-]{0,255}") +# Public effort telemetry is an enum, even when a provider accepts arbitrary text. +# Unrecognized provider-specific values must not become a channel for credentials. +_REASONING_EFFORTS = frozenset({"none", "minimal", "low", "medium", "high", "xhigh", "max", "auto"}) _CREDENTIAL_PREFIXES = ( "sk-", "nvapi-", @@ -54,7 +58,9 @@ "asia", "aws-secret-", ) -_UNPREFIXED_CREDENTIAL = re.compile(r"(?:[0-9a-fA-F]{32,64}|[A-Za-z0-9_+/=-]{40,88})\Z") +_UNPREFIXED_CREDENTIAL = re.compile(r"(?:[0-9a-fA-F]{32,64}|[A-Za-z0-9_+/=]{40,88})\Z") +_AUTHORIZATION_CREDENTIAL = re.compile(r"(?:authorization\s*:\s*)?(?:bearer|basic)(?:\s|:)", re.I) +_COMPACT_CREDENTIAL = re.compile(r"[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]*){2,4}\Z") _CHAT_MODEL_CONTROLS: dict[ int, @@ -67,17 +73,40 @@ _CHAT_MODEL_CONTROLS_LOCK = threading.Lock() +def _is_compact_credential(value: str) -> bool: + """Recognize JSON JWT/JWE headers without rejecting dotted model versions.""" + if not _COMPACT_CREDENTIAL.fullmatch(value): + return False + header = value.partition(".")[0] + try: + decoded = json.loads(base64.urlsafe_b64decode(header + "=" * (-len(header) % 4))) + except (ValueError, RecursionError): + return False + return isinstance(decoded, dict) + + def looks_like_credential(value: object) -> bool: """Return whether a printable label resembles a common secret value.""" if not isinstance(value, str): return False candidate = value.strip() lowered = candidate.lower() - return lowered.startswith(_CREDENTIAL_PREFIXES) or bool( - _UNPREFIXED_CREDENTIAL.fullmatch(candidate) + return ( + lowered.startswith(_CREDENTIAL_PREFIXES) + or bool(_AUTHORIZATION_CREDENTIAL.match(candidate)) + or _is_compact_credential(candidate) + # A full hyphenated model name is not an opaque base64 credential. + # Still reject long opaque components embedded in a namespaced label. + or any(_UNPREFIXED_CREDENTIAL.fullmatch(part) for part in candidate.split("-")) ) +def safe_reasoning_effort(value: object) -> str | None: + """Return a recognized effort value safe for public configuration telemetry.""" + candidate = value.strip() if isinstance(value, str) else "" + return candidate if candidate in _REASONING_EFFORTS else None + + class InferenceUsageRecord(TypedDict): """One provider-reported inference request, safe to serialize.""" @@ -120,15 +149,8 @@ def _forwarded_controls(value: Mapping[str, object] | None) -> dict[str, float | and _MIN_SAMPLING_SEED <= raw <= _MAX_SAMPLING_SEED ): controls[name] = raw - elif isinstance(raw, str): - setting = raw.strip() - if ( - _SAFE_SETTING_RE.fullmatch(setting) - and "://" not in setting - and "@" not in setting - and not looks_like_credential(setting) - ): - controls[name] = setting + elif (setting := safe_reasoning_effort(raw)) is not None: + controls[name] = setting return controls diff --git a/src/skillspector/llm_provenance.py b/src/skillspector/llm_provenance.py index e0fef908a..b41f15c2e 100644 --- a/src/skillspector/llm_provenance.py +++ b/src/skillspector/llm_provenance.py @@ -12,7 +12,7 @@ from collections.abc import Mapping, Sequence from importlib.metadata import distribution, version -from skillspector.inference_usage import looks_like_credential, provider_name +from skillspector.inference_usage import looks_like_credential, provider_name, safe_reasoning_effort from skillspector.providers import get_active_provider, get_model_config_provider from skillspector.providers.chat_models import ( MAX_SAMPLING_SEED, @@ -59,7 +59,8 @@ } ) _SAFE_LABEL = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/+\-]{0,255}") -_SAFE_SETTING = re.compile(r"[A-Za-z0-9][A-Za-z0-9 ._:/+\-]{0,255}") +_SAFE_DEPLOYMENT = re.compile(r"[A-Za-z0-9][A-Za-z0-9._\-]{0,255}") +_SAFE_API_VERSION = re.compile(r"[0-9]{4}-[0-9]{2}-[0-9]{2}(?:-preview)?") _MAX_SAFE_SEED = MAX_SAMPLING_SEED _MIN_SAFE_SEED = MIN_SAMPLING_SEED _CONTROL_SOURCES = frozenset( @@ -85,26 +86,24 @@ def _safe_label(value: object, fallback: str = "unknown") -> str: return fallback -def _safe_setting(value: object, fallback: str = "unknown") -> str: - """Return a bounded printable setting while allowing provider-specific spaces.""" - candidate = value if isinstance(value, str) else "" - candidate = candidate.strip() - if ( - _SAFE_SETTING.fullmatch(candidate) - and "://" not in candidate - and "@" not in candidate - and not looks_like_credential(candidate) - ): - return candidate - return fallback - - def _safe_optional_label(value: object) -> str | None: """Return a safe label or ``None`` without inventing a placeholder.""" label = _safe_label(value, fallback="") return label or None +def _safe_deployment(value: object) -> str | None: + """Deployment names have a narrower alphabet than namespaced model IDs.""" + candidate = _safe_optional_label(value) + return candidate if candidate and _SAFE_DEPLOYMENT.fullmatch(candidate) else None + + +def _safe_api_version(value: object) -> str | None: + """Azure API versions are dates with an optional preview suffix.""" + candidate = value.strip() if isinstance(value, str) else "" + return candidate if _SAFE_API_VERSION.fullmatch(candidate) else None + + def _capture_source_revision() -> tuple[str, str]: """Return an injected/packaged VCS identity without invoking Git.""" injected = os.environ.get("SKILLSPECTOR_BUILD_REVISION", "").strip() @@ -135,9 +134,9 @@ def _capture_provider_routing(resolved_adapter: str) -> dict[str, object]: } raw_deployment = os.environ.get("AZURE_OPENAI_DEPLOYMENT", "").strip() - deployment = _safe_optional_label(raw_deployment) + deployment = _safe_deployment(raw_deployment) raw_api_version = os.environ.get("AZURE_OPENAI_API_VERSION", "").strip() - api_version = _safe_optional_label(raw_api_version or _AZURE_OPENAI_DEFAULT_API_VERSION) + api_version = _safe_api_version(raw_api_version or _AZURE_OPENAI_DEFAULT_API_VERSION) return { "deployment_override": deployment, "deployment_source": ( @@ -182,7 +181,7 @@ def _requested_effort(raw: str) -> tuple[str | None, str]: if not raw: return None, "provider_default" value = resolve_reasoning_effort() - value = _safe_setting(value, fallback="") + value = safe_reasoning_effort(value) return (value, "environment") if value else (None, "invalid_environment") @@ -297,7 +296,7 @@ def _sanitize_observed_control_values( else: invalid = True else: - setting = _safe_setting(candidate, fallback="") + setting = safe_reasoning_effort(candidate) if setting: observed.append(setting) else: @@ -349,7 +348,7 @@ def _sanitize_control( else None ) else: - requested = _safe_setting(requested, fallback="") or None + requested = safe_reasoning_effort(requested) if source == "environment" and requested is None: source = ( "out_of_range" @@ -432,7 +431,7 @@ def _sanitize_provider_routing(value: object, *, resolved_adapter: str) -> dict[ } raw = value if isinstance(value, Mapping) else {} - deployment = _safe_optional_label(raw.get("deployment_override")) + deployment = _safe_deployment(raw.get("deployment_override")) deployment_source = raw.get("deployment_source") if not isinstance(deployment_source, str) or deployment_source not in _DEPLOYMENT_SOURCES: deployment_source = "unknown" @@ -441,7 +440,7 @@ def _sanitize_provider_routing(value: object, *, resolved_adapter: str) -> dict[ elif deployment_source != "environment": deployment = None - api_version = _safe_optional_label(raw.get("api_version")) + api_version = _safe_api_version(raw.get("api_version")) api_version_source = raw.get("api_version_source") if not isinstance(api_version_source, str) or api_version_source not in _API_VERSION_SOURCES: api_version_source = "unknown" diff --git a/src/skillspector/nodes/report.py b/src/skillspector/nodes/report.py index f7af100d5..9269343b5 100644 --- a/src/skillspector/nodes/report.py +++ b/src/skillspector/nodes/report.py @@ -1150,7 +1150,14 @@ def _build_metadata( # some coverage was lost) into one boolean. execution_enabled = use_llm if llm_execution_enabled is None else llm_execution_enabled unavailable_before_execution = bool(use_llm and not execution_enabled) - llm_executed = bool(use_llm and execution_enabled) + response_observed = any( + isinstance(record, Mapping) and record.get("usage_source") == "provider_response" + for record in inference_usage or [] + ) + # Enablement alone does not prove execution: every analyzer may have + # returned not_applicable. Failed attempts still count, as do successful + # provider responses whose transport supplied no token counters. + llm_executed = bool(use_llm and execution_enabled and (attempted or response_observed)) meta_analysis_applied = ( use_llm and execution_enabled and provider_available and meta_analyzer_succeeded ) diff --git a/tests/nodes/test_report.py b/tests/nodes/test_report.py index fcdd85978..f896171d8 100644 --- a/tests/nodes/test_report.py +++ b/tests/nodes/test_report.py @@ -1355,6 +1355,10 @@ def test_report_llm_degraded_when_all_calls_failed(monkeypatch: pytest.MonkeyPat assert meta["llm_degraded"] is True assert meta["llm_calls_attempted"] == 3 assert meta["llm_calls_succeeded"] == 0 + # Attempted calls remain execution evidence even when none returns a response. + assert meta["llm_provenance"]["provider"]["effective_adapter"] == "unknown" + assert meta["llm_provenance"]["determinism"]["classification"] == "nondeterministic" + assert meta["llm_provenance"]["determinism"]["control_status"] == "configuration_unknown" # Distinct error reasons are surfaced (deduped). assert "claude empty stdout" in meta["llm_error"] assert "static analysis only" in meta["llm_error"] @@ -1698,6 +1702,7 @@ def test_json_report_preserves_counterless_cli_provider_observation( assert meta["inference_usage"] == [] assert meta["llm_provenance"]["provider"]["effective_adapter"] == "claude_cli" + assert meta["llm_provenance"]["determinism"]["classification"] == "nondeterministic" assert meta["llm_provenance"]["determinism"]["control_status"] == "provider_defaults" @@ -2171,6 +2176,14 @@ def test_explicit_all_not_applicable_semantic_pass_stays_safe( assert result["risk_recommendation"] == "SAFE" assert "llm_degraded" not in metadata + assert "llm_calls_attempted" not in metadata + assert metadata["llm_provenance"]["provider"]["effective_adapter"] == "not_applicable" + assert metadata["llm_provenance"]["determinism"] == { + "classification": "not_applicable", + "control_status": "not_applied", + "provider_guarantee": False, + "reason": "LLM analysis was not executed for this scan.", + } def test_unavailable_provider_floors_recommendation_even_with_success_records( diff --git a/tests/nodes/test_report_provenance_boundary.py b/tests/nodes/test_report_provenance_boundary.py new file mode 100644 index 000000000..1d7681ccf --- /dev/null +++ b/tests/nodes/test_report_provenance_boundary.py @@ -0,0 +1,133 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Credential boundaries for captured and response-derived report provenance.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from skillspector.cli import app +from skillspector.llm_provenance import LLM_ANALYZER_SLOTS +from skillspector.nodes.report import report +from skillspector.state import SkillspectorState + +_JWT = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJzeW50aGV0aWMifQ.synthetic_signature" +_AUTHORIZATION = "Bearer synthetic-secret-token-123456" +_LONG_MODEL = "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B" + + +def test_no_llm_cli_report_rejects_credentials_from_environment( + monkeypatch: pytest.MonkeyPatch, + safe_skill_dir: Path, + tmp_path: Path, +) -> None: + """Configuration capture must be safe even when no provider call is made.""" + provider = type("AzureOpenAIProvider", (), {})() + monkeypatch.setattr("skillspector.llm_provenance.get_active_provider", lambda: provider) + monkeypatch.setattr("skillspector.llm_provenance.get_model_config_provider", lambda: provider) + monkeypatch.setenv("SKILLSPECTOR_MODEL", _JWT) + monkeypatch.setenv("SKILLSPECTOR_REASONING_EFFORT", _AUTHORIZATION) + monkeypatch.setenv("AZURE_OPENAI_DEPLOYMENT", _JWT) + monkeypatch.setenv("AZURE_OPENAI_API_VERSION", _JWT) + output = tmp_path / "report.json" + + result = CliRunner().invoke( + app, + ["scan", str(safe_skill_dir), "--no-llm", "--format", "json", "--output", str(output)], + ) + + assert result.exit_code == 0, result.output + serialized = output.read_text() + provenance = json.loads(serialized)["metadata"]["llm_provenance"] + assert _JWT not in serialized + assert _AUTHORIZATION not in serialized + assert {item["model"] for item in provenance["analyzers"]} == {"redacted"} + assert provenance["sampling"]["reasoning_effort"]["requested"] is None + assert provenance["sampling"]["reasoning_effort"]["source"] == "invalid_environment" + assert provenance["provider"]["routing"]["deployment_override"] is None + assert provenance["provider"]["routing"]["api_version"] is None + + +@pytest.mark.parametrize("use_llm", [False, True]) +def test_report_rejects_credentials_in_raw_provenance_and_response_controls( + monkeypatch: pytest.MonkeyPatch, use_llm: bool +) -> None: + """Final serialization revalidates raw state instead of trusting capture.""" + monkeypatch.setattr("skillspector.nodes.report.is_llm_available", lambda: (True, None)) + state: SkillspectorState = { + "filtered_findings": [], + "component_metadata": [], + "has_executable_scripts": False, + "manifest": {}, + "output_format": "json", + "use_llm": use_llm, + "llm_call_log": [], + "inference_usage": [ + { + "node": "semantic_developer_intent", + "request_kind": "structured_output", + "provider": "openai", + "model": model, + "model_source": "provider_response", + "usage_source": "provider_response", + "total_tokens": 1, + "requested_controls": {"reasoning_effort": _AUTHORIZATION}, + "forwarded_controls": {"reasoning_effort": _AUTHORIZATION}, + } + for model in (_JWT, _LONG_MODEL) + ], + "llm_provenance": { + "provider": { + "configured_adapter": "azure_openai", + "resolved_adapter": "azure_openai", + "routing": { + "deployment_override": _JWT, + "deployment_source": "environment", + "api_version": _JWT, + "api_version_source": "environment", + }, + }, + "analyzers": [{"analyzer_id": slot, "model": _JWT} for slot in LLM_ANALYZER_SLOTS], + "sampling": { + "reasoning_effort": { + "requested": _AUTHORIZATION, + "source": "environment", + "adapter_support": True, + } + }, + }, + } + + serialized = report(state)["report_body"] + metadata = json.loads(serialized)["metadata"] + provenance = metadata["llm_provenance"] + + assert _JWT not in serialized + assert _AUTHORIZATION not in serialized + assert {item["model"] for item in provenance["analyzers"]} == {"redacted"} + assert provenance["sampling"]["reasoning_effort"]["requested"] is None + assert provenance["sampling"]["reasoning_effort"]["forwarded_to_client"] is None + assert provenance["provider"]["routing"]["deployment_override"] is None + assert provenance["provider"]["routing"]["api_version"] is None + assert [record["model"] for record in metadata["inference_usage"]] == [_LONG_MODEL] + + +def test_no_llm_cli_report_preserves_long_model_identifiers( + monkeypatch: pytest.MonkeyPatch, safe_skill_dir: Path, tmp_path: Path +) -> None: + monkeypatch.setenv("SKILLSPECTOR_MODEL", _LONG_MODEL) + output = tmp_path / "report.json" + + result = CliRunner().invoke( + app, + ["scan", str(safe_skill_dir), "--no-llm", "--format", "json", "--output", str(output)], + ) + + assert result.exit_code == 0, result.output + provenance = json.loads(output.read_text())["metadata"]["llm_provenance"] + assert {item["model"] for item in provenance["analyzers"]} == {_LONG_MODEL} diff --git a/tests/unit/test_inference_usage.py b/tests/unit/test_inference_usage.py index 6a08fd615..83f886396 100644 --- a/tests/unit/test_inference_usage.py +++ b/tests/unit/test_inference_usage.py @@ -7,6 +7,7 @@ from types import SimpleNamespace +import pytest from langchain_core.messages import AIMessage from langchain_core.outputs import ChatGeneration, LLMResult @@ -78,7 +79,11 @@ def test_collector_marks_response_received_without_usage_counters() -> None: assert sanitize_inference_usage(observation) == [] -def test_constructor_control_registry_drops_credential_shaped_effort() -> None: +@pytest.mark.parametrize( + "effort", + ["github_pat_fake-value", "Bearer synthetic-secret-token-123456", "unrecognized setting"], +) +def test_constructor_control_registry_drops_credential_shaped_effort(effort: str) -> None: class _ChatModel: pass @@ -88,12 +93,12 @@ class _ChatModel: { "temperature": 0.2, "seed": 7, - "reasoning_effort": "github_pat_fake-value", + "reasoning_effort": effort, }, requested_controls={ "temperature": 0.2, "seed": 7, - "reasoning_effort": "github_pat_fake-value", + "reasoning_effort": effort, }, ) @@ -394,6 +399,7 @@ def test_report_sanitizer_rejects_url_and_userinfo_model_labels() -> None: {**common, "model": "https://key@private-host/v1"}, {**common, "model": "key@private-host"}, {**common, "model": "github_pat_fake-value"}, + {**common, "model": "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJzeW50aGV0aWMifQ.signature"}, {**common, "model": "0123456789abcdef" * 2}, ] ) diff --git a/tests/unit/test_llm_provenance.py b/tests/unit/test_llm_provenance.py index 2162dd451..9f36903ee 100644 --- a/tests/unit/test_llm_provenance.py +++ b/tests/unit/test_llm_provenance.py @@ -428,6 +428,10 @@ def test_public_projection_drops_unknown_fields_and_redacts_unsafe_labels() -> N "xoxb-fake-value", "hf_fake-value", "AIza-fake-value", + "Bearer synthetic-secret-token-123456", + "Basic c3ludGhldGljOnNlY3JldA==", + "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJzeW50aGV0aWMifQ.fake_signature", + "ewogICJhbGciOiAibm9uZSIKfQ.eyJzdWIiOiJzeW50aGV0aWMifQ.", "0123456789abcdef" * 2, "AbCdEfGhIjKlMnOpQrSt" * 2, ], @@ -541,7 +545,7 @@ def test_public_projection_ignores_unhashable_analyzer_ids() -> None: assert {item["model"] for item in result["analyzers"]} == {"redacted"} -def test_provider_specific_reasoning_effort_is_recorded_exactly( +def test_unknown_provider_specific_reasoning_effort_is_not_published( monkeypatch: pytest.MonkeyPatch, ) -> None: monkeypatch.setattr("skillspector.llm_provenance.get_active_provider", lambda: OpenAIProvider()) @@ -559,10 +563,9 @@ def test_provider_specific_reasoning_effort_is_recorded_exactly( ), ) - assert result["sampling"]["reasoning_effort"]["requested"] == "provider specific value" - assert ( - result["sampling"]["reasoning_effort"]["forwarded_to_client"] == "provider specific value" - ) + assert result["sampling"]["reasoning_effort"]["requested"] is None + assert result["sampling"]["reasoning_effort"]["forwarded_to_client"] is None + assert "provider specific value" not in json.dumps(result) def test_invalid_and_unknown_controls_are_not_called_provider_defaults(