diff --git a/src/skillspector/inference_usage.py b/src/skillspector/inference_usage.py index 56cf6320b..d684e583f 100644 --- a/src/skillspector/inference_usage.py +++ b/src/skillspector/inference_usage.py @@ -10,8 +10,11 @@ from __future__ import annotations +import base64 +import json import re import threading +import weakref from collections.abc import Mapping, Sequence from typing import NotRequired, TypedDict @@ -28,6 +31,80 @@ "total_tokens", ) _MAX_TOKEN_COUNT = (1 << 63) - 1 +_MIN_SAMPLING_SEED = -(1 << 63) +_MAX_SAMPLING_SEED = (1 << 63) - 1 +_FORWARDED_CONTROL_NAMES = ("temperature", "seed", "reasoning_effort") +# Public effort telemetry is an enum, even when a provider accepts arbitrary text. +# Unrecognized provider-specific values must not become a channel for credentials. +_REASONING_EFFORTS = frozenset({"none", "minimal", "low", "medium", "high", "xhigh", "max", "auto"}) +_CREDENTIAL_PREFIXES = ( + "sk-", + "nvapi-", + "ghp_", + "gho_", + "ghu_", + "ghs_", + "ghr_", + "github_pat_", + "glpat-", + "bearer-", + "xoxb-", + "xoxp-", + "xoxa-", + "xoxr-", + "hf_", + "aiza", + "akia", + "asia", + "aws-secret-", +) +_UNPREFIXED_CREDENTIAL = re.compile(r"(?:[0-9a-fA-F]{32,64}|[A-Za-z0-9_+/=]{40,88})\Z") +_AUTHORIZATION_CREDENTIAL = re.compile(r"(?:authorization\s*:\s*)?(?:bearer|basic)(?:\s|:)", re.I) +_COMPACT_CREDENTIAL = re.compile(r"[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]*){2,4}\Z") + +_CHAT_MODEL_CONTROLS: dict[ + int, + tuple[ + weakref.ReferenceType[object], + dict[str, float | int | str | None], + dict[str, float | int | str | None], + ], +] = {} +_CHAT_MODEL_CONTROLS_LOCK = threading.Lock() + + +def _is_compact_credential(value: str) -> bool: + """Recognize JSON JWT/JWE headers without rejecting dotted model versions.""" + if not _COMPACT_CREDENTIAL.fullmatch(value): + return False + header = value.partition(".")[0] + try: + decoded = json.loads(base64.urlsafe_b64decode(header + "=" * (-len(header) % 4))) + except (ValueError, RecursionError): + return False + return isinstance(decoded, dict) + + +def looks_like_credential(value: object) -> bool: + """Return whether a printable label resembles a common secret value.""" + if not isinstance(value, str): + return False + candidate = value.strip() + lowered = candidate.lower() + return ( + lowered.startswith(_CREDENTIAL_PREFIXES) + or bool(_AUTHORIZATION_CREDENTIAL.match(candidate)) + or _is_compact_credential(candidate) + # A full hyphenated model name is not an opaque base64 credential. + # Still reject long opaque components embedded in a namespaced label. + or any(_UNPREFIXED_CREDENTIAL.fullmatch(part) for part in candidate.split("-")) + ) + + +def safe_reasoning_effort(value: object) -> str | None: + """Return a recognized effort value safe for public configuration telemetry.""" + candidate = value.strip() if isinstance(value, str) else "" + return candidate if candidate in _REASONING_EFFORTS else None class InferenceUsageRecord(TypedDict): @@ -45,6 +122,127 @@ class InferenceUsageRecord(TypedDict): cache_write_tokens: NotRequired[int] reasoning_tokens: NotRequired[int] total_tokens: NotRequired[int] + # Internal-only construction evidence. ``sanitize_inference_usage`` never + # includes this field in the public token-usage projection; the provenance + # sanitizer consumes it separately after a provider response is observed. + requested_controls: NotRequired[dict[str, float | int | str | None]] + forwarded_controls: NotRequired[dict[str, float | int | str | None]] + + +def _forwarded_controls(value: Mapping[str, object] | None) -> dict[str, float | int | str | None]: + """Return the fixed, non-secret sampling-control construction record.""" + source = value or {} + controls: dict[str, float | int | str | None] = {} + for name in _FORWARDED_CONTROL_NAMES: + if name not in source: + continue + raw = source.get(name) + if raw is None: + controls[name] = None + elif name == "temperature": + if isinstance(raw, (int, float)) and not isinstance(raw, bool) and 0 <= float(raw) <= 1: + controls[name] = float(raw) + elif name == "seed": + if ( + isinstance(raw, int) + and not isinstance(raw, bool) + and _MIN_SAMPLING_SEED <= raw <= _MAX_SAMPLING_SEED + ): + controls[name] = raw + elif (setting := safe_reasoning_effort(raw)) is not None: + controls[name] = setting + return controls + + +def register_chat_model_controls( + chat_model: object, + forwarded_controls: Mapping[str, object], + *, + requested_controls: Mapping[str, object] | None = None, +) -> None: + """Associate a model with requested and normalized request controls.""" + model_id = id(chat_model) + sanitized_requested = _forwarded_controls(requested_controls) + sanitized_forwarded = _forwarded_controls(forwarded_controls) + + def _discard(model_ref: weakref.ReferenceType[object]) -> None: + with _CHAT_MODEL_CONTROLS_LOCK: + current = _CHAT_MODEL_CONTROLS.get(model_id) + if current is not None and current[0] is model_ref: + _CHAT_MODEL_CONTROLS.pop(model_id, None) + + try: + model_ref = weakref.ref(chat_model, _discard) + except TypeError: + return + with _CHAT_MODEL_CONTROLS_LOCK: + _CHAT_MODEL_CONTROLS[model_id] = ( + model_ref, + sanitized_requested, + sanitized_forwarded, + ) + + +def chat_model_controls(chat_model: object | None) -> dict[str, float | int | str | None]: + """Return detached construction evidence for *chat_model*, when recorded.""" + if chat_model is None: + return {} + with _CHAT_MODEL_CONTROLS_LOCK: + current = _CHAT_MODEL_CONTROLS.get(id(chat_model)) + if current is None or current[0]() is not chat_model: + return {} + return current[2].copy() + + +def chat_model_requested_controls( + chat_model: object | None, +) -> dict[str, float | int | str | None]: + """Return the controls resolved when *chat_model* was constructed.""" + if chat_model is None: + return {} + with _CHAT_MODEL_CONTROLS_LOCK: + current = _CHAT_MODEL_CONTROLS.get(id(chat_model)) + if current is None or current[0]() is not chat_model: + return {} + return current[1].copy() + + +def retained_chat_model_controls( + chat_model: object, + names: Sequence[str], +) -> dict[str, float | int | str | None]: + """Return controls retained by the normalized provider request payload. + + LangChain may accept a constructor option and then remove it for a + provider/model combination. Provenance must describe the request that the + adapter will send, not the raw constructor arguments supplied before that + normalization. + """ + selected = [name for name in names if name in _FORWARDED_CONTROL_NAMES] + controls: dict[str, object] = dict.fromkeys(selected) + payload: Mapping[str, object] | None = None + payload_builder = getattr(chat_model, "_get_request_payload", None) + if callable(payload_builder): + try: + candidate = payload_builder("SkillSpector provenance probe") + except (TypeError, ValueError): + candidate = None + if isinstance(candidate, Mapping): + payload = candidate + + if payload is not None: + output_config = payload.get("output_config") + output_config = output_config if isinstance(output_config, Mapping) else {} + for name in selected: + if name == "reasoning_effort": + controls[name] = payload.get(name, output_config.get("effort")) + else: + controls[name] = payload.get(name) + else: + for name in selected: + attribute = "effort" if name == "reasoning_effort" else name + controls[name] = getattr(chat_model, name, getattr(chat_model, attribute, None)) + return _forwarded_controls(controls) def _mapping(value: object) -> Mapping[str, object]: @@ -98,7 +296,12 @@ def _strict_label(value: object) -> str | None: def _strict_model_label(value: object) -> str | None: """Return a model label only when it cannot encode a URL or userinfo.""" candidate = _strict_label(value) - if candidate is None or "://" in candidate or "@" in candidate: + if ( + candidate is None + or "://" in candidate + or "@" in candidate + or looks_like_credential(candidate) + ): return None return candidate @@ -110,14 +313,18 @@ def _model_label(value: object, fallback: str = "unknown") -> str: def provider_name(provider: object) -> str: """Return a stable provider label without endpoint or credential data.""" names = { + "AntigravityCLIProvider": "antigravity_cli", "AnthropicProvider": "anthropic", "AnthropicProxyProvider": "anthropic_proxy", + "AzureOpenAIProvider": "azure_openai", "BedrockProvider": "bedrock", "ClaudeCLIProvider": "claude_cli", "CodexCLIProvider": "codex_cli", "GeminiCLIProvider": "gemini_cli", "NvBuildProvider": "nv_build", "NvInferenceProvider": "nv_inference", + "OllamaProvider": "ollama", + "OpenAICompatibleProvider": "openai_compatible", "OpenAIProvider": "openai", "OpencodeCLIProvider": "opencode_cli", } @@ -297,11 +504,15 @@ def __init__( request_kind: str, provider: str, requested_model: str, + requested_controls: Mapping[str, object] | None = None, + forwarded_controls: Mapping[str, object] | None = None, ) -> None: self._node = node self._request_kind = request_kind self._provider = provider self._requested_model = requested_model + self._requested_controls = _forwarded_controls(requested_controls) + self._forwarded_controls = _forwarded_controls(forwarded_controls) self._records: list[InferenceUsageRecord] = [] self._response_received = False self._lock = threading.Lock() @@ -328,12 +539,32 @@ def on_llm_end(self, response: LLMResult, **kwargs: object) -> None: with self._lock: self._response_received = True if record is not None: + if self._requested_controls: + record["requested_controls"] = self._requested_controls.copy() + if self._forwarded_controls: + record["forwarded_controls"] = self._forwarded_controls.copy() self._records.append(record) + else: + self._records.append(self._response_observation()) + + def _response_observation(self) -> InferenceUsageRecord: + """Build counter-less, internal-only evidence of a completed response.""" + return { + "node": _label(self._node), + "request_kind": _label(self._request_kind), + "provider": _label(self._provider), + "model": _model_label(self._requested_model), + "model_source": "requested_model", + "usage_source": "provider_response", + "requested_controls": self._requested_controls.copy(), + "forwarded_controls": self._forwarded_controls.copy(), + } def mark_response_received(self) -> None: """Record a completed response from a non-LangChain transport.""" with self._lock: self._response_received = True + self._records.append(self._response_observation()) def set_provider(self, provider: str) -> None: """Set the effective provider before the first response is observed.""" @@ -343,6 +574,16 @@ def set_provider(self, provider: str) -> None: raise RuntimeError("cannot change inference provider after a response") self._provider = label + def set_controls( + self, + requested: Mapping[str, object] | None, + forwarded: Mapping[str, object] | None, + ) -> None: + """Update constructor/request evidence before the next response.""" + with self._lock: + self._requested_controls = _forwarded_controls(requested) + self._forwarded_controls = _forwarded_controls(forwarded) + @property def response_received(self) -> bool: """Whether the provider returned, even when it reported no token usage.""" @@ -350,9 +591,19 @@ def response_received(self) -> bool: return self._response_received def snapshot(self) -> list[InferenceUsageRecord]: - """Return detached copies safe for graph-state serialization.""" + """Return usage plus counter-less response evidence for provenance.""" with self._lock: - return [record.copy() for record in self._records] + snapshot: list[InferenceUsageRecord] = [] + for record in self._records: + detached = record.copy() + controls = record.get("forwarded_controls") + if isinstance(controls, dict): + detached["forwarded_controls"] = controls.copy() + requested = record.get("requested_controls") + if isinstance(requested, dict): + detached["requested_controls"] = requested.copy() + snapshot.append(detached) + return snapshot def sanitize_inference_usage( diff --git a/src/skillspector/llm_analyzer_base.py b/src/skillspector/llm_analyzer_base.py index b3e0aae03..ad8f4afd7 100644 --- a/src/skillspector/llm_analyzer_base.py +++ b/src/skillspector/llm_analyzer_base.py @@ -41,7 +41,11 @@ from langchain_openai import ChatOpenAI from pydantic import BaseModel, Field, ValidationError, field_validator -from skillspector.inference_usage import InferenceUsageRecord +from skillspector.inference_usage import ( + InferenceUsageRecord, + chat_model_controls, + chat_model_requested_controls, +) from skillspector.inspection_ledger import ( AnalyzerStatusEvent, InspectionLedgerEvent, @@ -57,6 +61,7 @@ _AgentCLIMessage, _ainvoke_with_usage, _invoke_with_usage, + chat_model_provider_name, get_chat_model, new_inference_usage_collector, ) @@ -724,6 +729,13 @@ def _model_for_call(self) -> tuple[object, object | None]: return self._llm, self._structured_llm llm = get_chat_model(model=self.model, timeout=remaining) _uses_native_connection_retries(llm, max_retries=0) + effective_provider = chat_model_provider_name(llm) + if effective_provider is not None: + self._usage_collector.set_provider(effective_provider) + self._usage_collector.set_controls( + chat_model_requested_controls(llm), + chat_model_controls(llm), + ) structured = ( llm.with_structured_output(self.response_schema) if self.response_schema else None ) diff --git a/src/skillspector/llm_provenance.py b/src/skillspector/llm_provenance.py new file mode 100644 index 000000000..b41f15c2e --- /dev/null +++ b/src/skillspector/llm_provenance.py @@ -0,0 +1,652 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Sanitized, scan-level provenance for LLM-backed analysis configuration.""" + +from __future__ import annotations + +import json +import math +import os +import re +from collections.abc import Mapping, Sequence +from importlib.metadata import distribution, version + +from skillspector.inference_usage import looks_like_credential, provider_name, safe_reasoning_effort +from skillspector.providers import get_active_provider, get_model_config_provider +from skillspector.providers.chat_models import ( + MAX_SAMPLING_SEED, + MIN_SAMPLING_SEED, + resolve_reasoning_effort, + resolve_seed, + resolve_temperature, +) + +LLM_PROVENANCE_SCHEMA_VERSION = 1 +LLM_ANALYZER_SLOTS = ( + "mcp_tool_poisoning", + "semantic_developer_intent", + "semantic_quality_policy", + "semantic_security_discovery", + "meta_analyzer", +) + +_TEMPERATURE_ADAPTERS = frozenset( + { + "anthropic", + "anthropic_proxy", + "azure_openai", + "bedrock", + "nv_build", + "nv_inference", + "ollama", + "openai", + "openai_compatible", + } +) +_SEED_ADAPTERS = frozenset( + {"azure_openai", "nv_build", "nv_inference", "ollama", "openai", "openai_compatible"} +) +_REASONING_EFFORT_ADAPTERS = frozenset( + { + "anthropic", + "anthropic_proxy", + "nv_build", + "nv_inference", + "ollama", + "openai", + "openai_compatible", + } +) +_SAFE_LABEL = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/+\-]{0,255}") +_SAFE_DEPLOYMENT = re.compile(r"[A-Za-z0-9][A-Za-z0-9._\-]{0,255}") +_SAFE_API_VERSION = re.compile(r"[0-9]{4}-[0-9]{2}-[0-9]{2}(?:-preview)?") +_MAX_SAFE_SEED = MAX_SAMPLING_SEED +_MIN_SAFE_SEED = MIN_SAMPLING_SEED +_CONTROL_SOURCES = frozenset( + {"environment", "provider_default", "unset", "invalid_environment", "out_of_range", "unknown"} +) +_DEPLOYMENT_SOURCES = frozenset({"environment", "resolved_model", "not_applicable", "unknown"}) +_API_VERSION_SOURCES = frozenset({"environment", "provider_default", "not_applicable", "unknown"}) +_AZURE_OPENAI_DEFAULT_API_VERSION = "2024-06-01" +_SOURCE_REVISION = re.compile(r"[0-9a-fA-F]{7,64}") +_SOURCE_REVISION_SOURCES = frozenset({"build_environment", "package_vcs_metadata", "unknown"}) + + +def _safe_label(value: object, fallback: str = "unknown") -> str: + candidate = value if isinstance(value, str) else "" + candidate = candidate.strip() + if ( + _SAFE_LABEL.fullmatch(candidate) + and "://" not in candidate + and "@" not in candidate + and not looks_like_credential(candidate) + ): + return candidate + return fallback + + +def _safe_optional_label(value: object) -> str | None: + """Return a safe label or ``None`` without inventing a placeholder.""" + label = _safe_label(value, fallback="") + return label or None + + +def _safe_deployment(value: object) -> str | None: + """Deployment names have a narrower alphabet than namespaced model IDs.""" + candidate = _safe_optional_label(value) + return candidate if candidate and _SAFE_DEPLOYMENT.fullmatch(candidate) else None + + +def _safe_api_version(value: object) -> str | None: + """Azure API versions are dates with an optional preview suffix.""" + candidate = value.strip() if isinstance(value, str) else "" + return candidate if _SAFE_API_VERSION.fullmatch(candidate) else None + + +def _capture_source_revision() -> tuple[str, str]: + """Return an injected/packaged VCS identity without invoking Git.""" + injected = os.environ.get("SKILLSPECTOR_BUILD_REVISION", "").strip() + if _SOURCE_REVISION.fullmatch(injected): + return injected.lower(), "build_environment" + + try: + direct_url = distribution("skillspector").read_text("direct_url.json") + metadata = json.loads(direct_url) if direct_url else {} + except (json.JSONDecodeError, OSError, TypeError): + metadata = {} + vcs_info = metadata.get("vcs_info") if isinstance(metadata, Mapping) else None + vcs_info = vcs_info if isinstance(vcs_info, Mapping) else {} + packaged = vcs_info.get("commit_id") + if isinstance(packaged, str) and _SOURCE_REVISION.fullmatch(packaged.strip()): + return packaged.strip().lower(), "package_vcs_metadata" + return "unknown", "unknown" + + +def _capture_provider_routing(resolved_adapter: str) -> dict[str, object]: + """Capture non-secret provider routing inputs used by client construction.""" + if resolved_adapter != "azure_openai": + return { + "deployment_override": None, + "deployment_source": "not_applicable", + "api_version": None, + "api_version_source": "not_applicable", + } + + raw_deployment = os.environ.get("AZURE_OPENAI_DEPLOYMENT", "").strip() + deployment = _safe_deployment(raw_deployment) + raw_api_version = os.environ.get("AZURE_OPENAI_API_VERSION", "").strip() + api_version = _safe_api_version(raw_api_version or _AZURE_OPENAI_DEFAULT_API_VERSION) + return { + "deployment_override": deployment, + "deployment_source": ( + "environment" if deployment else "unknown" if raw_deployment else "resolved_model" + ), + "api_version": api_version, + "api_version_source": ( + "environment" + if raw_api_version and api_version + else "unknown" + if raw_api_version + else "provider_default" + ), + } + + +def _requested_temperature(raw: str) -> tuple[float | None, str]: + if not raw: + return None, "provider_default" + try: + value = resolve_temperature() + except ValueError: + return None, "invalid_environment" + if value is None: + return None, "provider_default" + return value, "environment" + + +def _requested_seed(raw: str) -> tuple[int | None, str]: + if not raw: + return None, "unset" + try: + value = resolve_seed() + except ValueError: + return None, "out_of_range" if raw.lstrip("+-").isdigit() else "invalid_environment" + if value is None: + return None, "unset" + return value, "environment" + + +def _requested_effort(raw: str) -> tuple[str | None, str]: + if not raw: + return None, "provider_default" + value = resolve_reasoning_effort() + value = safe_reasoning_effort(value) + return (value, "environment") if value else (None, "invalid_environment") + + +def _control( + requested: float | int | str | None, + source: str, + *, + adapter_support: bool, +) -> dict[str, object]: + return { + "requested": requested, + "source": source, + # Client forwarding is confirmed later from provider-response telemetry. + # Configuration capture alone cannot prove that an analyzer made a call. + "forwarded_to_client": None, + "adapter_support": adapter_support, + # SkillSpector can prove what it passed to the client constructor, not + # whether a gateway/model honored the optional control. + "provider_support": "unknown", + } + + +def capture_llm_provenance(model_config: Mapping[str, object]) -> dict[str, object]: + """Capture resolved LLM configuration once, before analyzer execution.""" + configured_adapter = provider_name(get_active_provider()) + # This provider resolves model defaults before any client is constructed. + # Runtime response telemetry supplies the actual effective provider later. + resolved_adapter = provider_name(get_model_config_provider()) + package_version = version("skillspector") + source_revision, source_revision_source = _capture_source_revision() + + temperature, temperature_source = _requested_temperature( + os.environ.get("SKILLSPECTOR_TEMPERATURE", "").strip() + ) + seed, seed_source = _requested_seed(os.environ.get("SKILLSPECTOR_SEED", "").strip()) + effort, effort_source = _requested_effort( + os.environ.get("SKILLSPECTOR_REASONING_EFFORT", "").strip() + ) + + analyzers = [ + { + "analyzer_id": slot, + "model": _safe_label(model_config.get(slot), fallback="redacted"), + "model_source": "resolved_configuration", + "analyzer_revision": { + "value": package_version, + "source": "skillspector_package", + "source_revision": { + "value": source_revision, + "source": source_revision_source, + }, + }, + } + for slot in LLM_ANALYZER_SLOTS + ] + sampling = { + "temperature": _control( + temperature, + temperature_source, + adapter_support=resolved_adapter in _TEMPERATURE_ADAPTERS, + ), + "seed": _control( + seed, + seed_source, + adapter_support=resolved_adapter in _SEED_ADAPTERS, + ), + "reasoning_effort": _control( + effort, + effort_source, + adapter_support=resolved_adapter in _REASONING_EFFORT_ADAPTERS, + ), + } + return { + "schema_version": LLM_PROVENANCE_SCHEMA_VERSION, + "provider": { + "configured_adapter": _safe_label(configured_adapter), + "resolved_adapter": _safe_label(resolved_adapter), + "routing": _capture_provider_routing(resolved_adapter), + "service": "unknown", + }, + "analyzers": analyzers, + "sampling": sampling, + } + + +def _sanitize_observed_control_values( + name: str, + values: Sequence[object], +) -> tuple[list[float | int | str | None], bool]: + observed: list[float | int | str | None] = [] + invalid = False + for candidate in values: + if candidate is None: + observed.append(None) + elif name == "temperature": + if ( + isinstance(candidate, (int, float)) + and not isinstance(candidate, bool) + and math.isfinite(candidate) + and 0 <= candidate <= 1 + ): + observed.append(float(candidate)) + else: + invalid = True + elif name == "seed": + if ( + isinstance(candidate, int) + and not isinstance(candidate, bool) + and _MIN_SAFE_SEED <= candidate <= _MAX_SAFE_SEED + ): + observed.append(candidate) + else: + invalid = True + else: + setting = safe_reasoning_effort(candidate) + if setting: + observed.append(setting) + else: + invalid = True + return observed, invalid + + +def _sanitize_control( + name: str, + value: object, + *, + use_llm: bool, + effective_adapters: Sequence[str], + observed_requested_values: Sequence[object], + observed_forwarded_values: Sequence[object], +) -> dict[str, object]: + raw = value if isinstance(value, Mapping) else {} + source = raw.get("source") + source = source if isinstance(source, str) and source in _CONTROL_SOURCES else "unknown" + adapter_support = raw.get("adapter_support") + adapter_support = adapter_support if isinstance(adapter_support, bool) else False + if effective_adapters: + supported_adapters = ( + _TEMPERATURE_ADAPTERS + if name == "temperature" + else _SEED_ADAPTERS + if name == "seed" + else _REASONING_EFFORT_ADAPTERS + ) + adapter_support = all(adapter in supported_adapters for adapter in effective_adapters) + + raw_requested = raw.get("requested") + requested = raw_requested + if name == "temperature": + requested = ( + float(requested) + if isinstance(requested, (int, float)) + and not isinstance(requested, bool) + and math.isfinite(requested) + and 0 <= requested <= 1 + else None + ) + elif name == "seed": + requested = ( + requested + if isinstance(requested, int) + and not isinstance(requested, bool) + and _MIN_SAFE_SEED <= requested <= _MAX_SAFE_SEED + else None + ) + else: + requested = safe_reasoning_effort(requested) + if source == "environment" and requested is None: + source = ( + "out_of_range" + if name == "seed" + and isinstance(raw_requested, int) + and not isinstance(raw_requested, bool) + else "invalid_environment" + ) + if source != "environment": + requested = None + + # The constructor-time request supersedes the provisional preflight + # capture. Forwarding is derived separately from the normalized provider + # payload because a model adapter may accept and then omit a control. + forwarded: float | int | str | None = None + if use_llm and adapter_support and observed_requested_values: + observed, invalid_observation = _sanitize_observed_control_values( + name, observed_requested_values + ) + distinct = [] + for candidate in observed: + if candidate not in distinct: + distinct.append(candidate) + if ( + invalid_observation + or len(observed) != len(observed_requested_values) + or len(distinct) != 1 + ): + requested = None + source = "unknown" + else: + actual = distinct[0] + if actual is None: + requested = None + source = "unset" if name == "seed" else "provider_default" + else: + requested = actual + source = "environment" + + if use_llm and adapter_support and observed_forwarded_values: + observed, invalid_observation = _sanitize_observed_control_values( + name, observed_forwarded_values + ) + distinct = [] + for candidate in observed: + if candidate not in distinct: + distinct.append(candidate) + if ( + not invalid_observation + and len(observed) == len(observed_forwarded_values) + and len(distinct) == 1 + and distinct[0] is not None + ): + forwarded = distinct[0] + + return { + "requested": requested, + "source": source, + "forwarded_to_client": forwarded, + "adapter_support": adapter_support, + "provider_support": "unknown", + } + + +def _sanitize_provider_routing(value: object, *, resolved_adapter: str) -> dict[str, object]: + """Return the fixed routing contract without endpoints or credentials.""" + if resolved_adapter == "unknown": + return { + "deployment_override": None, + "deployment_source": "unknown", + "api_version": None, + "api_version_source": "unknown", + } + if resolved_adapter != "azure_openai": + return { + "deployment_override": None, + "deployment_source": "not_applicable", + "api_version": None, + "api_version_source": "not_applicable", + } + + raw = value if isinstance(value, Mapping) else {} + deployment = _safe_deployment(raw.get("deployment_override")) + deployment_source = raw.get("deployment_source") + if not isinstance(deployment_source, str) or deployment_source not in _DEPLOYMENT_SOURCES: + deployment_source = "unknown" + if deployment_source == "environment" and deployment is None: + deployment_source = "unknown" + elif deployment_source != "environment": + deployment = None + + api_version = _safe_api_version(raw.get("api_version")) + api_version_source = raw.get("api_version_source") + if not isinstance(api_version_source, str) or api_version_source not in _API_VERSION_SOURCES: + api_version_source = "unknown" + if api_version_source in {"environment", "provider_default"} and api_version is None: + api_version_source = "unknown" + elif api_version_source not in {"environment", "provider_default"}: + api_version = None + + return { + "deployment_override": deployment, + "deployment_source": deployment_source, + "api_version": api_version, + "api_version_source": api_version_source, + } + + +def _provider_response_records(inference_usage: object) -> list[Mapping[object, object]]: + """Return internal provider-response evidence, including counter-less calls.""" + records = inference_usage if isinstance(inference_usage, Sequence) else [] + return [ + record + for record in records + if isinstance(record, Mapping) and record.get("usage_source") == "provider_response" + ] + + +def _effective_adapters(records: Sequence[Mapping[object, object]]) -> list[str]: + """Return providers proven by provider-response telemetry.""" + adapters = { + adapter + for record in records + if (adapter := _safe_optional_label(record.get("provider"))) is not None + } + return sorted(adapters) + + +def _observed_controls( + records: Sequence[Mapping[object, object]], + field: str, +) -> dict[str, list[object]]: + """Collect fixed-field constructor controls from successful calls.""" + observed = {name: [] for name in ("temperature", "seed", "reasoning_effort")} + for record in records: + controls = record.get(field) + if not isinstance(controls, Mapping): + continue + for name in observed: + if name in controls: + observed[name].append(controls.get(name)) + return observed + + +def _expected_observed_controls(effective_adapters: Sequence[str]) -> set[str]: + """Return controls whose constructor state is observable for all adapters.""" + expected: set[str] = set() + for name, supported in ( + ("temperature", _TEMPERATURE_ADAPTERS), + ("seed", _SEED_ADAPTERS), + ("reasoning_effort", _REASONING_EFFORT_ADAPTERS), + ): + if effective_adapters and all(adapter in supported for adapter in effective_adapters): + expected.add(name) + return expected + + +def sanitize_llm_provenance( + value: object, + *, + use_llm: bool, + inference_usage: object = None, +) -> dict[str, object]: + """Return the fixed public provenance projection without arbitrary state.""" + raw = value if isinstance(value, Mapping) else {} + raw_provider = raw.get("provider") + provider = raw_provider if isinstance(raw_provider, Mapping) else {} + configured_adapter = _safe_label(provider.get("configured_adapter")) + resolved_adapter = _safe_label(provider.get("resolved_adapter")) + response_records = _provider_response_records(inference_usage) if use_llm else [] + effective_adapters = _effective_adapters(response_records) + observed_requested_controls = _observed_controls(response_records, "requested_controls") + observed_forwarded_controls = _observed_controls(response_records, "forwarded_controls") + effective_adapter = ( + "not_applicable" + if not use_llm + else effective_adapters[0] + if len(effective_adapters) == 1 + else "mixed" + if effective_adapters + else "unknown" + ) + + raw_analyzers = raw.get("analyzers") + by_id: dict[str, Mapping[object, object]] = {} + if isinstance(raw_analyzers, list): + for item in raw_analyzers: + if not isinstance(item, Mapping): + continue + analyzer_id = item.get("analyzer_id") + if isinstance(analyzer_id, str) and analyzer_id in LLM_ANALYZER_SLOTS: + by_id[analyzer_id] = item + analyzers: list[dict[str, object]] = [] + for slot in LLM_ANALYZER_SLOTS: + item = by_id.get(slot) + item = item if isinstance(item, Mapping) else {} + revision = item.get("analyzer_revision") + revision = revision if isinstance(revision, Mapping) else {} + raw_source_revision = revision.get("source_revision") + raw_source_revision = ( + raw_source_revision if isinstance(raw_source_revision, Mapping) else {} + ) + source_revision = raw_source_revision.get("value") + source_revision = ( + source_revision.lower() + if isinstance(source_revision, str) and _SOURCE_REVISION.fullmatch(source_revision) + else "unknown" + ) + source_revision_source = raw_source_revision.get("source") + if ( + not isinstance(source_revision_source, str) + or source_revision_source not in _SOURCE_REVISION_SOURCES + or source_revision == "unknown" + ): + source_revision_source = "unknown" + analyzers.append( + { + "analyzer_id": slot, + "model": _safe_label(item.get("model"), fallback="redacted"), + "model_source": "resolved_configuration", + "analyzer_revision": { + "value": _safe_label(revision.get("value")), + "source": "skillspector_package", + "source_revision": { + "value": source_revision, + "source": source_revision_source, + }, + }, + } + ) + + raw_sampling = raw.get("sampling") + sampling = raw_sampling if isinstance(raw_sampling, Mapping) else {} + sanitized_sampling = { + name: _sanitize_control( + name, + sampling.get(name), + use_llm=use_llm, + effective_adapters=effective_adapters, + observed_requested_values=observed_requested_controls[name], + observed_forwarded_values=observed_forwarded_controls[name], + ) + for name in ("temperature", "seed", "reasoning_effort") + } + expected_observations = _expected_observed_controls(effective_adapters) + controls_observed = all( + all( + isinstance(record.get("requested_controls"), Mapping) + and name in record.get("requested_controls", {}) + and isinstance(record.get("forwarded_controls"), Mapping) + and name in record.get("forwarded_controls", {}) + for record in response_records + ) + for name in expected_observations + ) + requested_controls = [ + control for control in sanitized_sampling.values() if control["source"] == "environment" + ] + if not use_llm: + control_status = "not_applied" + elif any( + control["source"] in {"invalid_environment", "out_of_range"} + for control in sanitized_sampling.values() + ): + control_status = "invalid_configuration" + elif any(control["source"] == "unknown" for control in sanitized_sampling.values()): + control_status = "configuration_unknown" + elif not effective_adapters or not controls_observed: + control_status = "controls_not_observed" + elif not requested_controls: + control_status = "provider_defaults" + elif all(control["forwarded_to_client"] is not None for control in requested_controls): + control_status = "best_effort_controls_forwarded" + else: + control_status = "controls_partially_forwarded" + + return { + "schema_version": LLM_PROVENANCE_SCHEMA_VERSION, + "provider": { + "configured_adapter": configured_adapter, + "resolved_adapter": resolved_adapter, + "effective_adapter": effective_adapter, + "effective_adapters": effective_adapters, + "service": "unknown", + "routing": _sanitize_provider_routing( + provider.get("routing"), resolved_adapter=resolved_adapter + ), + }, + "analyzers": analyzers, + "sampling": sanitized_sampling, + "determinism": { + "classification": "nondeterministic" if use_llm else "not_applicable", + "control_status": control_status, + "provider_guarantee": False, + "reason": ( + "Optional controls do not guarantee identical provider output." + if use_llm + else "LLM analysis was not executed for this scan." + ), + }, + } diff --git a/src/skillspector/llm_utils.py b/src/skillspector/llm_utils.py index e9924498a..c8f4d6872 100644 --- a/src/skillspector/llm_utils.py +++ b/src/skillspector/llm_utils.py @@ -46,7 +46,12 @@ from langchain_core.language_models.chat_models import BaseChatModel from langchain_core.runnables import Runnable -from skillspector.inference_usage import InferenceUsageCollector, provider_name +from skillspector.inference_usage import ( + InferenceUsageCollector, + chat_model_controls, + chat_model_requested_controls, + provider_name, +) from skillspector.model_info import get_max_input_tokens, get_max_output_tokens from skillspector.providers import ( create_chat_model, @@ -444,6 +449,8 @@ def new_inference_usage_collector( request_kind=request_kind, provider=effective_provider, requested_model=model, + requested_controls=chat_model_requested_controls(chat_model), + forwarded_controls=chat_model_controls(chat_model), ) @@ -480,8 +487,13 @@ def chat_completion( chat_model=chat_model, ) effective_provider = chat_model_provider_name(chat_model) - if usage_collector is not None and effective_provider is not None: - collector.set_provider(effective_provider) + if usage_collector is not None: + if effective_provider is not None: + collector.set_provider(effective_provider) + collector.set_controls( + chat_model_requested_controls(chat_model), + chat_model_controls(chat_model), + ) response = _invoke_with_usage(chat_model, prompt, collector) if hasattr(response, "text"): return response.text # type: ignore[union-attr] diff --git a/src/skillspector/nodes/build_context.py b/src/skillspector/nodes/build_context.py index d45239f1e..c7ee8fd47 100644 --- a/src/skillspector/nodes/build_context.py +++ b/src/skillspector/nodes/build_context.py @@ -60,6 +60,7 @@ LedgerRecordType, ledger_event, ) +from skillspector.llm_provenance import capture_llm_provenance from skillspector.logging_config import get_logger from skillspector.nested_artifacts import ( expected_container_type, @@ -3281,6 +3282,7 @@ def _mark_runtime_partial(affected_paths: list[str], first_limited_path: str) -> or any(bool(metadata.get("executable")) for metadata in excluded_component_metadata) ) + model_config = build_model_config() result: dict[str, object] = { "components": components, "llm_components": llm_components, @@ -3313,7 +3315,8 @@ def _mark_runtime_partial(affected_paths: list[str], first_limited_path: str) -> "python_ast_cache_key": python_ast_cache_key, "manifest": manifest, "previous_manifest": None, - "model_config": build_model_config(), + "model_config": model_config, + "llm_provenance": capture_llm_provenance(model_config), "component_metadata": component_metadata, "has_executable_scripts": has_executable_scripts, "workflow_resource_budget": workflow_budget, diff --git a/src/skillspector/nodes/report.py b/src/skillspector/nodes/report.py index 14150a26d..9269343b5 100644 --- a/src/skillspector/nodes/report.py +++ b/src/skillspector/nodes/report.py @@ -42,6 +42,7 @@ AnalysisCompleteness, finalize_ledger, ) +from skillspector.llm_provenance import sanitize_llm_provenance from skillspector.llm_utils import is_llm_available from skillspector.logging_config import get_logger from skillspector.models import Finding @@ -1105,6 +1106,7 @@ def _build_metadata( use_llm: bool, llm_call_log: Sequence[Mapping[str, object]] | None = None, inference_usage: Sequence[Mapping[str, object]] | None = None, + llm_provenance: object = None, transitive_targets_scanned: int | None = None, transitive_bytes_scanned: int | None = None, transitive_truncation_reasons: Sequence[str] | None = None, @@ -1148,10 +1150,19 @@ def _build_metadata( # some coverage was lost) into one boolean. execution_enabled = use_llm if llm_execution_enabled is None else llm_execution_enabled unavailable_before_execution = bool(use_llm and not execution_enabled) + response_observed = any( + isinstance(record, Mapping) and record.get("usage_source") == "provider_response" + for record in inference_usage or [] + ) + # Enablement alone does not prove execution: every analyzer may have + # returned not_applicable. Failed attempts still count, as do successful + # provider responses whose transport supplied no token counters. + llm_executed = bool(use_llm and execution_enabled and (attempted or response_observed)) meta_analysis_applied = ( use_llm and execution_enabled and provider_available and meta_analyzer_succeeded ) + sanitized_inference_usage = sanitize_inference_usage(inference_usage) meta: dict[str, object] = { "has_executable_scripts": has_executable_scripts, "skillspector_version": skillspector_version, @@ -1163,7 +1174,16 @@ def _build_metadata( # A list (including an empty list) makes observability explicit. Empty # means the provider/transport supplied no counters; it is never an # estimated zero-cost assertion. - "inference_usage": sanitize_inference_usage(inference_usage), + "inference_usage": sanitized_inference_usage, + "llm_provenance": sanitize_llm_provenance( + llm_provenance, + use_llm=llm_executed, + # Counter-less responses and constructor controls are internal + # provenance evidence. The public inference_usage projection above + # intentionally omits both, so provenance must inspect the raw + # records and apply its own fixed-field sanitizer. + inference_usage=inference_usage, + ), } if not meta_analysis_applied: meta["filtering_mode"] = "heuristic" @@ -1220,6 +1240,7 @@ def _format_json( use_llm: bool = True, llm_call_log: Sequence[Mapping[str, object]] | None = None, inference_usage: Sequence[Mapping[str, object]] | None = None, + llm_provenance: object = None, analysis_completeness: Mapping[str, object] | None = None, suppressed: list[SuppressedFinding] | None = None, execution_successful: bool = True, @@ -1268,6 +1289,7 @@ def _format_json( use_llm, llm_call_log, inference_usage, + llm_provenance, transitive_targets_scanned, transitive_bytes_scanned, transitive_truncation_reasons, @@ -1532,6 +1554,7 @@ def report(state: SkillspectorState) -> dict[str, object]: else [] ) inference_usage = state.get("inference_usage") or [] + llm_provenance = state.get("llm_provenance") transitive_targets_scanned = state.get("transitive_targets_scanned") transitive_bytes_scanned = state.get("transitive_bytes_scanned") transitive_truncation_reasons = [ @@ -1711,6 +1734,7 @@ def report(state: SkillspectorState) -> dict[str, object]: use_llm=llm_requested, llm_call_log=llm_call_log, inference_usage=inference_usage, + llm_provenance=llm_provenance, analysis_completeness=analysis_completeness, suppressed=suppressed, execution_successful=execution_successful, diff --git a/src/skillspector/providers/anthropic/provider.py b/src/skillspector/providers/anthropic/provider.py index 4ac4a1540..310b5f132 100644 --- a/src/skillspector/providers/anthropic/provider.py +++ b/src/skillspector/providers/anthropic/provider.py @@ -31,6 +31,10 @@ from langchain_core.language_models.chat_models import BaseChatModel from pydantic import SecretStr +from skillspector.inference_usage import ( + register_chat_model_controls, + retained_chat_model_controls, +) from skillspector.providers import registry from skillspector.providers.chat_models import resolve_reasoning_effort, resolve_sampling_parameters @@ -80,8 +84,21 @@ def create_chat_model( effort = resolve_reasoning_effort() if effort is not None: kwargs["effort"] = effort - kwargs.update(resolve_sampling_parameters()) - return ChatAnthropic(**kwargs) + sampling_parameters = resolve_sampling_parameters() + kwargs.update(sampling_parameters) + chat_model = ChatAnthropic(**kwargs) + register_chat_model_controls( + chat_model, + retained_chat_model_controls( + chat_model, + ("temperature", "reasoning_effort"), + ), + requested_controls={ + "temperature": sampling_parameters.get("temperature"), + "reasoning_effort": effort, + }, + ) + return chat_model def get_context_length(self, model: str) -> int | None: return registry.lookup_context_length(REGISTRY_PATH, model) diff --git a/src/skillspector/providers/anthropic_proxy/provider.py b/src/skillspector/providers/anthropic_proxy/provider.py index 364fcb7e0..e3f206020 100644 --- a/src/skillspector/providers/anthropic_proxy/provider.py +++ b/src/skillspector/providers/anthropic_proxy/provider.py @@ -52,6 +52,10 @@ from langchain_core.language_models.chat_models import BaseChatModel from pydantic import SecretStr +from skillspector.inference_usage import ( + register_chat_model_controls, + retained_chat_model_controls, +) from skillspector.providers import registry from skillspector.providers.chat_models import resolve_reasoning_effort, resolve_sampling_parameters @@ -244,8 +248,21 @@ def create_chat_model( effort = resolve_reasoning_effort() if effort is not None: kwargs["effort"] = effort - kwargs.update(resolve_sampling_parameters()) - return _ChatAnthropicProxy(**kwargs) + sampling_parameters = resolve_sampling_parameters() + kwargs.update(sampling_parameters) + chat_model = _ChatAnthropicProxy(**kwargs) + register_chat_model_controls( + chat_model, + retained_chat_model_controls( + chat_model, + ("temperature", "reasoning_effort"), + ), + requested_controls={ + "temperature": sampling_parameters.get("temperature"), + "reasoning_effort": effort, + }, + ) + return chat_model def get_context_length(self, model: str) -> int | None: return registry.lookup_context_length(REGISTRY_PATH, model) diff --git a/src/skillspector/providers/azure_openai/provider.py b/src/skillspector/providers/azure_openai/provider.py index 0d1653bd8..ff556bcb4 100644 --- a/src/skillspector/providers/azure_openai/provider.py +++ b/src/skillspector/providers/azure_openai/provider.py @@ -36,6 +36,10 @@ from langchain_openai import AzureChatOpenAI from pydantic import SecretStr +from skillspector.inference_usage import ( + register_chat_model_controls, + retained_chat_model_controls, +) from skillspector.providers import registry from skillspector.providers.chat_models import resolve_sampling_parameters @@ -80,8 +84,18 @@ def create_chat_model( "max_tokens": max_tokens, "timeout": timeout, } - kwargs.update(resolve_sampling_parameters(include_seed=True)) - return AzureChatOpenAI(**kwargs) + sampling_parameters = resolve_sampling_parameters(include_seed=True) + kwargs.update(sampling_parameters) + chat_model = AzureChatOpenAI(**kwargs) + register_chat_model_controls( + chat_model, + retained_chat_model_controls(chat_model, ("temperature", "seed")), + requested_controls={ + "temperature": sampling_parameters.get("temperature"), + "seed": sampling_parameters.get("seed"), + }, + ) + return chat_model def get_context_length(self, model: str) -> int | None: return registry.lookup_context_length(REGISTRY_PATH, model) diff --git a/src/skillspector/providers/bedrock/provider.py b/src/skillspector/providers/bedrock/provider.py index 1e7a6ff0e..b954279c2 100644 --- a/src/skillspector/providers/bedrock/provider.py +++ b/src/skillspector/providers/bedrock/provider.py @@ -40,6 +40,10 @@ from langchain_aws import ChatBedrockConverse from langchain_core.language_models.chat_models import BaseChatModel +from skillspector.inference_usage import ( + register_chat_model_controls, + retained_chat_model_controls, +) from skillspector.providers import registry from skillspector.providers.chat_models import resolve_sampling_parameters @@ -130,9 +134,16 @@ def create_chat_model( } if model.startswith("arn:"): kwargs["provider"] = "anthropic" - kwargs.update(resolve_sampling_parameters()) - - return ChatBedrockConverse(**kwargs) + sampling_parameters = resolve_sampling_parameters() + kwargs.update(sampling_parameters) + + chat_model = ChatBedrockConverse(**kwargs) + register_chat_model_controls( + chat_model, + retained_chat_model_controls(chat_model, ("temperature",)), + requested_controls={"temperature": sampling_parameters.get("temperature")}, + ) + return chat_model def get_context_length(self, model: str) -> int | None: return registry.lookup_context_length(REGISTRY_PATH, model) diff --git a/src/skillspector/providers/chat_models.py b/src/skillspector/providers/chat_models.py index aa13d44ae..d16912af2 100644 --- a/src/skillspector/providers/chat_models.py +++ b/src/skillspector/providers/chat_models.py @@ -25,7 +25,14 @@ from langchain_openai import ChatOpenAI from pydantic import SecretStr +from skillspector.inference_usage import ( + register_chat_model_controls, + retained_chat_model_controls, +) + logger = logging.getLogger(__name__) +MIN_SAMPLING_SEED = -(1 << 63) +MAX_SAMPLING_SEED = (1 << 63) - 1 def resolve_reasoning_effort() -> str | None: @@ -34,25 +41,44 @@ def resolve_reasoning_effort() -> str | None: return reasoning_effort or None +def resolve_temperature() -> float | None: + """Resolve the optional temperature using the provider validation contract.""" + raw_temperature = os.environ.get("SKILLSPECTOR_TEMPERATURE", "").strip() + if not raw_temperature: + return None + try: + temperature = float(raw_temperature) + except ValueError as exc: + raise ValueError("SKILLSPECTOR_TEMPERATURE must be a number between 0 and 1") from exc + if not 0 <= temperature <= 1: + raise ValueError("SKILLSPECTOR_TEMPERATURE must be between 0 and 1") + return temperature + + +def resolve_seed() -> int | None: + """Resolve the optional seed within the portable signed 64-bit contract.""" + raw_seed = os.environ.get("SKILLSPECTOR_SEED", "").strip() + if not raw_seed: + return None + try: + seed = int(raw_seed) + except ValueError as exc: + raise ValueError("SKILLSPECTOR_SEED must be an integer") from exc + if not MIN_SAMPLING_SEED <= seed <= MAX_SAMPLING_SEED: + raise ValueError("SKILLSPECTOR_SEED must be a signed 64-bit integer") + return seed + + def resolve_sampling_parameters(*, include_seed: bool = False) -> dict[str, float | int]: """Resolve optional, validated sampling controls for hosted providers.""" parameters: dict[str, float | int] = {} - raw_temperature = os.environ.get("SKILLSPECTOR_TEMPERATURE", "").strip() - if raw_temperature: - try: - temperature = float(raw_temperature) - except ValueError as exc: - raise ValueError("SKILLSPECTOR_TEMPERATURE must be a number between 0 and 1") from exc - if not 0 <= temperature <= 1: - raise ValueError("SKILLSPECTOR_TEMPERATURE must be between 0 and 1") + temperature = resolve_temperature() + if temperature is not None: parameters["temperature"] = temperature - raw_seed = os.environ.get("SKILLSPECTOR_SEED", "").strip() - if include_seed and raw_seed: - try: - parameters["seed"] = int(raw_seed) - except ValueError as exc: - raise ValueError("SKILLSPECTOR_SEED must be an integer") from exc + seed = resolve_seed() if include_seed else None + if seed is not None: + parameters["seed"] = seed return parameters @@ -104,5 +130,19 @@ def create_openai_compatible_chat_model( reasoning_effort = resolve_reasoning_effort() if reasoning_effort: kwargs["reasoning_effort"] = reasoning_effort - kwargs.update(resolve_sampling_parameters(include_seed=True)) - return ChatOpenAI(**kwargs) + sampling_parameters = resolve_sampling_parameters(include_seed=True) + kwargs.update(sampling_parameters) + chat_model = ChatOpenAI(**kwargs) + register_chat_model_controls( + chat_model, + retained_chat_model_controls( + chat_model, + ("temperature", "seed", "reasoning_effort"), + ), + requested_controls={ + "temperature": sampling_parameters.get("temperature"), + "seed": sampling_parameters.get("seed"), + "reasoning_effort": reasoning_effort, + }, + ) + return chat_model diff --git a/src/skillspector/state.py b/src/skillspector/state.py index 1a887e76f..d395b6e24 100644 --- a/src/skillspector/state.py +++ b/src/skillspector/state.py @@ -304,6 +304,9 @@ class SkillspectorState(TypedDict, total=False): # Model IDs per LLM-using node: e.g. {"default": "...", "meta_analyzer": "..."} model_config: dict[str, str] + # Sanitized LLM configuration captured with model resolution. The report + # projects this independently from provider token-usage telemetry. + llm_provenance: dict[str, object] # Component metadata for reporting and risk scoring (from build_context) component_metadata: list[dict[str, object]] diff --git a/tests/nodes/test_report.py b/tests/nodes/test_report.py index 1be577f1f..f896171d8 100644 --- a/tests/nodes/test_report.py +++ b/tests/nodes/test_report.py @@ -1355,6 +1355,10 @@ def test_report_llm_degraded_when_all_calls_failed(monkeypatch: pytest.MonkeyPat assert meta["llm_degraded"] is True assert meta["llm_calls_attempted"] == 3 assert meta["llm_calls_succeeded"] == 0 + # Attempted calls remain execution evidence even when none returns a response. + assert meta["llm_provenance"]["provider"]["effective_adapter"] == "unknown" + assert meta["llm_provenance"]["determinism"]["classification"] == "nondeterministic" + assert meta["llm_provenance"]["determinism"]["control_status"] == "configuration_unknown" # Distinct error reasons are surfaced (deduped). assert "claude empty stdout" in meta["llm_error"] assert "static analysis only" in meta["llm_error"] @@ -1547,6 +1551,161 @@ def test_json_report_exposes_only_sanitized_provider_usage( ] +def test_json_report_exposes_captured_llm_provenance( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("skillspector.nodes.report.is_llm_available", lambda: (True, None)) + state: SkillspectorState = { + "filtered_findings": [], + "component_metadata": [], + "has_executable_scripts": False, + "manifest": {}, + "output_format": "json", + "use_llm": True, + "llm_call_log": [], + "inference_usage": [ + { + "node": "semantic_developer_intent", + "request_kind": "structured_output", + "provider": "openai", + "model": "safe/model:1", + "model_source": "requested_model", + "usage_source": "provider_response", + "total_tokens": 1, + "requested_controls": { + "temperature": 0.0, + "seed": 7, + "reasoning_effort": None, + }, + "forwarded_controls": { + "temperature": 0.0, + "seed": 7, + "reasoning_effort": None, + }, + } + ], + "llm_provenance": { + "provider": { + "configured_adapter": "anthropic", + "resolved_adapter": "openai", + "service": "private-service-name", + }, + "analyzers": [ + { + "analyzer_id": "semantic_developer_intent", + "model": "safe/model:1", + "analyzer_revision": {"value": "2.11.2", "prompt": "must not leak"}, + } + ], + "sampling": { + "temperature": { + "requested": 0.0, + "source": "environment", + "forwarded_to_client": 0.0, + "adapter_support": True, + }, + "seed": { + "requested": 7, + "source": "environment", + "forwarded_to_client": 7, + "adapter_support": True, + }, + }, + "endpoint": "https://private.example.test", + }, + } + + meta = _meta_from_json_report(state) + provenance = meta["llm_provenance"] + + assert provenance["provider"] == { + "configured_adapter": "anthropic", + "resolved_adapter": "openai", + "effective_adapter": "openai", + "effective_adapters": ["openai"], + "service": "unknown", + "routing": { + "deployment_override": None, + "deployment_source": "not_applicable", + "api_version": None, + "api_version_source": "not_applicable", + }, + } + intent = next( + item + for item in provenance["analyzers"] + if item["analyzer_id"] == "semantic_developer_intent" + ) + assert intent["model"] == "safe/model:1" + assert intent["analyzer_revision"] == { + "value": "2.11.2", + "source": "skillspector_package", + "source_revision": {"value": "unknown", "source": "unknown"}, + } + assert provenance["sampling"]["temperature"]["forwarded_to_client"] == 0.0 + assert provenance["sampling"]["seed"]["forwarded_to_client"] == 7 + assert provenance["determinism"]["classification"] == "nondeterministic" + assert "private.example" not in json.dumps(meta) + assert "must not leak" not in json.dumps(meta) + + +def test_json_report_preserves_counterless_cli_provider_observation( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """CLI success remains provenance evidence even without token counters.""" + monkeypatch.setattr("skillspector.nodes.report.is_llm_available", lambda: (True, None)) + state: SkillspectorState = { + "filtered_findings": [], + "component_metadata": [], + "has_executable_scripts": False, + "manifest": {}, + "output_format": "json", + "use_llm": True, + "llm_call_log": [], + "inference_usage": [ + { + "node": "semantic_developer_intent", + "request_kind": "structured_output", + "provider": "claude_cli", + "model": "claude-sonnet-4-6", + "model_source": "requested_model", + "usage_source": "provider_response", + "forwarded_controls": {}, + } + ], + "llm_provenance": { + "provider": { + "configured_adapter": "claude_cli", + "resolved_adapter": "claude_cli", + }, + "sampling": { + "temperature": { + "requested": None, + "source": "provider_default", + "adapter_support": False, + }, + "seed": { + "requested": None, + "source": "unset", + "adapter_support": False, + }, + "reasoning_effort": { + "requested": None, + "source": "provider_default", + "adapter_support": False, + }, + }, + }, + } + + meta = _meta_from_json_report(state) + + assert meta["inference_usage"] == [] + assert meta["llm_provenance"]["provider"]["effective_adapter"] == "claude_cli" + assert meta["llm_provenance"]["determinism"]["classification"] == "nondeterministic" + assert meta["llm_provenance"]["determinism"]["control_status"] == "provider_defaults" + + def test_report_no_llm_failures_not_counted_as_degraded(monkeypatch: pytest.MonkeyPatch) -> None: """use_llm False -> failures (if any) never mark the scan degraded.""" monkeypatch.setattr("skillspector.nodes.report.is_llm_available", lambda: (True, None)) @@ -2017,6 +2176,14 @@ def test_explicit_all_not_applicable_semantic_pass_stays_safe( assert result["risk_recommendation"] == "SAFE" assert "llm_degraded" not in metadata + assert "llm_calls_attempted" not in metadata + assert metadata["llm_provenance"]["provider"]["effective_adapter"] == "not_applicable" + assert metadata["llm_provenance"]["determinism"] == { + "classification": "not_applicable", + "control_status": "not_applied", + "provider_guarantee": False, + "reason": "LLM analysis was not executed for this scan.", + } def test_unavailable_provider_floors_recommendation_even_with_success_records( @@ -2158,13 +2325,35 @@ def test_preflight_unavailable_log_does_not_claim_runtime_calls_failed( "output_format": "json", "use_llm": False, "llm_requested": True, + # Stale/caller-supplied response evidence cannot turn a preflight- + # disabled scan into one that claims LLM execution. + "inference_usage": [ + { + "node": "semantic_developer_intent", + "request_kind": "structured_output", + "provider": "openai", + "model": "gpt-5.4", + "model_source": "requested_model", + "usage_source": "provider_response", + "total_tokens": 1, + "forwarded_controls": {"seed": 7}, + } + ], } with caplog.at_level(logging.WARNING, logger="skillspector.nodes.report"): - report(state) + result = report(state) assert "unavailable during preflight" in caplog.text assert "0/0" not in caplog.text + provenance = json.loads(result["report_body"])["metadata"]["llm_provenance"] + assert provenance["provider"]["effective_adapter"] == "not_applicable" + assert provenance["determinism"] == { + "classification": "not_applicable", + "control_status": "not_applied", + "provider_guarantee": False, + "reason": "LLM analysis was not executed for this scan.", + } def test_non_degraded_clean_scan_stays_safe() -> None: diff --git a/tests/nodes/test_report_provenance_boundary.py b/tests/nodes/test_report_provenance_boundary.py new file mode 100644 index 000000000..1d7681ccf --- /dev/null +++ b/tests/nodes/test_report_provenance_boundary.py @@ -0,0 +1,133 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Credential boundaries for captured and response-derived report provenance.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from skillspector.cli import app +from skillspector.llm_provenance import LLM_ANALYZER_SLOTS +from skillspector.nodes.report import report +from skillspector.state import SkillspectorState + +_JWT = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJzeW50aGV0aWMifQ.synthetic_signature" +_AUTHORIZATION = "Bearer synthetic-secret-token-123456" +_LONG_MODEL = "deepseek-ai/DeepSeek-R1-Distill-Qwen-32B" + + +def test_no_llm_cli_report_rejects_credentials_from_environment( + monkeypatch: pytest.MonkeyPatch, + safe_skill_dir: Path, + tmp_path: Path, +) -> None: + """Configuration capture must be safe even when no provider call is made.""" + provider = type("AzureOpenAIProvider", (), {})() + monkeypatch.setattr("skillspector.llm_provenance.get_active_provider", lambda: provider) + monkeypatch.setattr("skillspector.llm_provenance.get_model_config_provider", lambda: provider) + monkeypatch.setenv("SKILLSPECTOR_MODEL", _JWT) + monkeypatch.setenv("SKILLSPECTOR_REASONING_EFFORT", _AUTHORIZATION) + monkeypatch.setenv("AZURE_OPENAI_DEPLOYMENT", _JWT) + monkeypatch.setenv("AZURE_OPENAI_API_VERSION", _JWT) + output = tmp_path / "report.json" + + result = CliRunner().invoke( + app, + ["scan", str(safe_skill_dir), "--no-llm", "--format", "json", "--output", str(output)], + ) + + assert result.exit_code == 0, result.output + serialized = output.read_text() + provenance = json.loads(serialized)["metadata"]["llm_provenance"] + assert _JWT not in serialized + assert _AUTHORIZATION not in serialized + assert {item["model"] for item in provenance["analyzers"]} == {"redacted"} + assert provenance["sampling"]["reasoning_effort"]["requested"] is None + assert provenance["sampling"]["reasoning_effort"]["source"] == "invalid_environment" + assert provenance["provider"]["routing"]["deployment_override"] is None + assert provenance["provider"]["routing"]["api_version"] is None + + +@pytest.mark.parametrize("use_llm", [False, True]) +def test_report_rejects_credentials_in_raw_provenance_and_response_controls( + monkeypatch: pytest.MonkeyPatch, use_llm: bool +) -> None: + """Final serialization revalidates raw state instead of trusting capture.""" + monkeypatch.setattr("skillspector.nodes.report.is_llm_available", lambda: (True, None)) + state: SkillspectorState = { + "filtered_findings": [], + "component_metadata": [], + "has_executable_scripts": False, + "manifest": {}, + "output_format": "json", + "use_llm": use_llm, + "llm_call_log": [], + "inference_usage": [ + { + "node": "semantic_developer_intent", + "request_kind": "structured_output", + "provider": "openai", + "model": model, + "model_source": "provider_response", + "usage_source": "provider_response", + "total_tokens": 1, + "requested_controls": {"reasoning_effort": _AUTHORIZATION}, + "forwarded_controls": {"reasoning_effort": _AUTHORIZATION}, + } + for model in (_JWT, _LONG_MODEL) + ], + "llm_provenance": { + "provider": { + "configured_adapter": "azure_openai", + "resolved_adapter": "azure_openai", + "routing": { + "deployment_override": _JWT, + "deployment_source": "environment", + "api_version": _JWT, + "api_version_source": "environment", + }, + }, + "analyzers": [{"analyzer_id": slot, "model": _JWT} for slot in LLM_ANALYZER_SLOTS], + "sampling": { + "reasoning_effort": { + "requested": _AUTHORIZATION, + "source": "environment", + "adapter_support": True, + } + }, + }, + } + + serialized = report(state)["report_body"] + metadata = json.loads(serialized)["metadata"] + provenance = metadata["llm_provenance"] + + assert _JWT not in serialized + assert _AUTHORIZATION not in serialized + assert {item["model"] for item in provenance["analyzers"]} == {"redacted"} + assert provenance["sampling"]["reasoning_effort"]["requested"] is None + assert provenance["sampling"]["reasoning_effort"]["forwarded_to_client"] is None + assert provenance["provider"]["routing"]["deployment_override"] is None + assert provenance["provider"]["routing"]["api_version"] is None + assert [record["model"] for record in metadata["inference_usage"]] == [_LONG_MODEL] + + +def test_no_llm_cli_report_preserves_long_model_identifiers( + monkeypatch: pytest.MonkeyPatch, safe_skill_dir: Path, tmp_path: Path +) -> None: + monkeypatch.setenv("SKILLSPECTOR_MODEL", _LONG_MODEL) + output = tmp_path / "report.json" + + result = CliRunner().invoke( + app, + ["scan", str(safe_skill_dir), "--no-llm", "--format", "json", "--output", str(output)], + ) + + assert result.exit_code == 0, result.output + provenance = json.loads(output.read_text())["metadata"]["llm_provenance"] + assert {item["model"] for item in provenance["analyzers"]} == {_LONG_MODEL} diff --git a/tests/nodes/test_semantic_quality_policy.py b/tests/nodes/test_semantic_quality_policy.py index d22c9a0d3..46d7d5ccb 100644 --- a/tests/nodes/test_semantic_quality_policy.py +++ b/tests/nodes/test_semantic_quality_policy.py @@ -25,6 +25,7 @@ from langchain_core.outputs import ChatGeneration, LLMResult from langchain_core.runnables import Runnable, RunnableConfig +from skillspector.inference_usage import sanitize_inference_usage from skillspector.inspection_ledger import LedgerReason, finalize_ledger from skillspector.llm_analyzer_base import LLMAnalysisResult, LLMFinding from skillspector.llm_utils import AgentCLIChatModel @@ -339,7 +340,8 @@ def test_post_response_value_error_without_usage_records_partial_coverage(self) result = node({"file_cache": {"SKILL.md": "# Skill"}}) assert result["findings"] == [] - assert result["inference_usage"] == [] + assert result["inference_usage"][0]["usage_source"] == "provider_response" + assert sanitize_inference_usage(result["inference_usage"]) == [] assert result["inspection_ledger"] assert result["inspection_ledger"][0]["outcome"] == "skipped" assert result["inspection_ledger"][0]["reason_code"] == ( diff --git a/tests/unit/test_inference_usage.py b/tests/unit/test_inference_usage.py index 72b182804..83f886396 100644 --- a/tests/unit/test_inference_usage.py +++ b/tests/unit/test_inference_usage.py @@ -7,12 +7,16 @@ from types import SimpleNamespace +import pytest from langchain_core.messages import AIMessage from langchain_core.outputs import ChatGeneration, LLMResult from skillspector.inference_usage import ( InferenceUsageCollector, _usage_record, + chat_model_controls, + chat_model_requested_controls, + register_chat_model_controls, sanitize_inference_usage, ) @@ -69,7 +73,37 @@ def test_collector_marks_response_received_without_usage_counters() -> None: collector.on_llm_end(LLMResult(generations=[[ChatGeneration(message=message)]], llm_output={})) assert collector.response_received is True - assert collector.snapshot() == [] + observation = collector.snapshot() + assert observation[0]["provider"] == "codex_cli" + assert observation[0]["usage_source"] == "provider_response" + assert sanitize_inference_usage(observation) == [] + + +@pytest.mark.parametrize( + "effort", + ["github_pat_fake-value", "Bearer synthetic-secret-token-123456", "unrecognized setting"], +) +def test_constructor_control_registry_drops_credential_shaped_effort(effort: str) -> None: + class _ChatModel: + pass + + model = _ChatModel() + register_chat_model_controls( + model, + { + "temperature": 0.2, + "seed": 7, + "reasoning_effort": effort, + }, + requested_controls={ + "temperature": 0.2, + "seed": 7, + "reasoning_effort": effort, + }, + ) + + assert chat_model_controls(model) == {"temperature": 0.2, "seed": 7} + assert chat_model_requested_controls(model) == {"temperature": 0.2, "seed": 7} def test_raw_anthropic_usage_adds_external_cache_counters_to_prompt_total() -> None: @@ -364,6 +398,9 @@ def test_report_sanitizer_rejects_url_and_userinfo_model_labels() -> None: [ {**common, "model": "https://key@private-host/v1"}, {**common, "model": "key@private-host"}, + {**common, "model": "github_pat_fake-value"}, + {**common, "model": "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJzeW50aGV0aWMifQ.signature"}, + {**common, "model": "0123456789abcdef" * 2}, ] ) == [] diff --git a/tests/unit/test_llm_provenance.py b/tests/unit/test_llm_provenance.py new file mode 100644 index 000000000..9f36903ee --- /dev/null +++ b/tests/unit/test_llm_provenance.py @@ -0,0 +1,702 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for sanitized, scan-level LLM provenance.""" + +from __future__ import annotations + +import json + +import pytest + +from skillspector.inference_usage import provider_name +from skillspector.llm_provenance import ( + LLM_ANALYZER_SLOTS, + capture_llm_provenance, + sanitize_llm_provenance, +) +from skillspector.llm_utils import new_inference_usage_collector +from skillspector.providers.chat_models import create_openai_compatible_chat_model + + +class OpenAIProvider: + pass + + +class AnthropicProvider: + pass + + +class AzureOpenAIProvider: + pass + + +class NvInferenceProvider: + pass + + +def _models(value: str = "safe/model:1") -> dict[str, str]: + return dict.fromkeys(LLM_ANALYZER_SLOTS, value) + + +def _usage( + provider: str, + controls: dict[str, object] | None = None, +) -> list[dict[str, object]]: + record: dict[str, object] = { + "provider": provider, + "usage_source": "provider_response", + } + if controls is not None: + record["requested_controls"] = controls + record["forwarded_controls"] = controls + return [record] + + +def test_capture_records_resolved_adapters_models_and_forwarded_controls( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "skillspector.llm_provenance.get_active_provider", lambda: AnthropicProvider() + ) + monkeypatch.setattr( + "skillspector.llm_provenance.get_model_config_provider", lambda: OpenAIProvider() + ) + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0") + monkeypatch.setenv("SKILLSPECTOR_SEED", "0") + monkeypatch.setenv("SKILLSPECTOR_REASONING_EFFORT", "low") + + captured = capture_llm_provenance(_models()) + result = sanitize_llm_provenance( + captured, + use_llm=True, + inference_usage=_usage( + "openai", + {"temperature": 0.0, "seed": 0, "reasoning_effort": "low"}, + ), + ) + + assert result["provider"] == { + "configured_adapter": "anthropic", + "resolved_adapter": "openai", + "effective_adapter": "openai", + "effective_adapters": ["openai"], + "service": "unknown", + "routing": { + "deployment_override": None, + "deployment_source": "not_applicable", + "api_version": None, + "api_version_source": "not_applicable", + }, + } + assert [item["analyzer_id"] for item in result["analyzers"]] == list(LLM_ANALYZER_SLOTS) + assert {item["model"] for item in result["analyzers"]} == {"safe/model:1"} + assert {item["analyzer_revision"]["source"] for item in result["analyzers"]} == { + "skillspector_package" + } + assert result["sampling"]["temperature"]["forwarded_to_client"] == 0.0 + assert result["sampling"]["seed"]["forwarded_to_client"] == 0 + assert result["sampling"]["reasoning_effort"]["forwarded_to_client"] == "low" + assert result["determinism"] == { + "classification": "nondeterministic", + "control_status": "best_effort_controls_forwarded", + "provider_guarantee": False, + "reason": "Optional controls do not guarantee identical provider output.", + } + + +def test_editable_source_build_reports_unknown_source_revision( + monkeypatch: pytest.MonkeyPatch, +) -> None: + class _EditableDistribution: + @staticmethod + def read_text(name: str) -> str | None: + assert name == "direct_url.json" + return json.dumps( + { + "url": "file:///workspace/SkillSpector", + "dir_info": {"editable": True}, + } + ) + + monkeypatch.delenv("SKILLSPECTOR_BUILD_REVISION", raising=False) + monkeypatch.setattr( + "skillspector.llm_provenance.distribution", lambda _name: _EditableDistribution() + ) + monkeypatch.setattr("skillspector.llm_provenance.get_active_provider", lambda: OpenAIProvider()) + monkeypatch.setattr( + "skillspector.llm_provenance.get_model_config_provider", lambda: OpenAIProvider() + ) + + result = sanitize_llm_provenance( + capture_llm_provenance(_models()), + use_llm=False, + ) + + assert result["analyzers"][0]["analyzer_revision"]["source_revision"] == { + "value": "unknown", + "source": "unknown", + } + + +def test_injected_build_revision_is_reported_separately_from_package_version( + monkeypatch: pytest.MonkeyPatch, +) -> None: + revision = "ABCDEF0123456789ABCDEF0123456789ABCDEF01" + monkeypatch.setenv("SKILLSPECTOR_BUILD_REVISION", revision) + monkeypatch.setattr("skillspector.llm_provenance.get_active_provider", lambda: OpenAIProvider()) + monkeypatch.setattr( + "skillspector.llm_provenance.get_model_config_provider", lambda: OpenAIProvider() + ) + + result = sanitize_llm_provenance( + capture_llm_provenance(_models()), + use_llm=False, + ) + analyzer_revision = result["analyzers"][0]["analyzer_revision"] + + assert analyzer_revision["source"] == "skillspector_package" + assert analyzer_revision["source_revision"] == { + "value": revision.lower(), + "source": "build_environment", + } + + +def test_seed_is_requested_but_not_claimed_forwarded_for_anthropic( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "skillspector.llm_provenance.get_active_provider", lambda: AnthropicProvider() + ) + monkeypatch.setattr( + "skillspector.llm_provenance.get_model_config_provider", lambda: AnthropicProvider() + ) + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0.1") + monkeypatch.setenv("SKILLSPECTOR_SEED", "7") + + result = sanitize_llm_provenance( + capture_llm_provenance(_models()), + use_llm=True, + inference_usage=_usage( + "anthropic", + {"temperature": 0.1, "reasoning_effort": None}, + ), + ) + + assert result["sampling"]["temperature"]["forwarded_to_client"] == 0.1 + assert result["sampling"]["seed"] == { + "requested": 7, + "source": "environment", + "forwarded_to_client": None, + "adapter_support": False, + "provider_support": "unknown", + } + assert result["determinism"]["control_status"] == "controls_partially_forwarded" + + +def test_capture_is_stable_if_environment_changes_before_report( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("skillspector.llm_provenance.get_active_provider", lambda: OpenAIProvider()) + monkeypatch.setattr( + "skillspector.llm_provenance.get_model_config_provider", lambda: OpenAIProvider() + ) + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0.2") + monkeypatch.setenv("SKILLSPECTOR_SEED", "11") + captured = capture_llm_provenance(_models()) + + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0.9") + monkeypatch.setenv("SKILLSPECTOR_SEED", "99") + result = sanitize_llm_provenance( + captured, + use_llm=True, + inference_usage=_usage("openai"), + ) + + assert result["sampling"]["temperature"]["requested"] == 0.2 + assert result["sampling"]["seed"]["requested"] == 11 + + +def test_constructor_observation_supersedes_stale_configuration_capture( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """Forwarded controls come from the client constructor, not preflight state.""" + monkeypatch.setattr("skillspector.llm_provenance.get_active_provider", lambda: OpenAIProvider()) + monkeypatch.setattr( + "skillspector.llm_provenance.get_model_config_provider", lambda: OpenAIProvider() + ) + monkeypatch.setattr("skillspector.llm_utils.get_active_provider", lambda: OpenAIProvider()) + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0.2") + monkeypatch.setenv("SKILLSPECTOR_SEED", "11") + captured = capture_llm_provenance(_models("gpt-4o")) + + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0.9") + monkeypatch.setenv("SKILLSPECTOR_SEED", "99") + monkeypatch.setenv("SKILLSPECTOR_REASONING_EFFORT", "high") + + chat_model = create_openai_compatible_chat_model( + model="gpt-4o", + credentials=("test-key", None), + max_tokens=128, + ) + collector = new_inference_usage_collector( + node="semantic_developer_intent", + request_kind="structured_output", + model="gpt-4o", + chat_model=chat_model, + ) + collector.mark_response_received() + + result = sanitize_llm_provenance( + captured, + use_llm=True, + inference_usage=collector.snapshot(), + ) + + assert result["sampling"]["temperature"]["requested"] == 0.9 + assert result["sampling"]["temperature"]["forwarded_to_client"] == 0.9 + assert result["sampling"]["seed"]["requested"] == 99 + assert result["sampling"]["seed"]["forwarded_to_client"] == 99 + assert result["sampling"]["reasoning_effort"]["requested"] == "high" + assert result["sampling"]["reasoning_effort"]["forwarded_to_client"] == "high" + + +def test_gpt_5_4_reports_only_controls_retained_by_request_payload( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("skillspector.llm_provenance.get_active_provider", lambda: OpenAIProvider()) + monkeypatch.setattr( + "skillspector.llm_provenance.get_model_config_provider", lambda: OpenAIProvider() + ) + monkeypatch.setattr("skillspector.llm_utils.get_active_provider", lambda: OpenAIProvider()) + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0.2") + monkeypatch.setenv("SKILLSPECTOR_SEED", "11") + captured = capture_llm_provenance(_models("gpt-5.4")) + + chat_model = create_openai_compatible_chat_model( + model="gpt-5.4", + credentials=("test-key", None), + max_tokens=128, + ) + collector = new_inference_usage_collector( + node="semantic_developer_intent", + request_kind="structured_output", + model="gpt-5.4", + chat_model=chat_model, + ) + collector.mark_response_received() + + result = sanitize_llm_provenance( + captured, + use_llm=True, + inference_usage=collector.snapshot(), + ) + + assert result["sampling"]["temperature"] == { + "requested": 0.2, + "source": "environment", + "forwarded_to_client": None, + "adapter_support": True, + "provider_support": "unknown", + } + assert result["sampling"]["seed"]["requested"] == 11 + assert result["sampling"]["seed"]["forwarded_to_client"] == 11 + assert result["determinism"]["control_status"] == "controls_partially_forwarded" + + +def test_configured_controls_are_not_claimed_forwarded_without_response_evidence( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("skillspector.llm_provenance.get_active_provider", lambda: OpenAIProvider()) + monkeypatch.setattr( + "skillspector.llm_provenance.get_model_config_provider", lambda: OpenAIProvider() + ) + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0.2") + monkeypatch.setenv("SKILLSPECTOR_SEED", "11") + + result = sanitize_llm_provenance(capture_llm_provenance(_models()), use_llm=True) + + assert result["provider"]["effective_adapter"] == "unknown" + assert all(control["forwarded_to_client"] is None for control in result["sampling"].values()) + assert result["determinism"]["control_status"] == "controls_not_observed" + + +def test_sanitizer_drops_unverified_forwarding_without_response_evidence() -> None: + result = sanitize_llm_provenance( + { + "sampling": { + "temperature": { + "requested": 0.2, + "source": "environment", + "forwarded_to_client": 0.2, + "adapter_support": True, + }, + "seed": { + "requested": None, + "source": "unset", + "forwarded_to_client": None, + "adapter_support": True, + }, + "reasoning_effort": { + "requested": None, + "source": "provider_default", + "forwarded_to_client": None, + "adapter_support": True, + }, + } + }, + use_llm=True, + ) + + assert result["sampling"]["temperature"]["forwarded_to_client"] is None + assert result["determinism"]["control_status"] == "controls_not_observed" + + +def test_static_scan_is_explicitly_not_applicable(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr("skillspector.llm_provenance.get_active_provider", lambda: OpenAIProvider()) + monkeypatch.setattr( + "skillspector.llm_provenance.get_model_config_provider", lambda: OpenAIProvider() + ) + + result = sanitize_llm_provenance(capture_llm_provenance(_models()), use_llm=False) + + assert result["determinism"]["classification"] == "not_applicable" + assert result["determinism"]["control_status"] == "not_applied" + assert result["provider"]["effective_adapter"] == "not_applicable" + assert all(control["forwarded_to_client"] is None for control in result["sampling"].values()) + + +def test_public_projection_drops_unknown_fields_and_redacts_unsafe_labels() -> None: + malicious = { + "provider": { + "configured_adapter": "https://user:secret@provider.test", + "effective_adapter": "sk-secret-value", + "endpoint": "https://private.example.test", + "routing": { + "deployment_override": "sk-secret-deployment", + "deployment_source": "environment", + "api_version": "https://private.example.test/version", + "api_version_source": "environment", + }, + }, + "analyzers": [ + { + "analyzer_id": LLM_ANALYZER_SLOTS[0], + "model": "nvapi-secret-value", + "analyzer_revision": {"value": "2.11.2", "prompt": "private prompt"}, + "credentials": "do-not-emit", + }, + {"analyzer_id": "unknown", "model": "private/model"}, + ], + "sampling": { + "temperature": { + "requested": float("nan"), + "source": "environment", + "forwarded_to_client": float("inf"), + "adapter_support": True, + "headers": {"authorization": "secret"}, + } + }, + "raw_prompt": "do-not-emit", + } + + result = sanitize_llm_provenance(malicious, use_llm=True) + serialized = json.dumps(result) + + assert result["provider"]["configured_adapter"] == "unknown" + assert result["provider"]["resolved_adapter"] == "unknown" + assert result["provider"]["effective_adapter"] == "unknown" + assert result["provider"]["routing"] == { + "deployment_override": None, + "deployment_source": "unknown", + "api_version": None, + "api_version_source": "unknown", + } + assert result["analyzers"][0]["model"] == "redacted" + assert len(result["analyzers"]) == len(LLM_ANALYZER_SLOTS) + assert "secret" not in serialized + assert "private.example" not in serialized + assert "do-not-emit" not in serialized + assert "private prompt" not in serialized + + +@pytest.mark.parametrize( + "credential", + [ + "AWS-secret-fake-value", + "github_pat_fake-value", + "xoxb-fake-value", + "hf_fake-value", + "AIza-fake-value", + "Bearer synthetic-secret-token-123456", + "Basic c3ludGhldGljOnNlY3JldA==", + "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJzeW50aGV0aWMifQ.fake_signature", + "ewogICJhbGciOiAibm9uZSIKfQ.eyJzdWIiOiJzeW50aGV0aWMifQ.", + "0123456789abcdef" * 2, + "AbCdEfGhIjKlMnOpQrSt" * 2, + ], +) +def test_capture_and_projection_do_not_emit_credential_shaped_overrides( + monkeypatch: pytest.MonkeyPatch, + credential: str, +) -> None: + provider = AzureOpenAIProvider() + monkeypatch.setattr("skillspector.llm_provenance.get_active_provider", lambda: provider) + monkeypatch.setattr("skillspector.llm_provenance.get_model_config_provider", lambda: provider) + monkeypatch.setenv("SKILLSPECTOR_REASONING_EFFORT", credential) + monkeypatch.setenv("AZURE_OPENAI_DEPLOYMENT", credential) + + result = sanitize_llm_provenance( + capture_llm_provenance(_models(credential)), + use_llm=False, + ) + serialized = json.dumps(result) + + assert credential not in serialized + assert {item["model"] for item in result["analyzers"]} == {"redacted"} + assert result["provider"]["routing"]["deployment_override"] is None + assert result["sampling"]["reasoning_effort"]["requested"] is None + + +def test_malformed_source_revision_is_not_emitted() -> None: + secret = "github_pat_fake-value" + result = sanitize_llm_provenance( + { + "analyzers": [ + { + "analyzer_id": LLM_ANALYZER_SLOTS[0], + "model": "safe/model:1", + "analyzer_revision": { + "value": "2.11.2", + "source_revision": { + "value": secret, + "source": "build_environment", + }, + }, + } + ] + }, + use_llm=False, + ) + + assert secret not in json.dumps(result) + assert result["analyzers"][0]["analyzer_revision"]["source_revision"] == { + "value": "unknown", + "source": "unknown", + } + + +def test_multiple_response_providers_are_reported_as_mixed() -> None: + result = sanitize_llm_provenance( + None, + use_llm=True, + inference_usage=[*_usage("openai"), *_usage("anthropic")], + ) + + assert result["provider"]["effective_adapter"] == "mixed" + assert result["provider"]["effective_adapters"] == ["anthropic", "openai"] + + +def test_public_projection_repairs_inconsistent_control_shapes() -> None: + result = sanitize_llm_provenance( + { + "sampling": { + "temperature": { + "requested": 999, + "source": "environment", + "forwarded_to_client": 999, + "adapter_support": True, + }, + "seed": { + "requested": 1.5, + "source": "environment", + "forwarded_to_client": 1.5, + "adapter_support": True, + }, + "reasoning_effort": { + "requested": "low", + "source": "environment", + "forwarded_to_client": "high", + "adapter_support": True, + }, + } + }, + use_llm=True, + ) + + assert result["sampling"]["temperature"]["requested"] is None + assert result["sampling"]["temperature"]["source"] == "invalid_environment" + assert result["sampling"]["temperature"]["forwarded_to_client"] is None + assert result["sampling"]["seed"]["requested"] is None + assert result["sampling"]["seed"]["source"] == "invalid_environment" + assert result["sampling"]["seed"]["forwarded_to_client"] is None + assert result["sampling"]["reasoning_effort"]["requested"] == "low" + assert result["sampling"]["reasoning_effort"]["forwarded_to_client"] is None + assert result["determinism"]["control_status"] == "invalid_configuration" + + +def test_public_projection_ignores_unhashable_analyzer_ids() -> None: + result = sanitize_llm_provenance( + {"analyzers": [{"analyzer_id": {}, "model": "private/model"}]}, + use_llm=True, + ) + + assert len(result["analyzers"]) == len(LLM_ANALYZER_SLOTS) + assert {item["model"] for item in result["analyzers"]} == {"redacted"} + + +def test_unknown_provider_specific_reasoning_effort_is_not_published( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("skillspector.llm_provenance.get_active_provider", lambda: OpenAIProvider()) + monkeypatch.setattr( + "skillspector.llm_provenance.get_model_config_provider", lambda: OpenAIProvider() + ) + monkeypatch.setenv("SKILLSPECTOR_REASONING_EFFORT", "provider specific value") + + result = sanitize_llm_provenance( + capture_llm_provenance(_models()), + use_llm=True, + inference_usage=_usage( + "openai", + {"temperature": None, "seed": None, "reasoning_effort": "provider specific value"}, + ), + ) + + assert result["sampling"]["reasoning_effort"]["requested"] is None + assert result["sampling"]["reasoning_effort"]["forwarded_to_client"] is None + assert "provider specific value" not in json.dumps(result) + + +def test_invalid_and_unknown_controls_are_not_called_provider_defaults( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr("skillspector.llm_provenance.get_active_provider", lambda: OpenAIProvider()) + monkeypatch.setattr( + "skillspector.llm_provenance.get_model_config_provider", lambda: OpenAIProvider() + ) + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "warm") + + invalid = sanitize_llm_provenance(capture_llm_provenance(_models()), use_llm=True) + unknown = sanitize_llm_provenance(None, use_llm=True) + + assert invalid["determinism"]["control_status"] == "invalid_configuration" + assert unknown["determinism"]["control_status"] == "configuration_unknown" + + +def test_effective_provider_comes_from_response_not_preflight_candidate( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """A Bedrock-to-OpenAI fallback must report the provider that answered.""" + bedrock_provider = type("BedrockProvider", (), {})() + monkeypatch.setattr("skillspector.llm_provenance.get_active_provider", lambda: bedrock_provider) + monkeypatch.setattr( + "skillspector.llm_provenance.get_model_config_provider", lambda: bedrock_provider + ) + monkeypatch.setenv("SKILLSPECTOR_SEED", "17") + + result = sanitize_llm_provenance( + capture_llm_provenance(_models()), + use_llm=True, + inference_usage=_usage( + "openai", + {"temperature": None, "seed": 17, "reasoning_effort": None}, + ), + ) + + assert result["provider"]["resolved_adapter"] == "bedrock" + assert result["provider"]["effective_adapter"] == "openai" + assert result["provider"]["effective_adapters"] == ["openai"] + assert result["sampling"]["seed"]["adapter_support"] is True + assert result["sampling"]["seed"]["forwarded_to_client"] == 17 + + +def test_nv_inference_preserves_all_observed_controls( + monkeypatch: pytest.MonkeyPatch, +) -> None: + provider = NvInferenceProvider() + monkeypatch.setattr("skillspector.llm_provenance.get_active_provider", lambda: provider) + monkeypatch.setattr("skillspector.llm_provenance.get_model_config_provider", lambda: provider) + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0.25") + monkeypatch.setenv("SKILLSPECTOR_SEED", "23") + monkeypatch.setenv("SKILLSPECTOR_REASONING_EFFORT", "high") + + result = sanitize_llm_provenance( + capture_llm_provenance(_models()), + use_llm=True, + inference_usage=_usage( + "nv_inference", + {"temperature": 0.25, "seed": 23, "reasoning_effort": "high"}, + ), + ) + + assert result["provider"]["effective_adapter"] == "nv_inference" + assert result["sampling"]["temperature"]["forwarded_to_client"] == 0.25 + assert result["sampling"]["seed"]["forwarded_to_client"] == 23 + assert result["sampling"]["reasoning_effort"]["forwarded_to_client"] == "high" + + +def test_azure_routing_records_deployment_and_api_version( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "skillspector.llm_provenance.get_active_provider", lambda: AzureOpenAIProvider() + ) + monkeypatch.setattr( + "skillspector.llm_provenance.get_model_config_provider", lambda: AzureOpenAIProvider() + ) + monkeypatch.setenv("AZURE_OPENAI_DEPLOYMENT", "production-v2") + monkeypatch.setenv("AZURE_OPENAI_API_VERSION", "2025-01-01") + + result = sanitize_llm_provenance( + capture_llm_provenance(_models("gpt-4o")), + use_llm=True, + inference_usage=_usage("azure_openai"), + ) + + assert result["analyzers"][0]["model"] == "gpt-4o" + assert result["provider"]["routing"] == { + "deployment_override": "production-v2", + "deployment_source": "environment", + "api_version": "2025-01-01", + "api_version_source": "environment", + } + + +def test_azure_routing_uses_model_and_default_api_version_without_overrides( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.setattr( + "skillspector.llm_provenance.get_active_provider", lambda: AzureOpenAIProvider() + ) + monkeypatch.setattr( + "skillspector.llm_provenance.get_model_config_provider", lambda: AzureOpenAIProvider() + ) + + result = sanitize_llm_provenance( + capture_llm_provenance(_models("gpt-4o")), + use_llm=True, + inference_usage=_usage("azure_openai"), + ) + + assert result["provider"]["routing"] == { + "deployment_override": None, + "deployment_source": "resolved_model", + "api_version": "2024-06-01", + "api_version_source": "provider_default", + } + + +@pytest.mark.parametrize( + ("class_name", "expected"), + [ + ("AzureOpenAIProvider", "azure_openai"), + ("OllamaProvider", "ollama"), + ("OpenAICompatibleProvider", "openai_compatible"), + ("AntigravityCLIProvider", "antigravity_cli"), + ], +) +def test_builtin_provider_names_are_canonical(class_name: str, expected: str) -> None: + provider_type = type(class_name, (), {}) + + assert provider_name(provider_type()) == expected diff --git a/tests/unit/test_llm_utils.py b/tests/unit/test_llm_utils.py index a538e064f..da464e43a 100644 --- a/tests/unit/test_llm_utils.py +++ b/tests/unit/test_llm_utils.py @@ -33,7 +33,7 @@ from skillspector import llm_utils from skillspector.constants import build_model_config -from skillspector.inference_usage import InferenceUsageCollector +from skillspector.inference_usage import InferenceUsageCollector, sanitize_inference_usage from skillspector.llm_utils import ( AgentCLIChatModel, StructuredOutputParseError, @@ -566,7 +566,8 @@ class _Schema(BaseModel): _invoke_with_usage(runnable, "prompt", collector) assert collector.response_received is True - assert collector.snapshot() == [] + assert collector.snapshot()[0]["provider"] == "claude_cli" + assert sanitize_inference_usage(collector.snapshot()) == [] async def test_concurrent_structured_usage_marks_each_async_response(self) -> None: class _Schema(BaseModel): @@ -597,7 +598,8 @@ class _Schema(BaseModel): assert all(isinstance(result, ValueError) for result in results) assert all(collector.response_received for collector in collectors) - assert all(collector.snapshot() == [] for collector in collectors) + assert all(collector.snapshot()[0]["provider"] == "claude_cli" for collector in collectors) + assert all(sanitize_inference_usage(collector.snapshot()) == [] for collector in collectors) def test_structured_usage_does_not_mark_pre_response_transport_failure(self) -> None: class _Schema(BaseModel): diff --git a/tests/unit/test_providers.py b/tests/unit/test_providers.py index 5c0852bc0..81ffa00d3 100644 --- a/tests/unit/test_providers.py +++ b/tests/unit/test_providers.py @@ -594,12 +594,37 @@ def fake_chat_openai(**kwargs: object) -> dict[str, object]: assert captured["temperature"] == 0.25 assert captured["seed"] == 42 + @pytest.mark.parametrize("seed", [-(1 << 63), (1 << 63) - 1]) + def test_signed_64_bit_seed_boundaries_are_forwarded( + self, + monkeypatch: pytest.MonkeyPatch, + seed: int, + ) -> None: + captured: dict[str, object] = {} + + def fake_chat_openai(**kwargs: object) -> dict[str, object]: + captured.update(kwargs) + return kwargs + + monkeypatch.setattr(chat_models, "ChatOpenAI", fake_chat_openai) + monkeypatch.setenv("SKILLSPECTOR_SEED", str(seed)) + + create_openai_compatible_chat_model( + model="gpt-5.4", + credentials=("sk-x", "http://localhost:1234/v1"), + max_tokens=123, + ) + + assert captured["seed"] == seed + @pytest.mark.parametrize( ("name", "value", "message"), [ ("SKILLSPECTOR_TEMPERATURE", "warm", "must be a number"), ("SKILLSPECTOR_TEMPERATURE", "1.1", "must be between 0 and 1"), ("SKILLSPECTOR_SEED", "4.2", "must be an integer"), + ("SKILLSPECTOR_SEED", str(1 << 63), "must be a signed 64-bit integer"), + ("SKILLSPECTOR_SEED", str(-(1 << 63) - 1), "must be a signed 64-bit integer"), ], ) def test_invalid_sampling_control_fails_before_model_construction(