From 90f85b1a22f624b38afd14f5c361b5b45c427da6 Mon Sep 17 00:00:00 2001 From: Jehu Gray Date: Wed, 12 Aug 2026 14:42:46 -0400 Subject: [PATCH] perf: fix mosaic latency, add dynamic model selector with live ARM discovery Latency: - Cap reasoning_effort for gpt-5/o-series models on raw chat-completions call sites (contextual_agent, semantic_translator tile-selection) to avoid unnecessary reasoning-token overhead on simple prompts. - Attempt reasoning_effort at Agent Service agent-creation time for reasoning models (analyst_agent), with a safe fallback since the installed azure-ai-agents SDK does not support the parameter. Model selector: - AnalystAgent now honors the frontend's selected model end-to-end via a new _ensure_model()/_create_agent_capped() mechanism that recreates the Agent Service agent definition on model change while preserving threads (thread_id is not tied to a specific agent_id). - ClarifierAgent (greeting/identity short-circuit) now accepts a per-call model override instead of using a fixed singleton deployment, and no longer unconditionally sends reasoning_effort to non-reasoning models. - Fixed a pre-existing bug where ClarifierAgent used AZURE_AI_PROJECT_ENDPOINT (an Agent Service project endpoint) instead of AZURE_OPENAI_ENDPOINT, causing a silently-swallowed 401 on every greeting. - Added GET /api/models: discovers real Azure OpenAI deployments via the ARM management API (with a 5-minute in-memory cache and a static fallback list), so the dropdown reflects whatever models are actually deployed in a given tenant instead of a hardcoded list. - ModelSelector.tsx now fetches from /api/models instead of guessing availability from the overall /api/health status. --- .../agents/analyst_agent/analyst_agent.py | 143 +++++- .../agents/clarifier_agent/clarifier_agent.py | 40 +- .../contextual_agent/contextual_agent.py | 19 +- .../container-app/fastapi_app.py | 434 +++++++++++++++++- .../container-app/pipeline/layer1_agents.py | 7 +- .../container-app/semantic_translator.py | 47 +- .../web-ui/src/components/ModelSelector.tsx | 47 +- planetary-explorer/web-ui/src/services/api.ts | 2 +- 8 files changed, 665 insertions(+), 74 deletions(-) diff --git a/planetary-explorer/container-app/agents/analyst_agent/analyst_agent.py b/planetary-explorer/container-app/agents/analyst_agent/analyst_agent.py index 5d9d9e0..c11cf02 100644 --- a/planetary-explorer/container-app/agents/analyst_agent/analyst_agent.py +++ b/planetary-explorer/container-app/agents/analyst_agent/analyst_agent.py @@ -72,8 +72,10 @@ class AnalystAgent: def __init__(self) -> None: self._agents_client = None self._agent_id: Optional[str] = None + self._current_model: Optional[str] = None self._initialized = False self._init_lock = asyncio.Lock() + self._model_lock = asyncio.Lock() self._threads: Dict[str, AnalystThread] = {} self._max_init_retries = 2 logger.info("AnalystAgent created (lazy init on first use)") @@ -113,7 +115,7 @@ async def _ensure_initialized(self) -> None: assert last_error is not None raise last_error - async def _do_initialize(self) -> None: + async def _do_initialize(self, model: Optional[str] = None) -> None: if not _AZURE_AVAILABLE: raise RuntimeError("azure.identity not installed") @@ -123,7 +125,11 @@ async def _do_initialize(self) -> None: endpoint = os.getenv("AZURE_AI_PROJECT_ENDPOINT") or os.getenv( "AZURE_OPENAI_ENDPOINT" ) - deployment = os.getenv("AZURE_OPENAI_DEPLOYMENT_NAME", "gpt-5") + # `model` is the user-selected deployment (from the frontend's model + # selector, threaded in via AnalyzeAgent.run -> AnalystAgent.run). + # Falls back to the env-configured default when no selection was + # made, preserving prior behavior for callers that don't pass one. + deployment = model or os.getenv("AZURE_OPENAI_DEPLOYMENT_NAME", "gpt-5") if not endpoint: raise ValueError( "AZURE_AI_PROJECT_ENDPOINT or AZURE_OPENAI_ENDPOINT must be set" @@ -140,18 +146,105 @@ async def _do_initialize(self) -> None: toolset.add(functions) self._agents_client.enable_auto_function_calls(toolset) - agent = await self._agents_client.create_agent( - model=deployment, - name="PlanetaryExplorerAnalyst", - instructions=ANALYST_AGENT_INSTRUCTIONS, - toolset=toolset, - ) + agent = await self._create_agent_capped(deployment, ANALYST_AGENT_INSTRUCTIONS, toolset) self._agent_id = agent.id + self._current_model = deployment self._initialized = True logger.info( "AnalystAgent initialized: agent_id=%s model=%s", agent.id, deployment ) + async def _ensure_model(self, model: Optional[str]) -> None: + """Ensure the live agent definition uses ``model``, if given. + + AnalystAgent is a long-lived singleton whose Azure AI Agent Service + agent definition is created once (model baked in at creation time). + Unlike ``semantic_translator.set_model()`` -- which just resets a + Semantic Kernel wrapper and lazily rebuilds it -- the Agent Service + SDK binds ``model`` at ``create_agent()`` time, so switching models + means recreating the agent definition. + + Threads are NOT tied to a specific agent definition (a thread_id is + passed alongside agent_id on every run), so recreating the agent + preserves all existing per-session conversation threads -- this is + cheap (a single API call) and doesn't lose any session state. + + No-op when ``model`` is falsy or already matches the live agent, so + the common case (no selection made, or same model as last turn) + costs nothing. + """ + if not model: + return + await self._ensure_initialized() + if model == self._current_model: + return + async with self._model_lock: + if model == self._current_model: + return + from .analyst_prompt import ANALYST_AGENT_INSTRUCTIONS + from .tools import create_analyst_functions + from azure.ai.agents.models import AsyncFunctionTool, AsyncToolSet # type: ignore + + logger.info( + "[ANALYST] Switching model %s -> %s (recreating agent, threads preserved)", + self._current_model, model, + ) + functions = AsyncFunctionTool(create_analyst_functions()) + toolset = AsyncToolSet() + toolset.add(functions) + self._agents_client.enable_auto_function_calls(toolset) # type: ignore[union-attr] + agent = await self._create_agent_capped(model, ANALYST_AGENT_INSTRUCTIONS, toolset) + self._agent_id = agent.id + self._current_model = model + + async def _create_agent_capped(self, model: str, instructions: str, toolset): + """Create an Agent Service agent, capping reasoning effort for gpt-5- + family models when possible. + + The Agents SDK's ``create_agent`` does not document a + ``reasoning_effort`` parameter the way the plain chat.completions + API does (see ContextualAgent/LoadAgent/etc., which already cap + this on gpt-5 for latency). Since the underlying REST API is built + on the same schema family as the Assistants/Responses API, it may + still accept the field as an undocumented passthrough -- but this + is unverified, so we try it and transparently fall back to + creating the agent without it if the service/SDK rejects the + extra kwarg. Never raises solely because of this attempt. + """ + model_lc = (model or "").lower() + is_reasoning = model_lc.startswith(("gpt-5", "o1", "o3", "o4")) + if is_reasoning: + try: + return await self._agents_client.create_agent( # type: ignore[union-attr] + model=model, + name="PlanetaryExplorerAnalyst", + instructions=instructions, + toolset=toolset, + reasoning_effort="minimal", + ) + except TypeError: + # SDK's typed signature rejects the kwarg outright (no + # passthrough support at all in this SDK version). + logger.info( + "[ANALYST] reasoning_effort not supported by installed " + "azure-ai-agents SDK version -- creating agent without it" + ) + except Exception as exc: + # Service-side rejection (e.g. unknown field in request + # body) surfaces as an HTTP error from the SDK, not a + # TypeError -- catch broadly here so a latency-cap attempt + # never breaks agent creation. + logger.info( + "[ANALYST] reasoning_effort rejected by service (%s) -- " + "creating agent without it", exc, + ) + return await self._agents_client.create_agent( # type: ignore[union-attr] + model=model, + name="PlanetaryExplorerAnalyst", + instructions=instructions, + toolset=toolset, + ) + # ------------------------------------------------------------------ # Thread management # ------------------------------------------------------------------ @@ -171,12 +264,19 @@ async def _get_or_create_thread(self, session_id: str) -> AnalystThread: # Main entry # ------------------------------------------------------------------ - async def run(self, request) -> "SynthesizedResponse": + async def run(self, request, model: Optional[str] = None) -> "SynthesizedResponse": """Run the ReAct loop for a single AnalysisRequest. Returns a SynthesizedResponse that's drop-in compatible with the old Orchestrator + Synthesizer output (so layer1_agents.AnalyzeAgent doesn't need to change its caller contract). + + Args: + model: Optional user-selected deployment name (from the + frontend model selector). When given and different from + the live agent's current model, the agent definition is + recreated on this model before the run -- see + ``_ensure_model``. """ from pipeline.contracts import ( AnalysisPlan, @@ -187,6 +287,7 @@ async def run(self, request) -> "SynthesizedResponse": from .session_context import AnalystSession, clear_session, get_session, set_session started = time.time() + await self._ensure_model(model) # Populate the ContextVar so tools see the session. # ``use_graphrag`` / ``stac_mode`` ride on the request via the @@ -217,12 +318,13 @@ async def run(self, request) -> "SynthesizedResponse": set_session(sess) try: - answer, tool_calls, evidence = await self._invoke_agent_service(request) + answer, tool_calls, evidence, skill_pack_id = await self._invoke_agent_service(request) except Exception as e: logger.exception("[ANALYST] run failed, returning fallback response") answer = self._fallback_answer(request, str(e)) tool_calls = [] evidence = [] + skill_pack_id = None # Aggregate sources from tool evidence sources: List[Source] = [] @@ -252,6 +354,9 @@ async def run(self, request) -> "SynthesizedResponse": if clarify_payload: structured_by_tool["clarify"] = clarify_payload + if skill_pack_id: + structured_by_tool["skill_pack"] = {"id": skill_pack_id} + # Build a degenerate plan record for back-compat with callers that # still serialize ``plan``. The plan is just the sequence of tools # that actually ran. @@ -298,6 +403,21 @@ async def _invoke_agent_service(self, request): augmented = self._build_message(request) + # Skill pack: pick 0-1 domain guidance packs matching this question + # and inject as additional_instructions for this run only -- pure + # prompt augmentation, no new tools, no model change. Mirrors the + # identical mechanism in EnhancedVisionAgent.analyze(). + skill_instructions = None + skill_pack_id = None + try: + from skills.skill_selector import select_skill_pack, render_skill_block + pack = select_skill_pack(request.question, applies_to="analyst_agent") + if pack: + skill_instructions = render_skill_block(pack) + skill_pack_id = pack.id + except Exception as skill_exc: + logger.warning("[SKILLS] Selection failed (continuing without): %s", skill_exc) + run = None for attempt in range(3): try: @@ -317,6 +437,7 @@ async def _invoke_agent_service(self, request): run = await self._agents_client.runs.create_and_process( thread_id=thread.thread_id, agent_id=self._agent_id, + additional_instructions=skill_instructions, ) break except Exception as e: @@ -359,7 +480,7 @@ async def _invoke_agent_service(self, request): # Tools recorded their results on the session ContextVar from .session_context import get_session evidence = list(get_session().evidence) - return answer, tool_calls, evidence + return answer, tool_calls, evidence, skill_pack_id # ------------------------------------------------------------------ # Helpers diff --git a/planetary-explorer/container-app/agents/clarifier_agent/clarifier_agent.py b/planetary-explorer/container-app/agents/clarifier_agent/clarifier_agent.py index 5473eda..57d086c 100644 --- a/planetary-explorer/container-app/agents/clarifier_agent/clarifier_agent.py +++ b/planetary-explorer/container-app/agents/clarifier_agent/clarifier_agent.py @@ -118,13 +118,18 @@ def __init__( "AZURE_OPENAI_CLARIFIER_DEPLOYMENT", os.getenv("AZURE_OPENAI_DEPLOYMENT_NAME", "gpt-5"), ) - self.endpoint = endpoint or os.getenv( - "AZURE_AI_PROJECT_ENDPOINT" - ) or os.getenv("AZURE_OPENAI_ENDPOINT") + # NOTE: ClarifierAgent talks to AzureOpenAI's plain chat-completions + # API (not the Agent Service), so it must use the Cognitive + # Services endpoint/token audience, not AZURE_AI_PROJECT_ENDPOINT + # (an ai.azure.com Agent Service project endpoint used elsewhere, + # e.g. by AnalystAgent). Using the project endpoint here caused a + # silent 401 "audience is incorrect (https://ai.azure.com)" on + # every greeting/identity short-circuit -- the code fell back to + # a static passthrough on error, which masked the failure. + self.endpoint = endpoint or os.getenv("AZURE_OPENAI_ENDPOINT") if not self.endpoint: raise ValueError( - "ClarifierAgent requires AZURE_AI_PROJECT_ENDPOINT or " - "AZURE_OPENAI_ENDPOINT to be set." + "ClarifierAgent requires AZURE_OPENAI_ENDPOINT to be set." ) self.api_version = api_version self._client: Optional[AsyncAzureOpenAI] = None @@ -155,13 +160,26 @@ def _get_client(self) -> AsyncAzureOpenAI: # ------------------------------------------------------------------ # Public entry point # ------------------------------------------------------------------ - async def decide(self, payload: ClarifierInput) -> ClarifierDecision: + async def decide( + self, payload: ClarifierInput, model: Optional[str] = None + ) -> ClarifierDecision: """ Run the clarifier prompt and return a ClarifierDecision. + Args: + payload: The clarifier input. + model: Optional per-call model override (the frontend's + model selector). Falls back to ``self.deployment`` (the + env-configured default) when not given, preserving prior + behavior for callers that don't pass one. This mirrors + the fix already applied to AnalystAgent.run(model=...) -- + without it, the greeting/identity short-circuit silently + ignored the user's model selection. + Falls back to a deterministic passthrough on any error so the request keeps working even if the LLM is unreachable. """ + _deployment = model or self.deployment try: pin_lat_lng = ( f"({payload.pin_lat:.4f}, {payload.pin_lng:.4f})" @@ -183,8 +201,13 @@ async def decide(self, payload: ClarifierInput) -> ClarifierDecision: ) client = self._get_client() + _model_lc = (_deployment or "").lower() + _is_reasoning = _model_lc.startswith(("gpt-5", "o1", "o3", "o4")) + _sampling_kwargs = ( + {"reasoning_effort": "minimal"} if _is_reasoning else {"temperature": 0.0} + ) response = await client.chat.completions.create( - model=self.deployment, + model=_deployment, messages=[ {"role": "system", "content": CLARIFIER_SYSTEM_PROMPT}, {"role": "user", "content": user_prompt}, @@ -193,8 +216,7 @@ async def decide(self, payload: ClarifierInput) -> ClarifierDecision: "type": "json_schema", "json_schema": CLARIFIER_DECISION_SCHEMA, }, - temperature=0.0, - reasoning_effort="minimal", + **_sampling_kwargs, ) content = response.choices[0].message.content or "{}" diff --git a/planetary-explorer/container-app/agents/contextual_agent/contextual_agent.py b/planetary-explorer/container-app/agents/contextual_agent/contextual_agent.py index 0236137..8ca5f64 100644 --- a/planetary-explorer/container-app/agents/contextual_agent/contextual_agent.py +++ b/planetary-explorer/container-app/agents/contextual_agent/contextual_agent.py @@ -107,13 +107,24 @@ async def run(self, payload: ContextualInput) -> ContextualResult: try: client = self._get_client() - # NOTE: temperature intentionally omitted. gpt-5 reasoning - # deployments only accept the default temperature (1); passing - # any other value returns HTTP 400 and bubbles up as the - # user-visible "LLM call failed" sentinel. + # Model-aware kwargs: gpt-5 / o-series reasoning models reject + # `temperature` (only default=1 supported) but accept + # `reasoning_effort`. Without capping reasoning_effort, gpt-5 + # burns many hidden reasoning tokens even on simple factual + # questions -- measured adding ~10-15s of latency to plain + # "what is NDVI"-style questions. Mirrors the same model-aware + # pattern used by LoadAgent / ClarifierAgent / CollectionSelector. + model_lc = (self.deployment or "").lower() + is_reasoning = model_lc.startswith(("gpt-5", "o1", "o3", "o4")) + extra: dict = {} + if is_reasoning: + extra["reasoning_effort"] = "minimal" + else: + extra["temperature"] = 0.3 resp = await client.chat.completions.create( model=self.deployment, messages=messages, + **extra, ) answer = resp.choices[0].message.content or "" except Exception as exc: # noqa: BLE001 diff --git a/planetary-explorer/container-app/fastapi_app.py b/planetary-explorer/container-app/fastapi_app.py index c700399..9a2c9ef 100644 --- a/planetary-explorer/container-app/fastapi_app.py +++ b/planetary-explorer/container-app/fastapi_app.py @@ -2524,6 +2524,133 @@ async def health_check(): _TRUE_VALUES = {"1", "true", "yes", "on"} +# Simple in-memory cache for /api/models -- deployment lists rarely change, +# so avoid hitting the ARM management API on every dropdown render. +_MODELS_CACHE: dict = {"data": None, "fetched_at": 0.0} +_MODELS_CACHE_TTL_SECONDS = 300 # 5 minutes + +# Static fallback used when ARM discovery isn't configured (missing env +# vars) or the management-plane call fails for any reason -- the dropdown +# should never come back empty just because discovery had a hiccup. +_FALLBACK_MODELS = [ + {"id": "gpt-4o-mini", "name": "GPT-4o mini (fast)", "isDefault": True, "isAvailable": True}, + {"id": "gpt-4o", "name": "GPT-4o", "isAvailable": True}, + {"id": "gpt-5", "name": "GPT-5 (deep reasoning)", "isAvailable": True}, +] + + +def _friendly_model_name(model_name: str, model_version: str) -> str: + """Best-effort human-readable label for a deployment. + + Falls back to the raw deployment name if we don't recognize the + model family -- this keeps unknown/future models (e.g. a + hypothetical "gpt-5.5") visible in the dropdown instead of hidden. + """ + name_lc = (model_name or "").lower() + if name_lc.startswith("gpt-5"): + return f"{model_name.upper()} (deep reasoning)" + if "mini" in name_lc: + return f"{model_name} (fast)" + if name_lc.startswith(("o1", "o3", "o4")): + return f"{model_name} (reasoning)" + return model_name + + +@app.get("/api/models") +async def get_available_models(): + """Return the Azure OpenAI deployments actually available in this + tenant, discovered live via the ARM management API, instead of a + hardcoded list. + + Requires AZURE_SUBSCRIPTION_ID, AZURE_RESOURCE_GROUP, and + AZURE_OPENAI_ACCOUNT_NAME to be set, plus the container app's managed + identity to have at least Reader on the Cognitive Services account + (a data-plane-only identity, as used for chat completions, cannot + list deployments -- that's a management-plane call). + + Falls back to a static default list (matching prior hardcoded + frontend behavior) if discovery isn't configured or fails, so the + model selector never renders empty. + """ + import time as _time + + now = _time.time() + if _MODELS_CACHE["data"] is not None and (now - _MODELS_CACHE["fetched_at"]) < _MODELS_CACHE_TTL_SECONDS: + return JSONResponse(content=_MODELS_CACHE["data"]) + + subscription_id = os.getenv("AZURE_SUBSCRIPTION_ID") + resource_group = os.getenv("AZURE_RESOURCE_GROUP") + account_name = os.getenv("AZURE_OPENAI_ACCOUNT_NAME") + + if not (subscription_id and resource_group and account_name): + logger.info( + "[MODELS] ARM discovery not configured (missing " + "AZURE_SUBSCRIPTION_ID/AZURE_RESOURCE_GROUP/AZURE_OPENAI_ACCOUNT_NAME) " + "-- using static fallback list" + ) + payload = {"models": _FALLBACK_MODELS, "source": "fallback"} + _MODELS_CACHE["data"] = payload + _MODELS_CACHE["fetched_at"] = now + return JSONResponse(content=payload) + + try: + from azure.identity.aio import DefaultAzureCredential as AsyncDefaultAzureCredential + + credential = AsyncDefaultAzureCredential() + token = await credential.get_token("https://management.azure.com/.default") + await credential.close() + + url = ( + f"https://management.azure.com/subscriptions/{subscription_id}" + f"/resourceGroups/{resource_group}" + f"/providers/Microsoft.CognitiveServices/accounts/{account_name}" + f"/deployments?api-version=2023-05-01" + ) + headers = {"Authorization": f"Bearer {token.token}"} + async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=10)) as session: + async with session.get(url, headers=headers) as resp: + if resp.status != 200: + raise RuntimeError(f"ARM deployments list returned HTTP {resp.status}") + body = await resp.json() + + models = [] + default_set = False + for entry in body.get("value", []): + deployment_name = entry.get("name") + props = entry.get("properties", {}) + model_info = props.get("model", {}) + model_name = model_info.get("name", deployment_name) + model_version = model_info.get("version", "") + provisioning_state = props.get("provisioningState", "") + is_available = provisioning_state.lower() == "succeeded" + is_default = not default_set and "mini" in deployment_name.lower() + if is_default: + default_set = True + models.append({ + "id": deployment_name, + "name": _friendly_model_name(model_name, model_version), + "isDefault": is_default, + "isAvailable": is_available, + }) + + if not models: + raise RuntimeError("ARM returned zero deployments") + if not default_set: + models[0]["isDefault"] = True + + payload = {"models": models, "source": "arm"} + _MODELS_CACHE["data"] = payload + _MODELS_CACHE["fetched_at"] = now + logger.info("[MODELS] Discovered %d live deployment(s) via ARM: %s", len(models), [m["id"] for m in models]) + return JSONResponse(content=payload) + + except Exception as exc: # noqa: BLE001 + logger.warning("[MODELS] ARM discovery failed (%s) -- using static fallback list", exc) + payload = {"models": _FALLBACK_MODELS, "source": "fallback"} + _MODELS_CACHE["data"] = payload + _MODELS_CACHE["fetched_at"] = now + return JSONResponse(content=payload) + def _env_flag(name: str, default: bool = False) -> bool: """Parse a string env var as a boolean flag. @@ -4241,6 +4368,243 @@ async def resilience_facilities(request: Request, region: str | None = None): } +# --------------------------------------------------------------------------- +# Copilot / M365 declarative agent — stateless Earth-observation Q&A +# --------------------------------------------------------------------------- +# Separate, narrow action from the Resilience plugin above (see +# m365/teams-app/plugin-query/). Wraps AnalystAgent's ReAct loop for +# general "what is X" / "how does Y work" Earth-science questions without +# requiring any of the map-session state (screenshot, loaded_collections, +# pin, tile_urls) the web UI provides — Teams/Copilot callers don't have a +# map, so this is intentionally a stateless, single-turn endpoint. +# --------------------------------------------------------------------------- +@app.get("/api/copilot/health") +async def copilot_health(): + """Readiness probe for the Copilot Q&A action. No auth, no external calls.""" + return {"status": "ready"} + + +@app.post("/api/copilot/ask") +async def copilot_ask(request: Request): + """Answer a stateless Earth-observation question via AnalystAgent. + + Body: + { + "question": "what is a spectral index?", # required + "location": "Gulf Coast", # optional, free text + "time_range": "last week" # optional, free text + } + + This calls the same v2 pipeline (ActionRouter -> AnalyzeAgent -> + AnalystAgent) used by /api/query, but with a minimal body containing + no map/session context, so the router reliably lands on ANALYZE for + knowledge questions. Returns a small, Copilot-friendly shape rather + than the full /api/query response (which includes map/tile fields + that have no meaning without a map). + """ + try: + body = await request.json() + except Exception: + body = {} + + question = (body.get("question") or "").strip() + if not question: + raise HTTPException(status_code=422, detail="'question' is required") + + location = body.get("location") + time_range = body.get("time_range") + natural_query = question + if location: + natural_query = f"{natural_query} (location: {location})" + if time_range: + natural_query = f"{natural_query} (time range: {time_range})" + + pipeline_body = { + "query": natural_query, + "session_id": f"copilot-{_uuid.uuid4().hex[:12]}", + } + + try: + from pipeline.dispatch import run_pipeline_v2 + result = await run_pipeline_v2(pipeline_body) + except Exception as exc: + logger.exception("[COPILOT] /api/copilot/ask pipeline failed") + raise HTTPException(status_code=502, detail=f"Pipeline error: {exc}") + + action = result.get("action") + answer = result.get("answer") or "" + + if action == "CLARIFY": + return { + "answer": answer or "Could you clarify your question?", + "needs_clarification": True, + "options": (result.get("structured") or {}).get("options", []), + } + + if action in ("NAVIGATE", "LOAD", "LOAD_AND_ANALYZE") and not answer: + # These actions expect a map to render into, which Copilot doesn't + # have. Give the user a clear, honest response instead of an + # empty/misleading one. + return { + "answer": ( + "That looks like a request to load or navigate to map data, " + "which requires the Planetary Explorer map view. Try asking " + "a general Earth-science question instead, or use the " + "Planetary Explorer web app for imagery/navigation requests." + ), + "needs_clarification": False, + "options": [], + } + + structured = result.get("structured") or {} + skill_pack = structured.get("skill_pack") if isinstance(structured, dict) else None + return { + "answer": answer or "I wasn't able to find an answer to that question.", + "needs_clarification": False, + "options": [], + "skill_pack": skill_pack.get("id") if isinstance(skill_pack, dict) else None, + } + + +@app.post("/api/copilot/vision") +async def copilot_vision(request: Request): + """Answer a visual/imagery question about a named location, statelessly. + + Body: + { + "question": "what does the vegetation look like here?", # required + "location": "Austin, Texas" # required + } + + Unlike the web UI's Vision Agent flow (which analyzes a screenshot/tiles + already loaded on the user's map), Copilot has no map session to draw + from. This endpoint closes that gap itself: + + 1. Geocode ``location`` to a bbox (EnhancedLocationResolver — same + resolver NavigateAgent uses for the web UI's "go to X" flow). + 2. Search Planetary Computer for the most recent low-cloud Sentinel-2 + scene covering that bbox (mirrors geoint/vision_analyzer.py's + ``_fetch_satellite_image`` pattern). + 3. Fetch a rendered PNG preview of that scene and base64-encode it. + 4. Pass the image + STAC context to EnhancedVisionAgent.analyze(), + the same vision pipeline the web UI uses. + + This is intentionally narrow: one image, one location, one question, + no multi-turn map session. Multi-turn visual analysis still requires + the Planetary Explorer web app. + """ + try: + body = await request.json() + except Exception: + body = {} + + question = (body.get("question") or "").strip() + location = (body.get("location") or "").strip() + if not question: + raise HTTPException(status_code=422, detail="'question' is required") + if not location: + raise HTTPException(status_code=422, detail="'location' is required") + + # Step 1: geocode + try: + from location_resolver import EnhancedLocationResolver + resolver = EnhancedLocationResolver() + bbox = await resolver.resolve_location_to_bbox(location) + except Exception as exc: + logger.exception("[COPILOT-VISION] geocoding failed") + raise HTTPException(status_code=502, detail=f"Geocoding error: {exc}") + if not bbox: + return { + "answer": f"I couldn't find a location matching '{location}'. Try a more specific place name.", + "needs_clarification": True, + "options": [], + } + + # Step 2 + 3: STAC search + fetch a rendered preview image, mirroring + # geoint/vision_analyzer.py's _fetch_satellite_image pattern (recent, + # low-cloud Sentinel-2 scene covering the bbox; 30-day window with a + # 60-day/relaxed-cloud fallback). + try: + import planetary_computer + from pystac_client import Client as _StacClient + + catalog = _StacClient.open( + "https://planetarycomputer.microsoft.com/api/stac/v1", + modifier=planetary_computer.sign_inplace, + ) + now = datetime.utcnow() + search = catalog.search( + collections=["sentinel-2-l2a"], + bbox=bbox, + datetime=f"{(now - timedelta(days=30)).isoformat()}Z/{now.isoformat()}Z", + query={"eo:cloud_cover": {"lt": 20}}, + limit=20, + ) + items = list(search.items()) + if not items: + search = catalog.search( + collections=["sentinel-2-l2a"], + bbox=bbox, + datetime=f"{(now - timedelta(days=60)).isoformat()}Z/{now.isoformat()}Z", + query={"eo:cloud_cover": {"lt": 30}}, + limit=10, + ) + items = list(search.items()) + if not items: + return { + "answer": f"I couldn't find recent, mostly-clear Sentinel-2 imagery for {location}. Try a different location or check back later.", + "needs_clarification": False, + "options": [], + } + item = sorted(items, key=lambda x: x.datetime, reverse=True)[0] + + preview_url = ( + f"https://planetarycomputer.microsoft.com/api/data/v1/item/preview.png" + f"?collection=sentinel-2-l2a&item={item.id}&assets=visual&width=512&height=512" + ) + signed_url = planetary_computer.sign_url(preview_url) + + import aiohttp + async with aiohttp.ClientSession() as session: + async with session.get(signed_url, timeout=aiohttp.ClientTimeout(total=45)) as resp: + if resp.status != 200: + raise RuntimeError(f"preview fetch returned HTTP {resp.status}") + image_bytes = await resp.read() + except Exception as exc: + logger.exception("[COPILOT-VISION] STAC search / image fetch failed") + raise HTTPException(status_code=502, detail=f"Imagery fetch error: {exc}") + + import base64 as _b64 + imagery_base64 = _b64.b64encode(image_bytes).decode("utf-8") + + # Step 4: vision analysis, same agent the web UI uses + try: + from agents.enhanced_vision_agent import get_enhanced_vision_agent + vision_agent = get_enhanced_vision_agent() + result = await vision_agent.analyze( + user_query=question, + session_id=f"copilot-vision-{_uuid.uuid4().hex[:12]}", + imagery_base64=imagery_base64, + map_bounds={ + "west": bbox[0], "south": bbox[1], "east": bbox[2], "north": bbox[3], + "center_lat": (bbox[1] + bbox[3]) / 2, "center_lng": (bbox[0] + bbox[2]) / 2, + }, + collections=["sentinel-2-l2a"], + ) + except Exception as exc: + logger.exception("[COPILOT-VISION] vision analysis failed") + raise HTTPException(status_code=502, detail=f"Vision analysis error: {exc}") + + answer = result.get("response") or result.get("analysis") or "" + return { + "answer": answer or "I wasn't able to analyze the imagery for that location.", + "needs_clarification": False, + "options": [], + "imagery_date": item.datetime.isoformat() if item.datetime else None, + "cloud_cover": item.properties.get("eo:cloud_cover"), + } + + # --------------------------------------------------------------------------- # Resilience assessment cache + snapshot endpoint # --------------------------------------------------------------------------- @@ -4388,7 +4752,7 @@ async def unified_query_processor(request: Request): natural_query = req_body.get('query') or req_body.get('user_query') or 'No query provided' session_id = req_body.get('session_id') or req_body.get('conversation_id') pin = req_body.get('pin') or req_body.get('vision_pin') # Pin {lat, lng} (web-ui sends 'vision_pin') - selected_model = req_body.get('model', 'gpt-5') # Model selection from frontend, default to gpt-5 + selected_model = req_body.get('model', 'gpt-4o-mini') # Model selection from frontend, default to fast model # ================================================================ # TOP-LEVEL GREETING / IDENTITY SHORT-CIRCUIT @@ -4456,16 +4820,19 @@ async def unified_query_processor(request: Request): get_clarifier_agent, ClarifierInput, ) _agent_top = get_clarifier_agent() - _decision_top = await _agent_top.decide(ClarifierInput( - query=natural_query, - has_rendered_map=False, - has_screenshot=False, - has_last_bbox=False, - pending_clarification=False, - has_pin=False, - prior_action=None, - prior_target_route=None, - )) + _decision_top = await _agent_top.decide( + ClarifierInput( + query=natural_query, + has_rendered_map=False, + has_screenshot=False, + has_last_bbox=False, + pending_clarification=False, + has_pin=False, + prior_action=None, + prior_target_route=None, + ), + model=selected_model, + ) if _decision_top.user_response: _llm_text = _decision_top.user_response _llm_options = list(_decision_top.options or []) @@ -5422,16 +5789,19 @@ async def unified_query_processor(request: Request): ) _agent = get_clarifier_agent() _pin = pin or req_body.get("vision_pin") - _decision = await _agent.decide(ClarifierInput( - query=natural_query, - has_rendered_map=_has_map, - has_screenshot=_has_shot, - has_last_bbox=_has_bbox, - pending_clarification=False, - has_pin=bool(_pin), - prior_action=router_action.get("action_type"), - prior_target_route=router_action.get("target_route"), - )) + _decision = await _agent.decide( + ClarifierInput( + query=natural_query, + has_rendered_map=_has_map, + has_screenshot=_has_shot, + has_last_bbox=_has_bbox, + pending_clarification=False, + has_pin=bool(_pin), + prior_action=router_action.get("action_type"), + prior_target_route=router_action.get("target_route"), + ), + model=selected_model, + ) if _decision.user_response: _llm_reply = _decision.user_response _llm_options = list(_decision.options or []) @@ -7216,6 +7586,28 @@ async def unified_query_processor(request: Request): collection_id.startswith(oc) or collection_id == oc for oc in optical_collections_needing_mosaic ) + + # PERF: gate mosaic registration on feature count, mirroring the + # Pro-mode threshold below (`pro_mosaic_eligible` requires >=2 + # features -- "registering a one-item mosaic adds latency without + # benefit"). The Public PC path had no such gate: every query -- + # including single-city, single-item results -- paid the full + # ~10-25s mosaic registration round-trip to + # planetarycomputer.microsoft.com/api/data/v1/mosaic/register even + # though a single feature already renders its own full-coverage + # tile with no gap to stitch. Measured as the dominant cost in the + # /api/query pipeline trace (11-26s out of 13-33s total). + # + # NAIP is exempted: its irregular county-scale polygons can leave + # visible gaps within a city-scale bbox even with a single feature + # (see comment above), so it always uses mosaic when eligible. + if needs_mosaic and collection_id != "naip" and (not features or len(features) < 2): + logger.info( + "[MOSAIC] Public mode -- skipping mosaic for %s " + "(features=%d, threshold=2)", + collection_id, len(features) if features else 0, + ) + needs_mosaic = False # MOSAIC ROUTING. # ``get_mosaic_tilejson_url`` registers a search against the diff --git a/planetary-explorer/container-app/pipeline/layer1_agents.py b/planetary-explorer/container-app/pipeline/layer1_agents.py index 8d82380..a300efd 100644 --- a/planetary-explorer/container-app/pipeline/layer1_agents.py +++ b/planetary-explorer/container-app/pipeline/layer1_agents.py @@ -446,7 +446,12 @@ async def run( # ------------------------------------------------------------ from agents.analyst_agent import get_analyst_agent - response = await get_analyst_agent().run(request) + # Honor the frontend model selector for this path too. Without + # this, AnalystAgent silently ignored the user's dropdown choice + # and always used AZURE_OPENAI_DEPLOYMENT_NAME, unlike the + # semantic_translator path which does respect it via set_model(). + _selected_model = body.get("model") if isinstance(body, dict) else None + response = await get_analyst_agent().run(request, model=_selected_model) # The AnalystAgent surfaces clarifications via a "clarify" key in # ``structured`` (set by the ask_user_to_clarify tool). Mirror the diff --git a/planetary-explorer/container-app/semantic_translator.py b/planetary-explorer/container-app/semantic_translator.py index 379f0f5..3ec14a2 100644 --- a/planetary-explorer/container-app/semantic_translator.py +++ b/planetary-explorer/container-app/semantic_translator.py @@ -3236,12 +3236,28 @@ async def tile_selector_agent( - Select tiles that fully cover the bounding box - Prioritize quality over quantity""" - # Use GPT-5 for tile selection - execution_settings = AzureChatPromptExecutionSettings( - max_completion_tokens=2000, - temperature=1.0, # GPT-5 only supports default temperature - top_p=0.95 - ) + # Tile selection runs against whichever model is currently + # active (the user-selectable model, via set_model()/ + # get_active_model()). gpt-5/o-series reasoning models + # reject non-default temperature and burn hidden reasoning + # tokens unless reasoning_effort is capped -- the same fix + # already validated on ContextualAgent. Classic chat models + # (gpt-4o*) reject reasoning_effort but accept temperature. + from semantic_kernel.connectors.ai.open_ai import AzureChatPromptExecutionSettings + _active_model_lc = (self.get_active_model() or "").lower() + _is_reasoning_model = _active_model_lc.startswith(("gpt-5", "o1", "o3", "o4")) + if _is_reasoning_model: + execution_settings = AzureChatPromptExecutionSettings( + max_completion_tokens=2000, + temperature=1.0, # GPT-5 only supports default temperature + reasoning_effort="minimal", + ) + else: + execution_settings = AzureChatPromptExecutionSettings( + max_completion_tokens=2000, + temperature=0.3, + top_p=0.95, + ) # Create chat history chat_history = ChatHistory() @@ -8052,11 +8068,28 @@ async def _generate_response_with_sk(self, prompt_template: str, user_query: str # Prepare data summary as formatted text for the LLM formatted_data_summary = self._format_data_summary_for_llm(data_summary) + # Use fast model (gpt-4o-mini) for final response wording -- this is + # a short "describe the results" narration, the same complexity + # class as the classification/datetime prompts elsewhere in this + # file that already pin service_id="chat-completion-4o". Without + # this, invoke_prompt fell back to the heavy "chat-completion" + # (gpt-5) service with no reasoning_effort cap, which measured as + # a major latency contributor on every successful STAC search + # response (folded into the TILE_URLS pipeline-trace span since + # this call executes just before the tile-URL build loop). + from semantic_kernel.connectors.ai.open_ai import AzureChatPromptExecutionSettings + execution_settings = AzureChatPromptExecutionSettings( + service_id="chat-completion-4o", + temperature=0.4, + max_completion_tokens=500 + ) + # Execute using SK 1.36.2 invoke_prompt arguments = KernelArguments( user_query=user_query, data_summary=formatted_data_summary, - conversation_context=conversation_context or "No previous conversation context." + conversation_context=conversation_context or "No previous conversation context.", + settings=execution_settings ) result = await self.kernel.invoke_prompt( prompt=prompt_template, diff --git a/planetary-explorer/web-ui/src/components/ModelSelector.tsx b/planetary-explorer/web-ui/src/components/ModelSelector.tsx index ea2eecb..d65449f 100644 --- a/planetary-explorer/web-ui/src/components/ModelSelector.tsx +++ b/planetary-explorer/web-ui/src/components/ModelSelector.tsx @@ -20,10 +20,20 @@ interface ModelSelectorProps { const DEFAULT_MODELS: ModelOption[] = [ { - id: 'gpt-5', - name: 'GPT-5', + id: 'gpt-4o-mini', + name: 'GPT-4o mini (fast)', isDefault: true, isAvailable: true + }, + { + id: 'gpt-4o', + name: 'GPT-4o', + isAvailable: true + }, + { + id: 'gpt-5', + name: 'GPT-5 (deep reasoning)', + isAvailable: true } ]; @@ -35,34 +45,31 @@ const ModelSelector: React.FC = ({ onModelChange, selectedMo ); const dropdownRef = useRef(null); - // Fetch available models from health endpoint + // Fetch the real list of deployed models from the backend, which + // discovers them live via Azure's ARM management API (see + // GET /api/models in fastapi_app.py). This replaces guessing model + // availability from the overall /api/health status -- any deployment + // actually present in the tenant (gpt-5, a future gpt-5.5, a custom + // fine-tune, etc.) shows up automatically with no code changes needed. + // Falls back to the static DEFAULT_MODELS list if the call fails, so + // the dropdown is never empty. useEffect(() => { const fetchAvailableModels = async () => { try { - const response = await authenticatedFetch(`${apiBaseUrl}/api/health`); + const response = await authenticatedFetch(`${apiBaseUrl}/api/models`); const data = await response.json(); - // Backend returns 'checks' (not 'connectivity_tests'), and doesn't include available_models list. - // If the OpenAI service is configured/connected, mark gpt-5 as available. - const openaiStatus = data.checks?.azure_openai?.status || data.connectivity_tests?.azure_openai?.status; - const isOpenAiOk = ['connected', 'configured', 'healthy', 'ok'].includes(openaiStatus?.toLowerCase() || ''); - - if (isOpenAiOk) { - // Mark gpt-5 as available since OpenAI is configured - setModels(prevModels => - prevModels.map(model => ({ - ...model, - isAvailable: model.id === 'gpt-5' ? true : model.isAvailable ?? false - })) - ); + if (Array.isArray(data.models) && data.models.length > 0) { + setModels(data.models); } } catch (err) { - console.error('Failed to fetch model availability:', err); + console.error('Failed to fetch available models, keeping fallback list:', err); } }; fetchAvailableModels(); - // Refresh every 30 seconds - const interval = setInterval(fetchAvailableModels, 30000); + // Refresh every 5 minutes -- deployment lists rarely change, and the + // backend itself caches the ARM call for the same window. + const interval = setInterval(fetchAvailableModels, 5 * 60 * 1000); return () => clearInterval(interval); }, [apiBaseUrl]); diff --git a/planetary-explorer/web-ui/src/services/api.ts b/planetary-explorer/web-ui/src/services/api.ts index 6126a4e..b2924c8 100644 --- a/planetary-explorer/web-ui/src/services/api.ts +++ b/planetary-explorer/web-ui/src/services/api.ts @@ -436,7 +436,7 @@ class ApiService { // Use the QueryRequest format for /query endpoint const requestData: any = { query: message, - model: selectedModel || 'gpt-5', // Default to GPT-5 + model: selectedModel || 'gpt-4o-mini', // Default to fast model preferences: { interface_type: 'planetary_explorer', data_source: 'planetary_computer',