From 1010dae9a562ae32a4f9d60765351b141c47a154 Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Sat, 31 Oct 2026 17:16:23 +0200 Subject: [PATCH 01/26] Add DXIL shader debug files and a second teapot round for GPU crashes Register two debug file kinds for GPU shader symbolication: dxil (DXC shader binary, DXBC magic, code_id = 16-hex Aftermath hash) and dxilpdb (DXC shader PDB, MSF magic with an ILDN part). Both share the DXC hash as debug_id and arrive through the existing chunk-upload and DIF assemble API, which now accepts an optional code_id. When teapot answers "partial" with dxil/dxil_pdb entries in missing_difs, look the files up by code_id / debug_id, presign the objectstore-backed ones and call teapot once more with shader_binaries / shader_pdbs and a "-2" idempotency key. Any failure in that round falls back to the first response. Add SENTRY_TEAPOT_INLINE_ATTACHMENT_FALLBACK (default off) so a dev server whose Relay keeps attachments out of objectstore can still hand teapot presigned URLs. --- src/sentry/api/endpoints/debug_files.py | 7 + src/sentry/conf/server.py | 6 + src/sentry/constants.py | 2 + src/sentry/lang/native/shader_difs.py | 139 ++++++ src/sentry/lang/native/teapot.py | 67 ++- src/sentry/models/debugfile.py | 82 +++- src/sentry/tasks/assemble.py | 5 +- src/sentry/tasks/gpu_crash.py | 52 +++ .../lang/native/test_teapot_shader_difs.py | 428 ++++++++++++++++++ tests/sentry/models/test_debugfile_shader.py | 119 +++++ 10 files changed, 901 insertions(+), 6 deletions(-) create mode 100644 src/sentry/lang/native/shader_difs.py create mode 100644 tests/sentry/lang/native/test_teapot_shader_difs.py create mode 100644 tests/sentry/models/test_debugfile_shader.py diff --git a/src/sentry/api/endpoints/debug_files.py b/src/sentry/api/endpoints/debug_files.py index 82011d9baec1..fff6d106615d 100644 --- a/src/sentry/api/endpoints/debug_files.py +++ b/src/sentry/api/endpoints/debug_files.py @@ -587,6 +587,7 @@ class AssembleRequestFile(TypedDict): name: str chunks: list[str] debug_id: NotRequired[str] + code_id: NotRequired[str] AssembleRequestPayload = dict[str, AssembleRequestFile] @@ -609,6 +610,11 @@ class AssembleRequestFile(TypedDict): "pattern": "^[A-Fa-f0-9-]+$", "description": "The debug ID to store the file under, overriding the one read from the file.", }, + "code_id": { + "type": "string", + "pattern": "^[A-Fa-f0-9]+$", + "description": "The code ID to store the file under, for file types the server cannot read it from.", + }, "chunks": { "type": "array", "items": {"type": "string", "pattern": _SHA1_PATTERN}, @@ -779,6 +785,7 @@ def batch_assemble( "project_id": project.id, "name": name, "debug_id": debug_id, + "code_id": file.get("code_id"), "checksum": checksum, "chunks": chunks, } diff --git a/src/sentry/conf/server.py b/src/sentry/conf/server.py index dc32bc683c7c..e4528bb10356 100644 --- a/src/sentry/conf/server.py +++ b/src/sentry/conf/server.py @@ -3417,6 +3417,12 @@ def custom_parameter_sort(parameter: _OperationParameter) -> tuple[str, int]: # Shared secret used to sign requests to teapot SENTRY_TEAPOT_SHARED_SECRET = os.getenv("SENTRY_TEAPOT_SHARED_SECRET", "") +# Dev-only: when Relay did not stream an attachment to objectstore, upload the +# cached bytes ourselves so teapot can still fetch them by presigned URL. +SENTRY_TEAPOT_INLINE_ATTACHMENT_FALLBACK = ( + os.getenv("SENTRY_TEAPOT_INLINE_ATTACHMENT_FALLBACK", "0") == "1" +) + SENTRY_REPLAYS_SERVICE_URL = "http://localhost:8090" # This is useful for testing SSO expiry flows diff --git a/src/sentry/constants.py b/src/sentry/constants.py index 1c35fca3253f..3a0c06b8d497 100644 --- a/src/sentry/constants.py +++ b/src/sentry/constants.py @@ -374,6 +374,8 @@ def parse_log_level(level: str | None) -> LogLevel | None: "application/x-il2cpp-json": "il2cpp", "application/x-portable-pdb": "portablepdb", "application/x-dartsymbolmap+json": "dartsymbolmap", + "application/x-dxil": "dxil", + "application/x-dxil-pdb": "dxilpdb", } NATIVE_UNKNOWN_STRING = "" diff --git a/src/sentry/lang/native/shader_difs.py b/src/sentry/lang/native/shader_difs.py new file mode 100644 index 000000000000..5743e0614b0a --- /dev/null +++ b/src/sentry/lang/native/shader_difs.py @@ -0,0 +1,139 @@ +"""Resolve the shader debug files teapot reports missing to presigned URLs. + +teapot's ``missing_difs`` names two shader kinds: ``dxil`` (the DXC shader +binary, keyed by the 16-hex Aftermath shader hash, stored as ``code_id``) and +``dxil_pdb`` (the DXC shader PDB, keyed by its DebugName, whose stem is the DXC +hash and therefore the ``debug_id``). Both are uploaded as regular project debug +files; only objectstore-backed rows can be presigned. +""" + +from __future__ import annotations + +import logging +import os +import re +from collections.abc import Mapping +from dataclasses import dataclass, field +from datetime import timedelta +from typing import Any + +from symbolic.debuginfo import normalize_debug_id +from symbolic.exceptions import SymbolicError + +from sentry.debug_files.debug_files import maybe_renew_debug_files +from sentry.models.debugfile import ProjectDebugFile +from sentry.models.project import Project +from sentry.objectstore import get_internal_download_url +from sentry.utils import metrics + +logger = logging.getLogger(__name__) + +MISSING_DIF_KIND_DXIL = "dxil" +MISSING_DIF_KIND_DXIL_PDB = "dxil_pdb" + +DIF_TYPE_DXIL = "dxil" +DIF_TYPE_DXIL_PDB = "dxilpdb" + +SHADER_DIF_URL_VALIDITY = timedelta(minutes=15) + +_CODE_ID_RE = re.compile(r"^[0-9a-fA-F]{16}$") +_PDB_SUFFIX_RE = re.compile(r"\.pdb$", re.IGNORECASE) + +ShaderFileRef = tuple[str, str] +"""``(key, storage_url)``: the key teapot asked for and a presigned GET URL.""" + + +@dataclass +class ShaderDifs: + binaries: list[ShaderFileRef] = field(default_factory=list) + pdbs: list[ShaderFileRef] = field(default_factory=list) + + def __bool__(self) -> bool: + return bool(self.binaries or self.pdbs) + + +def missing_shader_difs(response: Mapping[str, Any]) -> tuple[list[str], list[str]]: + """Split teapot's ``missing_difs`` into dxil code ids and dxil_pdb debug names.""" + code_ids: list[str] = [] + debug_names: list[str] = [] + for entry in response.get("missing_difs") or []: + if not isinstance(entry, dict): + continue + debug_id = entry.get("debug_id") + if not isinstance(debug_id, str) or not debug_id: + continue + kind = entry.get("kind") + if kind == MISSING_DIF_KIND_DXIL: + code_ids.append(debug_id) + elif kind == MISSING_DIF_KIND_DXIL_PDB: + debug_names.append(debug_id) + return code_ids, debug_names + + +def debug_id_from_debug_name(debug_name: str) -> str | None: + stem = _PDB_SUFFIX_RE.sub("", os.path.basename(debug_name.replace("\\", "/"))) + try: + return normalize_debug_id(stem) + except SymbolicError: + return None + + +def find_dxil_by_code_id(project: Project, code_id: str) -> ProjectDebugFile | None: + if not _CODE_ID_RE.match(code_id): + return None + difs = ProjectDebugFile.objects.filter( + project_id=project.id, code_id=code_id.lower() + ).order_by("-id") + return next((dif for dif in difs if dif.file_type == DIF_TYPE_DXIL), None) + + +def find_dxil_pdb_by_debug_name(project: Project, debug_name: str) -> ProjectDebugFile | None: + debug_id = debug_id_from_debug_name(debug_name) + if debug_id is None: + return None + difs = ProjectDebugFile.objects.filter(project_id=project.id, debug_id=debug_id).order_by( + "-id" + ) + return next((dif for dif in difs if dif.file_type == DIF_TYPE_DXIL_PDB), None) + + +def presign_dif(dif: ProjectDebugFile) -> str | None: + """Presigned GET URL for an objectstore-backed debug file, or None.""" + if not dif.storage_path: + logger.info( + "gpu_crash.shader_dif.not_in_objectstore", + extra={"dif_id": dif.id, "debug_id": dif.debug_id, "type": dif.file_type}, + ) + return None + return get_internal_download_url( + dif.get_objectstore_session(), dif.storage_path, token_validity=SHADER_DIF_URL_VALIDITY + ) + + +def resolve_shader_difs(project: Project, response: Mapping[str, Any]) -> ShaderDifs: + """Look up every shader DIF teapot reported missing and presign the ones we hold.""" + code_ids, debug_names = missing_shader_difs(response) + resolved = ShaderDifs() + found: list[ProjectDebugFile] = [] + + for code_id in dict.fromkeys(code_ids): + dif = find_dxil_by_code_id(project, code_id) + url = presign_dif(dif) if dif is not None else None + if dif is not None and url is not None: + found.append(dif) + resolved.binaries.append((code_id.lower(), url)) + + for debug_name in dict.fromkeys(debug_names): + dif = find_dxil_pdb_by_debug_name(project, debug_name) + url = presign_dif(dif) if dif is not None else None + if dif is not None and url is not None: + found.append(dif) + resolved.pdbs.append((debug_name, url)) + + missing = len(code_ids) + len(debug_names) + if missing: + metrics.incr("tasks.gpu_crash.shader_difs.missing", amount=missing) + if found: + maybe_renew_debug_files(found) + metrics.incr("tasks.gpu_crash.shader_difs.resolved", amount=len(found)) + return resolved diff --git a/src/sentry/lang/native/teapot.py b/src/sentry/lang/native/teapot.py index b7b0c65d97cf..8f39dd476aed 100644 --- a/src/sentry/lang/native/teapot.py +++ b/src/sentry/lang/native/teapot.py @@ -14,12 +14,14 @@ import logging import re from collections.abc import Sequence +from datetime import timedelta from typing import Any, Protocol import orjson import requests import sentry_sdk from django.conf import settings +from objectstore_client import TimeToLive from sentry import options from sentry.objectstore import UsecaseId, get_internal_download_url, get_session @@ -55,6 +57,18 @@ class TeapotAttachment(Protocol): stored_id: str | None +ShaderFileRef = tuple[str, str] +"""``(key, storage_url)`` for a shader DIF teapot asked for in ``missing_difs``: +the dxil ``code_id`` or the dxil_pdb ``debug_name``, plus a presigned GET URL.""" + +# Suffix on the idempotency/request ids of the second (shader DIF) round, so +# teapot's idempotency cache does not replay the first round's response. +SHADER_DIF_ROUND_SUFFIX = "-2" + +# Dev-only inline attachment uploads live just long enough for the decode. +INLINE_ATTACHMENT_TTL = timedelta(days=1) + + # Fallbacks for the `teapot.timeout-seconds` / `teapot.max-attempts` options. # The timeout is tight on purpose: decode is sub-second, so a slow teapot should # fail fast (the circuit breaker then takes over) rather than hold a worker. @@ -139,6 +153,8 @@ def symbolicate( self, dump: TeapotAttachment, shader_debug_info: Sequence[TeapotAttachment] | None = None, + shader_binaries: Sequence[ShaderFileRef] | None = None, + shader_pdbs: Sequence[ShaderFileRef] | None = None, ) -> dict[str, Any]: shader_debug_info = shader_debug_info or [] url = f"{self.base_url}/symbolicate" @@ -157,12 +173,23 @@ def symbolicate( for att in shader_debug_info ], } + request_id = self.event_id + if shader_binaries or shader_pdbs: + body["shader_binaries"] = [ + {"code_id": code_id, "storage_url": storage_url} + for code_id, storage_url in shader_binaries or [] + ] + body["shader_pdbs"] = [ + {"debug_name": debug_name, "storage_url": storage_url} + for debug_name, storage_url in shader_pdbs or [] + ] + request_id = f"{self.event_id}{SHADER_DIF_ROUND_SUFFIX}" data = orjson.dumps(body) headers = { "X-Teapot-Version": "1", - "X-Request-Id": self.event_id, + "X-Request-Id": request_id, "Content-Type": "application/json", - "Idempotency-Key": self.event_id, + "Idempotency-Key": request_id, **_auth_headers(data), } return self._send(url, headers=headers, data=data) @@ -171,9 +198,31 @@ def _storage_url(self, session: Any, att: TeapotAttachment) -> str: """Short-lived self-authenticating (presigned) GET URL for the attachment — the shared internal objectstore helper (direct link, no auth header).""" if not att.stored_id: - raise TeapotRequestError(f"attachment {att.name!r} is not in objectstore") + if not getattr(settings, "SENTRY_TEAPOT_INLINE_ATTACHMENT_FALLBACK", False): + raise TeapotRequestError(f"attachment {att.name!r} is not in objectstore") + att.stored_id = self._upload_inline(session, att) return get_internal_download_url(session, att.stored_id) + def _upload_inline(self, session: Any, att: TeapotAttachment) -> str: + """Dev fallback: Relay kept the attachment in the processing cache instead + of objectstore, so store the cached bytes ourselves and presign that key.""" + load_data = getattr(att, "load_data", None) + if load_data is None: + raise TeapotRequestError(f"attachment {att.name!r} has no cached data") + payload = load_data(self.project) + key = session.put( + payload, + content_type=getattr(att, "content_type", None) or "application/octet-stream", + filename=att.name, + expiration_policy=TimeToLive(INLINE_ATTACHMENT_TTL), + ) + metrics.incr("tasks.gpu_crash.teapot_inline_attachment_upload") + logger.info( + "teapot.inline_attachment_upload", + extra={"event_id": self.event_id, "name": att.name, "size": len(payload)}, + ) + return key + def _send(self, url: str, headers: dict[str, str], data: bytes) -> dict[str, Any]: def attempt() -> dict[str, Any]: try: @@ -226,6 +275,8 @@ def submit_to_teapot( event_id: str, dump: TeapotAttachment, shader_debug_info: Sequence[TeapotAttachment] | None = None, + shader_binaries: Sequence[ShaderFileRef] | None = None, + shader_pdbs: Sequence[ShaderFileRef] | None = None, ) -> dict[str, Any] | None: """Best-effort teapot invocation. @@ -233,11 +284,19 @@ def submit_to_teapot( URL, missing attachment, 4xx) — teapot is healthy, so the caller should not trip the circuit breaker. Raises ``TeapotUnavailable`` on a genuine outage (network, timeout, 5xx) so the caller can. + + ``shader_binaries`` / ``shader_pdbs`` turn this into the second round that + hands teapot presigned URLs for the shader DIFs it reported missing. """ try: client = TeapotClient(project=project, event_id=event_id) - return client.symbolicate(dump, shader_debug_info or []) + return client.symbolicate( + dump, + shader_debug_info or [], + shader_binaries=shader_binaries, + shader_pdbs=shader_pdbs, + ) except TeapotRequestError as e: logger.info("teapot.request_error", extra={"event_id": event_id, "error": str(e)}) return None diff --git a/src/sentry/models/debugfile.py b/src/sentry/models/debugfile.py index ac6099d05832..62a173ca21b1 100644 --- a/src/sentry/models/debugfile.py +++ b/src/sentry/models/debugfile.py @@ -69,6 +69,14 @@ _proguard_file_re = re.compile(r"/proguard/(?:mapping-)?(.*?)\.txt$") +# DXIL shader binaries (DXC output) carry the DXBC container magic; DXC shader +# PDBs are MSF files whose stream directory names an `ILDN` (debug name) part. +_DXIL_MAGIC = b"DXBC" +_MSF_MAGIC = b"Microsoft C/C++ MSF 7.00" +_DXIL_PDB_PART = b"ILDN" +_DXIL_PDB_SCAN_BYTES = 64 * 1024 +_SHADER_CODE_ID_RE = re.compile(r"^[0-9a-f]{16}$") + def _dif_file_extension(file_format: str, file_type: str | None) -> str: if file_format == "breakpad": @@ -95,6 +103,10 @@ def _dif_file_extension(file_format: str, file_type: str | None) -> str: return ".json" if file_format == "dartsymbolmap": return ".json" + if file_format == "dxil": + return ".dxil" + if file_format == "dxilpdb": + return ".pdb" return "" @@ -452,6 +464,8 @@ def clean_redundant_difs(project: Project, debug_id: str) -> None: bcsymbolmap_seen = False uuidmap_seen = False il2cpp_seen = False + dxil_seen = False + dxilpdb_seen = False for i, dif in enumerate(difs): mime_type = dif.get_content_type() if mime_type == DIF_MIMETYPES["bcsymbolmap"]: @@ -469,6 +483,16 @@ def clean_redundant_difs(project: Project, debug_id: str) -> None: il2cpp_seen = True else: dif.delete() + elif mime_type == DIF_MIMETYPES["dxil"]: + if not dxil_seen: + dxil_seen = True + else: + dif.delete() + elif mime_type == DIF_MIMETYPES["dxilpdb"]: + if not dxilpdb_seen: + dxilpdb_seen = True + else: + dif.delete() else: # We always keep the latest file. If it has no features, likely the # previous files did not have features either and will be removed, or we @@ -483,9 +507,10 @@ def detect_single_dif_from_path( path: str, name: str | None = None, debug_id: str | None = None, + code_id: str | None = None, ) -> DifMeta: """Like :func:`detect_dif_from_path`, but requires exactly one architecture.""" - result = detect_dif_from_path(path, name=name, debug_id=debug_id) + result = detect_dif_from_path(path, name=name, debug_id=debug_id, code_id=code_id) if len(result) != 1: raise BadDif("Object contains %s architectures (1 expected)" % len(result)) @@ -508,6 +533,8 @@ def _get_dif_object_name(meta: DifMeta) -> str: "uuidmap", "il2cpp", "dartsymbolmap", + "dxil", + "dxilpdb", ): return meta.name if meta.file_format == "breakpad": @@ -872,6 +899,8 @@ class DifKind(enum.Enum): UuidMap = enum.auto() Il2Cpp = enum.auto() DartSymbolMap = enum.auto() + Dxil = enum.auto() + DxilPdb = enum.auto() # TODO(flub): should we include proguard? The tradeoff is that we'd be matching the # regex of it twice. That cost is probably not too great to worry about. @@ -891,15 +920,41 @@ def determine_dif_kind(path: str) -> DifKind: return DifKind.Il2Cpp elif data.startswith(b"["): return DifKind.DartSymbolMap + elif data.startswith(_DXIL_MAGIC): + return DifKind.Dxil + elif _MSF_MAGIC.startswith(data): + data += fp.read(_DXIL_PDB_SCAN_BYTES - len(data)) + if data.startswith(_MSF_MAGIC) and _DXIL_PDB_PART in data: + return DifKind.DxilPdb + return DifKind.Object else: return DifKind.Object +def _shader_debug_id(debug_id: str | None, kind: str) -> str: + if debug_id is None: + raise BadDif(f"Missing debug_id for {kind}") + try: + return normalize_debug_id(debug_id) + except SymbolicError as e: + raise BadDif(f"Invalid debug_id for {kind}: {e}") + + +def _shader_code_id(code_id: str | None) -> str | None: + if code_id is None: + return None + code_id = code_id.lower() + if not _SHADER_CODE_ID_RE.match(code_id): + raise BadDif("Invalid code_id for dxil (expected 16 hex characters)") + return code_id + + def detect_dif_from_path( path: str, name: str | None = None, debug_id: str | None = None, accept_unknown: bool = False, + code_id: str | None = None, ) -> list[DifMeta]: """Detects which kind of Debug Information File (DIF) the file at `path` is. @@ -1014,6 +1069,31 @@ def detect_dif_from_path( data=data, ) ] + elif dif_kind == DifKind.Dxil: + logger.debug("File loaded as dxil: %s", path) + return [ + DifMeta( + file_format="dxil", + arch="any", + debug_id=_shader_debug_id(debug_id, "dxil"), + code_id=_shader_code_id(code_id), + name=name, + path=path, + data={"type": "dxil", "features": ["symtab"]}, + ) + ] + elif dif_kind == DifKind.DxilPdb: + logger.debug("File loaded as dxilpdb: %s", path) + return [ + DifMeta( + file_format="dxilpdb", + arch="any", + debug_id=_shader_debug_id(debug_id, "dxilpdb"), + name=name, + path=path, + data={"type": "dxilpdb", "features": ["debug"]}, + ) + ] else: # native debug information files (MachO, ELF or Breakpad) try: diff --git a/src/sentry/tasks/assemble.py b/src/sentry/tasks/assemble.py index ed95e2d7890f..1bdb2b6e354d 100644 --- a/src/sentry/tasks/assemble.py +++ b/src/sentry/tasks/assemble.py @@ -323,6 +323,7 @@ def assemble_dif( checksum: str, chunks: Sequence[str], debug_id: str | None = None, + code_id: str | None = None, **kwargs: object, ) -> None: """ @@ -372,7 +373,9 @@ def assemble_dif( # We only permit split difs to hit this endpoint. # The client is required to split them up first or we error. try: - meta = detect_single_dif_from_path(temp_file.name, name=name, debug_id=debug_id) + meta = detect_single_dif_from_path( + temp_file.name, name=name, debug_id=debug_id, code_id=code_id + ) if file is None: temp_file.seek(0) dif, created = create_dif_from_fileobj(project, meta, temp_file) diff --git a/src/sentry/tasks/gpu_crash.py b/src/sentry/tasks/gpu_crash.py index c5660b0246ab..40fff03c8abe 100644 --- a/src/sentry/tasks/gpu_crash.py +++ b/src/sentry/tasks/gpu_crash.py @@ -19,6 +19,7 @@ from sentry.ingest.event_payload import load_event_payload from sentry.killswitches import killswitch_matches_context from sentry.lang.native.gpu import apply_gpu_crash_symbolication +from sentry.lang.native.shader_difs import resolve_shader_difs from sentry.lang.native.teapot import TeapotAttachment, TeapotUnavailable, submit_to_teapot from sentry.lang.native.utils import ( find_all_shader_debug_attachments, @@ -163,6 +164,8 @@ def _try_symbolicate(data: MutableMapping[str, Any], project_id: int, event_id: return False breaker.record_success() + response = _maybe_resolve_shader_difs(project, event_id, dump, shaders, response) + applied = apply_gpu_crash_symbolication(data, response) metrics.incr( "tasks.gpu_crash.completed", @@ -179,3 +182,52 @@ def _try_symbolicate(data: MutableMapping[str, Any], project_id: int, event_id: ) sentry_sdk.capture_exception(e) return False + + +def _maybe_resolve_shader_difs( + project: Project, + event_id: str, + dump: TeapotAttachment, + shaders: list[Any], + response: dict[str, Any], +) -> dict[str, Any]: + """One extra teapot round with presigned URLs for the shader DIFs (dxil, + dxil_pdb) it reported missing. Best-effort: any failure keeps the first + response, and the breaker is left to the first round.""" + if response.get("status") != "partial": + return response + + try: + difs = resolve_shader_difs(project, response) + if not difs: + metrics.incr("tasks.gpu_crash.shader_difs", tags={"result": "unresolved"}) + return response + with metrics.timer("tasks.gpu_crash.teapot_shader_difs"): + second = submit_to_teapot( + project, + event_id, + dump, + shaders, + shader_binaries=difs.binaries, + shader_pdbs=difs.pdbs, + ) + except TeapotUnavailable: + metrics.incr("tasks.gpu_crash.shader_difs", tags={"result": "unavailable"}) + return response + except Exception as e: + metrics.incr("tasks.gpu_crash.shader_difs", tags={"result": "error"}) + logger.warning( + "tasks.gpu_crash.shader_difs_error", extra={"event_id": event_id, "error": repr(e)} + ) + sentry_sdk.capture_exception(e) + return response + + if second is None: + metrics.incr("tasks.gpu_crash.shader_difs", tags={"result": "request_error"}) + return response + + metrics.incr( + "tasks.gpu_crash.shader_difs", + tags={"result": second.get("status") or "unknown"}, + ) + return second diff --git a/tests/sentry/lang/native/test_teapot_shader_difs.py b/tests/sentry/lang/native/test_teapot_shader_difs.py new file mode 100644 index 000000000000..ff8016d6be36 --- /dev/null +++ b/tests/sentry/lang/native/test_teapot_shader_difs.py @@ -0,0 +1,428 @@ +"""Unit tests for the second (shader DIF) teapot round.""" + +from __future__ import annotations + +from collections.abc import Iterator +from typing import Any +from unittest import mock + +import orjson +import pytest +from django.conf import settings +from django.utils import timezone + +from sentry.lang.native.shader_difs import ( + ShaderDifs, + debug_id_from_debug_name, + missing_shader_difs, + resolve_shader_difs, +) +from sentry.lang.native.teapot import TeapotClient, TeapotRequestError, TeapotUnavailable +from sentry.models.debugfile import ProjectDebugFile +from sentry.models.project import Project +from sentry.tasks.gpu_crash import _try_symbolicate +from sentry.testutils.helpers.options import override_options +from sentry.testutils.pytest.fixtures import django_db_all + +DXC_HASH = "731322bf682bdc554eefdd89098b7383" +DEBUG_ID = "731322bf-682b-dc55-4eef-dd89098b7383" +DEBUG_NAME = f"{DXC_HASH}.pdb" +AFTERMATH_HASH = "0123456789abcdef" + +FIND_DUMP = "sentry.tasks.gpu_crash.find_gpu_crash_dump_attachment" +FIND_SHADERS = "sentry.tasks.gpu_crash.find_all_shader_debug_attachments" +SUBMIT_TEAPOT = "sentry.tasks.gpu_crash.submit_to_teapot" +APPLY = "sentry.tasks.gpu_crash.apply_gpu_crash_symbolication" +RESOLVE = "sentry.tasks.gpu_crash.resolve_shader_difs" + + +class _FakeProject: + def __init__(self, id: int = 42, organization_id: int = 7) -> None: + self.id = id + self.organization_id = organization_id + + +class _FakeAttachment: + def __init__( + self, + name: str = "dump.nv-gpudmp", + stored_id: str | None = "dump-obj-id", + data: bytes = b"dump", + ) -> None: + self.name = name + self.stored_id = stored_id + self.content_type = "application/octet-stream" + self._data = data + + def load_data(self, project: Any = None) -> bytes: + return self._data + + +class _FakeResponse: + def __init__(self, status_code: int, body: Any = None) -> None: + self.status_code = status_code + self._body = body if body is not None else {} + self.text = "" + + def json(self) -> Any: + return self._body + + +def _partial(**overrides: Any) -> dict[str, Any]: + base: dict[str, Any] = { + "status": "partial", + "fault_category": "shader_hang", + "missing_difs": [ + {"debug_id": AFTERMATH_HASH, "kind": "dxil"}, + {"debug_id": DEBUG_NAME, "kind": "dxil_pdb"}, + ], + } + base.update(overrides) + return base + + +def _completed() -> dict[str, Any]: + return {"status": "completed", "fault_category": "shader_hang", "missing_difs": []} + + +def _resolved() -> ShaderDifs: + return ShaderDifs( + binaries=[(AFTERMATH_HASH, "http://objectstore/dxil?sig=x")], + pdbs=[(DEBUG_NAME, "http://objectstore/pdb?sig=x")], + ) + + +@pytest.fixture +def teapot_session() -> Iterator[mock.Mock]: + session = mock.Mock() + session.put.return_value = "inline-key" + get_url = mock.Mock(side_effect=lambda _session, key: f"http://objectstore/{key}?sig=abc") + with ( + mock.patch.object(settings, "SENTRY_TEAPOT_URL", "http://teapot.test", create=True), + mock.patch("sentry.lang.native.teapot.get_session", return_value=session), + mock.patch("sentry.lang.native.teapot.get_internal_download_url", get_url), + ): + yield session + + +# --------------------------------------------------------------------------- +# TeapotClient — second round wire format +# --------------------------------------------------------------------------- + + +def test_client_second_round_sends_shader_arrays(teapot_session: mock.Mock) -> None: + resolved = _resolved() + with mock.patch("sentry.lang.native.teapot.requests.post") as mock_post: + mock_post.return_value = _FakeResponse(200, _completed()) + TeapotClient(_FakeProject(), "abc").symbolicate( + _FakeAttachment(), + [], + shader_binaries=resolved.binaries, + shader_pdbs=resolved.pdbs, + ) + + kwargs = mock_post.call_args.kwargs + assert kwargs["headers"]["Idempotency-Key"] == "abc-2" + assert kwargs["headers"]["X-Request-Id"] == "abc-2" + + body = orjson.loads(kwargs["data"]) + assert body["event_id"] == "abc" + assert body["dump"] == {"storage_url": "http://objectstore/dump-obj-id?sig=abc"} + assert body["shader_binaries"] == [ + {"code_id": AFTERMATH_HASH, "storage_url": "http://objectstore/dxil?sig=x"} + ] + assert body["shader_pdbs"] == [ + {"debug_name": DEBUG_NAME, "storage_url": "http://objectstore/pdb?sig=x"} + ] + + +def test_client_first_round_omits_shader_arrays(teapot_session: mock.Mock) -> None: + with mock.patch("sentry.lang.native.teapot.requests.post") as mock_post: + mock_post.return_value = _FakeResponse(200, _completed()) + TeapotClient(_FakeProject(), "abc").symbolicate(_FakeAttachment(), []) + + kwargs = mock_post.call_args.kwargs + assert kwargs["headers"]["Idempotency-Key"] == "abc" + assert kwargs["headers"]["X-Request-Id"] == "abc" + body = orjson.loads(kwargs["data"]) + assert "shader_binaries" not in body + assert "shader_pdbs" not in body + + +def test_client_inline_fallback_uploads_uncached_attachment(teapot_session: mock.Mock) -> None: + dump = _FakeAttachment(stored_id=None, data=b"gpu-dump-bytes") + with ( + mock.patch.object(settings, "SENTRY_TEAPOT_INLINE_ATTACHMENT_FALLBACK", True, create=True), + mock.patch("sentry.lang.native.teapot.requests.post") as mock_post, + ): + mock_post.return_value = _FakeResponse(200, _completed()) + TeapotClient(_FakeProject(), "abc").symbolicate(dump, []) + + teapot_session.put.assert_called_once() + assert teapot_session.put.call_args.args[0] == b"gpu-dump-bytes" + assert teapot_session.put.call_args.kwargs["filename"] == "dump.nv-gpudmp" + assert dump.stored_id == "inline-key" + body = orjson.loads(mock_post.call_args.kwargs["data"]) + assert body["dump"] == {"storage_url": "http://objectstore/inline-key?sig=abc"} + + +def test_client_inline_fallback_off_by_default(teapot_session: mock.Mock) -> None: + with ( + mock.patch.object( + settings, "SENTRY_TEAPOT_INLINE_ATTACHMENT_FALLBACK", False, create=True + ), + mock.patch("sentry.lang.native.teapot.requests.post") as mock_post, + ): + with pytest.raises(TeapotRequestError): + TeapotClient(_FakeProject(), "abc").symbolicate(_FakeAttachment(stored_id=None), []) + + teapot_session.put.assert_not_called() + assert mock_post.call_count == 0 + + +# --------------------------------------------------------------------------- +# shader_difs — resolving missing_difs against ProjectDebugFile +# --------------------------------------------------------------------------- + + +def test_missing_shader_difs_parsing() -> None: + response = _partial( + missing_difs=[ + {"debug_id": AFTERMATH_HASH, "kind": "dxil"}, + {"debug_id": DEBUG_NAME, "kind": "dxil_pdb"}, + {"debug_id": "deadbeef", "kind": "pdb"}, + {"kind": "dxil"}, + "garbage", + ] + ) + assert missing_shader_difs(response) == ([AFTERMATH_HASH], [DEBUG_NAME]) + assert missing_shader_difs({}) == ([], []) + + +@pytest.mark.parametrize( + "debug_name,expected", + ( + (DEBUG_NAME, DEBUG_ID), + (DXC_HASH.upper() + ".PDB", DEBUG_ID), + ("C:\\build\\shaders\\" + DEBUG_NAME, DEBUG_ID), + (DXC_HASH, DEBUG_ID), + ("not-a-hash.pdb", None), + ), +) +def test_debug_id_from_debug_name(debug_name: str, expected: str | None) -> None: + assert debug_id_from_debug_name(debug_name) == expected + + +def _create_dif( + project: Project, + dif_type: str, + code_id: str | None = None, + storage_path: str | None = "difs/shader", +) -> ProjectDebugFile: + content_type = "application/x-dxil" if dif_type == "dxil" else "application/x-dxil-pdb" + features = ["symtab"] if dif_type == "dxil" else ["debug"] + return ProjectDebugFile.objects.create( + project_id=project.id, + debug_id=DEBUG_ID, + code_id=code_id, + object_name="shader", + cpu_name="any", + checksum="0" * 40, + data={"type": dif_type, "features": features}, + storage_path=storage_path, + content_type=content_type, + file_size=64, + date_created=timezone.now(), + ) + + +@pytest.fixture +def dif_presign() -> Iterator[mock.Mock]: + get_url = mock.Mock( + side_effect=lambda _session, key, token_validity=None: f"http://objectstore/{key}?sig=x" + ) + with ( + mock.patch.object(ProjectDebugFile, "get_objectstore_session", return_value=mock.Mock()), + mock.patch("sentry.lang.native.shader_difs.get_internal_download_url", get_url), + ): + yield get_url + + +@django_db_all +def test_resolve_shader_difs_presigns_objectstore_rows( + default_project: Project, dif_presign: mock.Mock +) -> None: + _create_dif(default_project, "dxil", code_id=AFTERMATH_HASH, storage_path="difs/dxil") + _create_dif(default_project, "dxilpdb", storage_path="difs/pdb") + + resolved = resolve_shader_difs(default_project, _partial()) + + assert bool(resolved) is True + assert resolved.binaries == [(AFTERMATH_HASH, "http://objectstore/difs/dxil?sig=x")] + assert resolved.pdbs == [(DEBUG_NAME, "http://objectstore/difs/pdb?sig=x")] + # Token validity is the longer DIF one, not the attachment default. + assert dif_presign.call_args.kwargs["token_validity"] is not None + + +@django_db_all +def test_resolve_shader_difs_matches_code_id_case_insensitively( + default_project: Project, dif_presign: mock.Mock +) -> None: + _create_dif(default_project, "dxil", code_id=AFTERMATH_HASH) + response = _partial(missing_difs=[{"debug_id": AFTERMATH_HASH.upper(), "kind": "dxil"}]) + + resolved = resolve_shader_difs(default_project, response) + + assert resolved.binaries == [(AFTERMATH_HASH, "http://objectstore/difs/shader?sig=x")] + assert resolved.pdbs == [] + + +@django_db_all +def test_resolve_shader_difs_ignores_other_types( + default_project: Project, dif_presign: mock.Mock +) -> None: + # Same debug_id, but the row is the binary; teapot asked for the PDB. + _create_dif(default_project, "dxil", code_id=AFTERMATH_HASH) + response = _partial(missing_difs=[{"debug_id": DEBUG_NAME, "kind": "dxil_pdb"}]) + + resolved = resolve_shader_difs(default_project, response) + + assert bool(resolved) is False + + +@django_db_all +def test_resolve_shader_difs_skips_legacy_rows( + default_project: Project, dif_presign: mock.Mock +) -> None: + _create_dif(default_project, "dxil", code_id=AFTERMATH_HASH, storage_path=None) + _create_dif(default_project, "dxilpdb", storage_path=None) + + resolved = resolve_shader_difs(default_project, _partial()) + + assert bool(resolved) is False + dif_presign.assert_not_called() + + +@django_db_all +def test_resolve_shader_difs_nothing_missing(default_project: Project) -> None: + assert bool(resolve_shader_difs(default_project, _completed())) is False + + +# --------------------------------------------------------------------------- +# Task — second round orchestration +# --------------------------------------------------------------------------- + + +@pytest.fixture(autouse=True) +def circuit_breaker() -> Iterator[mock.Mock]: + breaker = mock.Mock() + breaker.should_allow_request.return_value = True + with mock.patch("sentry.tasks.gpu_crash._teapot_circuit_breaker", return_value=breaker): + yield breaker + + +@django_db_all +def test_task_runs_second_round_with_resolved_difs( + default_project: Project, circuit_breaker: mock.Mock +) -> None: + data: dict[str, Any] = {"event_id": "e", "project": default_project.id} + partial, completed = _partial(), _completed() + with ( + override_options({"teapot.enabled": True}), + mock.patch(FIND_DUMP, return_value=_FakeAttachment()), + mock.patch(FIND_SHADERS, return_value=[]), + mock.patch(RESOLVE, return_value=_resolved()) as resolve, + mock.patch(SUBMIT_TEAPOT, side_effect=[partial, completed]) as teapot, + mock.patch(APPLY, return_value=data) as apply, + ): + changed = _try_symbolicate(data, default_project.id, "e") + + assert changed is True + assert resolve.call_args.args[1] is partial + assert teapot.call_count == 2 + second = teapot.call_args_list[1] + assert second.kwargs["shader_binaries"] == _resolved().binaries + assert second.kwargs["shader_pdbs"] == _resolved().pdbs + assert apply.call_args.args[1] is completed + circuit_breaker.record_success.assert_called_once() + circuit_breaker.record_error.assert_not_called() + + +@django_db_all +def test_task_keeps_first_response_when_nothing_resolved(default_project: Project) -> None: + data: dict[str, Any] = {"event_id": "e", "project": default_project.id} + partial = _partial() + with ( + override_options({"teapot.enabled": True}), + mock.patch(FIND_DUMP, return_value=_FakeAttachment()), + mock.patch(FIND_SHADERS, return_value=[]), + mock.patch(RESOLVE, return_value=ShaderDifs()), + mock.patch(SUBMIT_TEAPOT, return_value=partial) as teapot, + mock.patch(APPLY, return_value=data) as apply, + ): + _try_symbolicate(data, default_project.id, "e") + + assert teapot.call_count == 1 + assert apply.call_args.args[1] is partial + + +@django_db_all +def test_task_skips_second_round_when_completed(default_project: Project) -> None: + data: dict[str, Any] = {"event_id": "e", "project": default_project.id} + with ( + override_options({"teapot.enabled": True}), + mock.patch(FIND_DUMP, return_value=_FakeAttachment()), + mock.patch(FIND_SHADERS, return_value=[]), + mock.patch(RESOLVE) as resolve, + mock.patch(SUBMIT_TEAPOT, return_value=_completed()) as teapot, + mock.patch(APPLY, return_value=data), + ): + _try_symbolicate(data, default_project.id, "e") + + assert teapot.call_count == 1 + resolve.assert_not_called() + + +@pytest.mark.parametrize("second_outcome", (TeapotUnavailable("down"), None, RuntimeError("x"))) +@django_db_all +def test_task_second_round_failure_falls_back_to_first_response( + default_project: Project, circuit_breaker: mock.Mock, second_outcome: Any +) -> None: + data: dict[str, Any] = {"event_id": "e", "project": default_project.id} + partial = _partial() + with ( + override_options({"teapot.enabled": True}), + mock.patch(FIND_DUMP, return_value=_FakeAttachment()), + mock.patch(FIND_SHADERS, return_value=[]), + mock.patch(RESOLVE, return_value=_resolved()), + mock.patch(SUBMIT_TEAPOT, side_effect=[partial, second_outcome]) as teapot, + mock.patch(APPLY, return_value=data) as apply, + mock.patch("sentry.tasks.gpu_crash.sentry_sdk.capture_exception"), + ): + changed = _try_symbolicate(data, default_project.id, "e") + + assert changed is True + assert teapot.call_count == 2 + assert apply.call_args.args[1] is partial + circuit_breaker.record_success.assert_called_once() + circuit_breaker.record_error.assert_not_called() + + +@django_db_all +def test_task_resolver_error_falls_back_to_first_response(default_project: Project) -> None: + data: dict[str, Any] = {"event_id": "e", "project": default_project.id} + partial = _partial() + with ( + override_options({"teapot.enabled": True}), + mock.patch(FIND_DUMP, return_value=_FakeAttachment()), + mock.patch(FIND_SHADERS, return_value=[]), + mock.patch(RESOLVE, side_effect=RuntimeError("db down")), + mock.patch(SUBMIT_TEAPOT, return_value=partial) as teapot, + mock.patch(APPLY, return_value=data) as apply, + mock.patch("sentry.tasks.gpu_crash.sentry_sdk.capture_exception"), + ): + changed = _try_symbolicate(data, default_project.id, "e") + + assert changed is True + assert teapot.call_count == 1 + assert apply.call_args.args[1] is partial diff --git a/tests/sentry/models/test_debugfile_shader.py b/tests/sentry/models/test_debugfile_shader.py new file mode 100644 index 000000000000..9649fd984c23 --- /dev/null +++ b/tests/sentry/models/test_debugfile_shader.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from sentry.constants import KNOWN_DIF_FORMATS +from sentry.models.debugfile import ( + DIF_MIMETYPES, + BadDif, + DifKind, + _dif_file_extension, + _get_dif_object_name, + detect_dif_from_path, + detect_single_dif_from_path, + determine_dif_kind, +) + +DXC_HASH = "731322bf682bdc554eefdd89098b7383" +DEBUG_ID = "731322bf-682b-dc55-4eef-dd89098b7383" +AFTERMATH_HASH = "0123456789abcdef" + +DXIL_BYTES = b"DXBC" + b"\x00" * 60 +MSF_MAGIC = b"Microsoft C/C++ MSF 7.00\r\n\x1aDS\x00\x00\x00" +DXIL_PDB_BYTES = MSF_MAGIC + b"\x00" * 4096 + b"ILDN" + b"\x00" * 64 +PLAIN_PDB_BYTES = MSF_MAGIC + b"\x00" * 4096 + + +def _write(tmp_path: Path, name: str, data: bytes) -> str: + path = tmp_path / name + path.write_bytes(data) + return str(path) + + +def test_shader_formats_registered() -> None: + assert KNOWN_DIF_FORMATS["application/x-dxil"] == "dxil" + assert KNOWN_DIF_FORMATS["application/x-dxil-pdb"] == "dxilpdb" + assert DIF_MIMETYPES["dxil"] == "application/x-dxil" + assert DIF_MIMETYPES["dxilpdb"] == "application/x-dxil-pdb" + assert _dif_file_extension("dxil", "dxil") == ".dxil" + assert _dif_file_extension("dxilpdb", "dxilpdb") == ".pdb" + + +def test_determine_dif_kind_dxil(tmp_path: Path) -> None: + assert determine_dif_kind(_write(tmp_path, "a.dxil", DXIL_BYTES)) == DifKind.Dxil + + +def test_determine_dif_kind_dxil_pdb(tmp_path: Path) -> None: + assert determine_dif_kind(_write(tmp_path, "a.pdb", DXIL_PDB_BYTES)) == DifKind.DxilPdb + + +def test_determine_dif_kind_plain_pdb_is_object(tmp_path: Path) -> None: + # A regular MSF PDB has no ILDN part and stays with symbolic. + assert determine_dif_kind(_write(tmp_path, "a.pdb", PLAIN_PDB_BYTES)) == DifKind.Object + + +def test_determine_dif_kind_ildn_outside_scan_window_is_object(tmp_path: Path) -> None: + data = MSF_MAGIC + b"\x00" * (64 * 1024) + b"ILDN" + assert determine_dif_kind(_write(tmp_path, "a.pdb", data)) == DifKind.Object + + +def test_detect_dxil(tmp_path: Path) -> None: + path = _write(tmp_path, "shader.dxil", DXIL_BYTES) + + (meta,) = detect_dif_from_path( + path, name="shader.dxil", debug_id=DXC_HASH, code_id=AFTERMATH_HASH.upper() + ) + + assert meta.file_format == "dxil" + assert meta.arch == "any" + assert meta.debug_id == DEBUG_ID + assert meta.code_id == AFTERMATH_HASH + assert meta.name == "shader.dxil" + assert meta.data == {"type": "dxil", "features": ["symtab"]} + assert _get_dif_object_name(meta) == "shader.dxil" + + +def test_detect_dxil_without_code_id(tmp_path: Path) -> None: + path = _write(tmp_path, "shader.dxil", DXIL_BYTES) + + meta = detect_single_dif_from_path(path, name="shader.dxil", debug_id=DEBUG_ID) + + assert meta.file_format == "dxil" + assert meta.code_id is None + + +def test_detect_dxil_pdb(tmp_path: Path) -> None: + path = _write(tmp_path, f"{DXC_HASH}.pdb", DXIL_PDB_BYTES) + + meta = detect_single_dif_from_path(path, name=f"{DXC_HASH}.pdb", debug_id=DXC_HASH) + + assert meta.file_format == "dxilpdb" + assert meta.arch == "any" + assert meta.debug_id == DEBUG_ID + assert meta.code_id is None + assert meta.data == {"type": "dxilpdb", "features": ["debug"]} + assert _get_dif_object_name(meta) == f"{DXC_HASH}.pdb" + + +@pytest.mark.parametrize("data", (DXIL_BYTES, DXIL_PDB_BYTES)) +def test_detect_shader_requires_debug_id(tmp_path: Path, data: bytes) -> None: + path = _write(tmp_path, "shader.bin", data) + + with pytest.raises(BadDif, match="Missing debug_id"): + detect_dif_from_path(path, name="shader.bin") + + +def test_detect_shader_rejects_invalid_debug_id(tmp_path: Path) -> None: + path = _write(tmp_path, "shader.dxil", DXIL_BYTES) + + with pytest.raises(BadDif, match="Invalid debug_id"): + detect_dif_from_path(path, name="shader.dxil", debug_id="not-a-debug-id") + + +def test_detect_dxil_rejects_invalid_code_id(tmp_path: Path) -> None: + path = _write(tmp_path, "shader.dxil", DXIL_BYTES) + + with pytest.raises(BadDif, match="Invalid code_id"): + detect_dif_from_path(path, name="shader.dxil", debug_id=DXC_HASH, code_id="abc") From b8cd274cd4acac8a69cea055003c70d5d0c108db Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Sat, 31 Oct 2026 18:10:55 +0200 Subject: [PATCH 02/26] Send the sibling shader PDB with each resolved DXIL binary --- src/sentry/lang/native/shader_difs.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/src/sentry/lang/native/shader_difs.py b/src/sentry/lang/native/shader_difs.py index 5743e0614b0a..3943de78ac3b 100644 --- a/src/sentry/lang/native/shader_difs.py +++ b/src/sentry/lang/native/shader_difs.py @@ -97,6 +97,24 @@ def find_dxil_pdb_by_debug_name(project: Project, debug_name: str) -> ProjectDeb return next((dif for dif in difs if dif.file_type == DIF_TYPE_DXIL_PDB), None) +def find_sibling_pdb(dxil: ProjectDebugFile) -> tuple[str, ProjectDebugFile] | None: + """The PDB uploaded next to a DXIL binary, keyed by the DebugName the decoder will ask for. + + The decoder requests a shader's PDB only after it has received the binary, so + sending the sibling PDB along with the binary saves one extra round trip. + """ + difs = ProjectDebugFile.objects.filter(project_id=dxil.project_id, debug_id=dxil.debug_id).order_by( + "-id" + ) + pdb = next((dif for dif in difs if dif.file_type == DIF_TYPE_DXIL_PDB), None) + if pdb is None: + return None + stem = os.path.splitext(os.path.basename(dxil.object_name or ""))[0] + if not stem: + stem = dxil.debug_id.replace("-", "")[:32] + return f"{stem}.pdb", pdb + + def presign_dif(dif: ProjectDebugFile) -> str | None: """Presigned GET URL for an objectstore-backed debug file, or None.""" if not dif.storage_path: @@ -122,6 +140,13 @@ def resolve_shader_difs(project: Project, response: Mapping[str, Any]) -> Shader if dif is not None and url is not None: found.append(dif) resolved.binaries.append((code_id.lower(), url)) + sibling = find_sibling_pdb(dif) + if sibling is not None: + debug_name, pdb = sibling + pdb_url = presign_dif(pdb) + if pdb_url is not None and debug_name.lower() not in {n.lower() for n in debug_names}: + found.append(pdb) + resolved.pdbs.append((debug_name, pdb_url)) for debug_name in dict.fromkeys(debug_names): dif = find_dxil_pdb_by_debug_name(project, debug_name) From 4b24dbbd2e18b941d7dd6cbee7fbc3bec819977a Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Sat, 31 Oct 2026 18:16:06 +0200 Subject: [PATCH 03/26] Name the sibling shader PDB after its own object name --- src/sentry/lang/native/shader_difs.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/sentry/lang/native/shader_difs.py b/src/sentry/lang/native/shader_difs.py index 3943de78ac3b..d2b77eb30c3a 100644 --- a/src/sentry/lang/native/shader_difs.py +++ b/src/sentry/lang/native/shader_difs.py @@ -109,10 +109,10 @@ def find_sibling_pdb(dxil: ProjectDebugFile) -> tuple[str, ProjectDebugFile] | N pdb = next((dif for dif in difs if dif.file_type == DIF_TYPE_DXIL_PDB), None) if pdb is None: return None - stem = os.path.splitext(os.path.basename(dxil.object_name or ""))[0] - if not stem: - stem = dxil.debug_id.replace("-", "")[:32] - return f"{stem}.pdb", pdb + name = os.path.basename((pdb.object_name or "").replace("\\", "/")) + if not _PDB_SUFFIX_RE.search(name): + name = f"{dxil.debug_id.replace('-', '')[:32]}.pdb" + return name, pdb def presign_dif(dif: ProjectDebugFile) -> str | None: From 0f41cef027f233883374387600a9420d6cd00560 Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Sun, 1 Nov 2026 16:07:55 +0200 Subject: [PATCH 04/26] Store the shader group on DXIL debug files and pass it to teapot --- src/sentry/api/endpoints/debug_files.py | 8 ++ src/sentry/lang/native/shader_difs.py | 31 +++-- src/sentry/lang/native/teapot.py | 27 ++++- src/sentry/models/debugfile.py | 46 +++++++- src/sentry/tasks/assemble.py | 7 +- .../lang/native/test_teapot_shader_difs.py | 65 +++++++++-- tests/sentry/models/test_debugfile_shader.py | 110 ++++++++++++++++++ 7 files changed, 262 insertions(+), 32 deletions(-) diff --git a/src/sentry/api/endpoints/debug_files.py b/src/sentry/api/endpoints/debug_files.py index fff6d106615d..5f00c41d6fe9 100644 --- a/src/sentry/api/endpoints/debug_files.py +++ b/src/sentry/api/endpoints/debug_files.py @@ -54,6 +54,7 @@ from sentry.debug_files.upload import find_missing_chunks from sentry.lang.native.sources import record_last_upload from sentry.models.debugfile import ( + SHADER_GROUP_MAX_LENGTH, ProguardArtifactRelease, ProjectDebugFile, build_proguard_reupload_dif_meta, @@ -588,6 +589,7 @@ class AssembleRequestFile(TypedDict): chunks: list[str] debug_id: NotRequired[str] code_id: NotRequired[str] + shader_group: NotRequired[str] AssembleRequestPayload = dict[str, AssembleRequestFile] @@ -615,6 +617,11 @@ class AssembleRequestFile(TypedDict): "pattern": "^[A-Fa-f0-9]+$", "description": "The code ID to store the file under, for file types the server cannot read it from.", }, + "shader_group": { + "type": "string", + "maxLength": SHADER_GROUP_MAX_LENGTH, + "description": "For `dxil` files: the shader group the binary belongs to, as recorded by the engine's shader symbol export.", + }, "chunks": { "type": "array", "items": {"type": "string", "pattern": _SHA1_PATTERN}, @@ -786,6 +793,7 @@ def batch_assemble( "name": name, "debug_id": debug_id, "code_id": file.get("code_id"), + "shader_group": file.get("shader_group"), "checksum": checksum, "chunks": chunks, } diff --git a/src/sentry/lang/native/shader_difs.py b/src/sentry/lang/native/shader_difs.py index d2b77eb30c3a..c4caadf03a03 100644 --- a/src/sentry/lang/native/shader_difs.py +++ b/src/sentry/lang/native/shader_difs.py @@ -21,6 +21,7 @@ from symbolic.exceptions import SymbolicError from sentry.debug_files.debug_files import maybe_renew_debug_files +from sentry.lang.native.teapot import ShaderBinaryRef from sentry.models.debugfile import ProjectDebugFile from sentry.models.project import Project from sentry.objectstore import get_internal_download_url @@ -45,7 +46,7 @@ @dataclass class ShaderDifs: - binaries: list[ShaderFileRef] = field(default_factory=list) + binaries: list[ShaderBinaryRef] = field(default_factory=list) pdbs: list[ShaderFileRef] = field(default_factory=list) def __bool__(self) -> bool: @@ -81,9 +82,9 @@ def debug_id_from_debug_name(debug_name: str) -> str | None: def find_dxil_by_code_id(project: Project, code_id: str) -> ProjectDebugFile | None: if not _CODE_ID_RE.match(code_id): return None - difs = ProjectDebugFile.objects.filter( - project_id=project.id, code_id=code_id.lower() - ).order_by("-id") + difs = ProjectDebugFile.objects.filter(project_id=project.id, code_id=code_id.lower()).order_by( + "-id" + ) return next((dif for dif in difs if dif.file_type == DIF_TYPE_DXIL), None) @@ -91,9 +92,7 @@ def find_dxil_pdb_by_debug_name(project: Project, debug_name: str) -> ProjectDeb debug_id = debug_id_from_debug_name(debug_name) if debug_id is None: return None - difs = ProjectDebugFile.objects.filter(project_id=project.id, debug_id=debug_id).order_by( - "-id" - ) + difs = ProjectDebugFile.objects.filter(project_id=project.id, debug_id=debug_id).order_by("-id") return next((dif for dif in difs if dif.file_type == DIF_TYPE_DXIL_PDB), None) @@ -103,9 +102,9 @@ def find_sibling_pdb(dxil: ProjectDebugFile) -> tuple[str, ProjectDebugFile] | N The decoder requests a shader's PDB only after it has received the binary, so sending the sibling PDB along with the binary saves one extra round trip. """ - difs = ProjectDebugFile.objects.filter(project_id=dxil.project_id, debug_id=dxil.debug_id).order_by( - "-id" - ) + difs = ProjectDebugFile.objects.filter( + project_id=dxil.project_id, debug_id=dxil.debug_id + ).order_by("-id") pdb = next((dif for dif in difs if dif.file_type == DIF_TYPE_DXIL_PDB), None) if pdb is None: return None @@ -115,6 +114,12 @@ def find_sibling_pdb(dxil: ProjectDebugFile) -> tuple[str, ProjectDebugFile] | N return name, pdb +def shader_group_of(dif: ProjectDebugFile) -> str | None: + """The shader group recorded at upload time (``data["shader_group"]``), or None.""" + group = (dif.data or {}).get("shader_group") + return group if isinstance(group, str) and group else None + + def presign_dif(dif: ProjectDebugFile) -> str | None: """Presigned GET URL for an objectstore-backed debug file, or None.""" if not dif.storage_path: @@ -139,12 +144,14 @@ def resolve_shader_difs(project: Project, response: Mapping[str, Any]) -> Shader url = presign_dif(dif) if dif is not None else None if dif is not None and url is not None: found.append(dif) - resolved.binaries.append((code_id.lower(), url)) + resolved.binaries.append(ShaderBinaryRef(code_id.lower(), url, shader_group_of(dif))) sibling = find_sibling_pdb(dif) if sibling is not None: debug_name, pdb = sibling pdb_url = presign_dif(pdb) - if pdb_url is not None and debug_name.lower() not in {n.lower() for n in debug_names}: + if pdb_url is not None and debug_name.lower() not in { + n.lower() for n in debug_names + }: found.append(pdb) resolved.pdbs.append((debug_name, pdb_url)) diff --git a/src/sentry/lang/native/teapot.py b/src/sentry/lang/native/teapot.py index 8f39dd476aed..194398426619 100644 --- a/src/sentry/lang/native/teapot.py +++ b/src/sentry/lang/native/teapot.py @@ -14,6 +14,7 @@ import logging import re from collections.abc import Sequence +from dataclasses import dataclass from datetime import timedelta from typing import Any, Protocol @@ -61,6 +62,23 @@ class TeapotAttachment(Protocol): """``(key, storage_url)`` for a shader DIF teapot asked for in ``missing_difs``: the dxil ``code_id`` or the dxil_pdb ``debug_name``, plus a presigned GET URL.""" + +@dataclass(frozen=True) +class ShaderBinaryRef: + """A dxil binary teapot asked for: its Aftermath ``code_id``, a presigned GET + URL, and the shader group recorded at upload time, if any.""" + + code_id: str + storage_url: str + group: str | None = None + + def as_request_entry(self) -> dict[str, str]: + entry = {"code_id": self.code_id, "storage_url": self.storage_url} + if self.group: + entry["group"] = self.group + return entry + + # Suffix on the idempotency/request ids of the second (shader DIF) round, so # teapot's idempotency cache does not replay the first round's response. SHADER_DIF_ROUND_SUFFIX = "-2" @@ -153,7 +171,7 @@ def symbolicate( self, dump: TeapotAttachment, shader_debug_info: Sequence[TeapotAttachment] | None = None, - shader_binaries: Sequence[ShaderFileRef] | None = None, + shader_binaries: Sequence[ShaderBinaryRef] | None = None, shader_pdbs: Sequence[ShaderFileRef] | None = None, ) -> dict[str, Any]: shader_debug_info = shader_debug_info or [] @@ -175,10 +193,7 @@ def symbolicate( } request_id = self.event_id if shader_binaries or shader_pdbs: - body["shader_binaries"] = [ - {"code_id": code_id, "storage_url": storage_url} - for code_id, storage_url in shader_binaries or [] - ] + body["shader_binaries"] = [ref.as_request_entry() for ref in shader_binaries or []] body["shader_pdbs"] = [ {"debug_name": debug_name, "storage_url": storage_url} for debug_name, storage_url in shader_pdbs or [] @@ -275,7 +290,7 @@ def submit_to_teapot( event_id: str, dump: TeapotAttachment, shader_debug_info: Sequence[TeapotAttachment] | None = None, - shader_binaries: Sequence[ShaderFileRef] | None = None, + shader_binaries: Sequence[ShaderBinaryRef] | None = None, shader_pdbs: Sequence[ShaderFileRef] | None = None, ) -> dict[str, Any] | None: """Best-effort teapot invocation. diff --git a/src/sentry/models/debugfile.py b/src/sentry/models/debugfile.py index 62a173ca21b1..56e6de2bebcf 100644 --- a/src/sentry/models/debugfile.py +++ b/src/sentry/models/debugfile.py @@ -77,6 +77,9 @@ _DXIL_PDB_SCAN_BYTES = 64 * 1024 _SHADER_CODE_ID_RE = re.compile(r"^[0-9a-f]{16}$") +# Upper bound on the free-text shader group recorded with a dxil debug file. +SHADER_GROUP_MAX_LENGTH = 512 + def _dif_file_extension(file_format: str, file_type: str | None) -> str: if file_format == "breakpad": @@ -508,9 +511,12 @@ def detect_single_dif_from_path( name: str | None = None, debug_id: str | None = None, code_id: str | None = None, + shader_group: str | None = None, ) -> DifMeta: """Like :func:`detect_dif_from_path`, but requires exactly one architecture.""" - result = detect_dif_from_path(path, name=name, debug_id=debug_id, code_id=code_id) + result = detect_dif_from_path( + path, name=name, debug_id=debug_id, code_id=code_id, shader_group=shader_group + ) if len(result) != 1: raise BadDif("Object contains %s architectures (1 expected)" % len(result)) @@ -558,6 +564,24 @@ def find_existing_dif(project: Project, meta: DifMeta, checksum: str) -> Project ) +def _backfill_shader_group(dif: ProjectDebugFile, meta: DifMeta) -> None: + """Record the shader group on an existing ``dxil`` row that was uploaded without one. + + Re-uploading the same shader binary is a no-op for the file, but a later cook may + know the group the shader belongs to. A group already on the row is kept as-is. + """ + if meta.file_format != "dxil": + return + shader_group = (meta.data or {}).get("shader_group") + if not shader_group: + return + data = dict(dif.data or {}) + if data.get("type") != "dxil" or data.get("shader_group"): + return + data["shader_group"] = shader_group + dif.update(data=data) + + def _get_fileobj_checksum_and_size(fileobj: IO[bytes]) -> tuple[str, int]: file_size = 0 hasher = hashlib.sha1() @@ -592,6 +616,7 @@ def create_dif_from_file( dif = find_existing_dif(project, meta, checksum) if dif is not None: + _backfill_shader_group(dif, meta) return dif, False object_name = _get_dif_object_name(meta) @@ -678,6 +703,7 @@ def create_dif_from_fileobj( dif = find_existing_dif(project, meta, checksum) if dif is not None: + _backfill_shader_group(dif, meta) return dif, False object_name = _get_dif_object_name(meta) @@ -949,12 +975,24 @@ def _shader_code_id(code_id: str | None) -> str | None: return code_id +def _shader_group(shader_group: str | None) -> str | None: + if shader_group is None: + return None + shader_group = shader_group.strip() + if not shader_group: + return None + if len(shader_group) > SHADER_GROUP_MAX_LENGTH: + raise BadDif(f"Invalid shader_group for dxil (max {SHADER_GROUP_MAX_LENGTH} characters)") + return shader_group + + def detect_dif_from_path( path: str, name: str | None = None, debug_id: str | None = None, accept_unknown: bool = False, code_id: str | None = None, + shader_group: str | None = None, ) -> list[DifMeta]: """Detects which kind of Debug Information File (DIF) the file at `path` is. @@ -1071,6 +1109,10 @@ def detect_dif_from_path( ] elif dif_kind == DifKind.Dxil: logger.debug("File loaded as dxil: %s", path) + data = {"type": "dxil", "features": ["symtab"]} + group = _shader_group(shader_group) + if group is not None: + data["shader_group"] = group return [ DifMeta( file_format="dxil", @@ -1079,7 +1121,7 @@ def detect_dif_from_path( code_id=_shader_code_id(code_id), name=name, path=path, - data={"type": "dxil", "features": ["symtab"]}, + data=data, ) ] elif dif_kind == DifKind.DxilPdb: diff --git a/src/sentry/tasks/assemble.py b/src/sentry/tasks/assemble.py index 1bdb2b6e354d..e6f69b388e71 100644 --- a/src/sentry/tasks/assemble.py +++ b/src/sentry/tasks/assemble.py @@ -324,6 +324,7 @@ def assemble_dif( chunks: Sequence[str], debug_id: str | None = None, code_id: str | None = None, + shader_group: str | None = None, **kwargs: object, ) -> None: """ @@ -374,7 +375,11 @@ def assemble_dif( # The client is required to split them up first or we error. try: meta = detect_single_dif_from_path( - temp_file.name, name=name, debug_id=debug_id, code_id=code_id + temp_file.name, + name=name, + debug_id=debug_id, + code_id=code_id, + shader_group=shader_group, ) if file is None: temp_file.seek(0) diff --git a/tests/sentry/lang/native/test_teapot_shader_difs.py b/tests/sentry/lang/native/test_teapot_shader_difs.py index ff8016d6be36..3c71eeaaaca2 100644 --- a/tests/sentry/lang/native/test_teapot_shader_difs.py +++ b/tests/sentry/lang/native/test_teapot_shader_difs.py @@ -17,7 +17,12 @@ missing_shader_difs, resolve_shader_difs, ) -from sentry.lang.native.teapot import TeapotClient, TeapotRequestError, TeapotUnavailable +from sentry.lang.native.teapot import ( + ShaderBinaryRef, + TeapotClient, + TeapotRequestError, + TeapotUnavailable, +) from sentry.models.debugfile import ProjectDebugFile from sentry.models.project import Project from sentry.tasks.gpu_crash import _try_symbolicate @@ -28,6 +33,7 @@ DEBUG_ID = "731322bf-682b-dc55-4eef-dd89098b7383" DEBUG_NAME = f"{DXC_HASH}.pdb" AFTERMATH_HASH = "0123456789abcdef" +SHADER_GROUP = "GlobalShaders" FIND_DUMP = "sentry.tasks.gpu_crash.find_gpu_crash_dump_attachment" FIND_SHADERS = "sentry.tasks.gpu_crash.find_all_shader_debug_attachments" @@ -85,9 +91,9 @@ def _completed() -> dict[str, Any]: return {"status": "completed", "fault_category": "shader_hang", "missing_difs": []} -def _resolved() -> ShaderDifs: +def _resolved(group: str | None = SHADER_GROUP) -> ShaderDifs: return ShaderDifs( - binaries=[(AFTERMATH_HASH, "http://objectstore/dxil?sig=x")], + binaries=[ShaderBinaryRef(AFTERMATH_HASH, "http://objectstore/dxil?sig=x", group)], pdbs=[(DEBUG_NAME, "http://objectstore/pdb?sig=x")], ) @@ -129,13 +135,37 @@ def test_client_second_round_sends_shader_arrays(teapot_session: mock.Mock) -> N assert body["event_id"] == "abc" assert body["dump"] == {"storage_url": "http://objectstore/dump-obj-id?sig=abc"} assert body["shader_binaries"] == [ - {"code_id": AFTERMATH_HASH, "storage_url": "http://objectstore/dxil?sig=x"} + { + "code_id": AFTERMATH_HASH, + "storage_url": "http://objectstore/dxil?sig=x", + "group": SHADER_GROUP, + } ] assert body["shader_pdbs"] == [ {"debug_name": DEBUG_NAME, "storage_url": "http://objectstore/pdb?sig=x"} ] +@pytest.mark.parametrize("group", (None, "")) +def test_client_second_round_omits_group_when_absent( + teapot_session: mock.Mock, group: str | None +) -> None: + resolved = _resolved(group=group) + with mock.patch("sentry.lang.native.teapot.requests.post") as mock_post: + mock_post.return_value = _FakeResponse(200, _completed()) + TeapotClient(_FakeProject(), "abc").symbolicate( + _FakeAttachment(), + [], + shader_binaries=resolved.binaries, + shader_pdbs=resolved.pdbs, + ) + + body = orjson.loads(mock_post.call_args.kwargs["data"]) + assert body["shader_binaries"] == [ + {"code_id": AFTERMATH_HASH, "storage_url": "http://objectstore/dxil?sig=x"} + ] + + def test_client_first_round_omits_shader_arrays(teapot_session: mock.Mock) -> None: with mock.patch("sentry.lang.native.teapot.requests.post") as mock_post: mock_post.return_value = _FakeResponse(200, _completed()) @@ -168,9 +198,7 @@ def test_client_inline_fallback_uploads_uncached_attachment(teapot_session: mock def test_client_inline_fallback_off_by_default(teapot_session: mock.Mock) -> None: with ( - mock.patch.object( - settings, "SENTRY_TEAPOT_INLINE_ATTACHMENT_FALLBACK", False, create=True - ), + mock.patch.object(settings, "SENTRY_TEAPOT_INLINE_ATTACHMENT_FALLBACK", False, create=True), mock.patch("sentry.lang.native.teapot.requests.post") as mock_post, ): with pytest.raises(TeapotRequestError): @@ -218,9 +246,13 @@ def _create_dif( dif_type: str, code_id: str | None = None, storage_path: str | None = "difs/shader", + shader_group: str | None = None, ) -> ProjectDebugFile: content_type = "application/x-dxil" if dif_type == "dxil" else "application/x-dxil-pdb" features = ["symtab"] if dif_type == "dxil" else ["debug"] + data: dict[str, Any] = {"type": dif_type, "features": features} + if shader_group is not None: + data["shader_group"] = shader_group return ProjectDebugFile.objects.create( project_id=project.id, debug_id=DEBUG_ID, @@ -228,7 +260,7 @@ def _create_dif( object_name="shader", cpu_name="any", checksum="0" * 40, - data={"type": dif_type, "features": features}, + data=data, storage_path=storage_path, content_type=content_type, file_size=64, @@ -252,13 +284,21 @@ def dif_presign() -> Iterator[mock.Mock]: def test_resolve_shader_difs_presigns_objectstore_rows( default_project: Project, dif_presign: mock.Mock ) -> None: - _create_dif(default_project, "dxil", code_id=AFTERMATH_HASH, storage_path="difs/dxil") + _create_dif( + default_project, + "dxil", + code_id=AFTERMATH_HASH, + storage_path="difs/dxil", + shader_group=SHADER_GROUP, + ) _create_dif(default_project, "dxilpdb", storage_path="difs/pdb") resolved = resolve_shader_difs(default_project, _partial()) assert bool(resolved) is True - assert resolved.binaries == [(AFTERMATH_HASH, "http://objectstore/difs/dxil?sig=x")] + assert resolved.binaries == [ + ShaderBinaryRef(AFTERMATH_HASH, "http://objectstore/difs/dxil?sig=x", SHADER_GROUP) + ] assert resolved.pdbs == [(DEBUG_NAME, "http://objectstore/difs/pdb?sig=x")] # Token validity is the longer DIF one, not the attachment default. assert dif_presign.call_args.kwargs["token_validity"] is not None @@ -273,7 +313,10 @@ def test_resolve_shader_difs_matches_code_id_case_insensitively( resolved = resolve_shader_difs(default_project, response) - assert resolved.binaries == [(AFTERMATH_HASH, "http://objectstore/difs/shader?sig=x")] + assert resolved.binaries == [ + ShaderBinaryRef(AFTERMATH_HASH, "http://objectstore/difs/shader?sig=x", None) + ] + assert resolved.binaries[0].group is None assert resolved.pdbs == [] diff --git a/tests/sentry/models/test_debugfile_shader.py b/tests/sentry/models/test_debugfile_shader.py index 9649fd984c23..51768f0e5302 100644 --- a/tests/sentry/models/test_debugfile_shader.py +++ b/tests/sentry/models/test_debugfile_shader.py @@ -1,20 +1,27 @@ from __future__ import annotations +import io from pathlib import Path +from unittest import mock import pytest from sentry.constants import KNOWN_DIF_FORMATS from sentry.models.debugfile import ( DIF_MIMETYPES, + SHADER_GROUP_MAX_LENGTH, BadDif, DifKind, + ProjectDebugFile, _dif_file_extension, _get_dif_object_name, + create_dif_from_fileobj, detect_dif_from_path, detect_single_dif_from_path, determine_dif_kind, ) +from sentry.models.project import Project +from sentry.testutils.pytest.fixtures import django_db_all DXC_HASH = "731322bf682bdc554eefdd89098b7383" DEBUG_ID = "731322bf-682b-dc55-4eef-dd89098b7383" @@ -117,3 +124,106 @@ def test_detect_dxil_rejects_invalid_code_id(tmp_path: Path) -> None: with pytest.raises(BadDif, match="Invalid code_id"): detect_dif_from_path(path, name="shader.dxil", debug_id=DXC_HASH, code_id="abc") + + +def test_detect_dxil_stores_shader_group(tmp_path: Path) -> None: + path = _write(tmp_path, "shader.dxil", DXIL_BYTES) + + meta = detect_single_dif_from_path( + path, name="shader.dxil", debug_id=DXC_HASH, shader_group=" GlobalShaders " + ) + + assert meta.data == {"type": "dxil", "features": ["symtab"], "shader_group": "GlobalShaders"} + + +@pytest.mark.parametrize("shader_group", (None, "", " ")) +def test_detect_dxil_drops_empty_shader_group(tmp_path: Path, shader_group: str | None) -> None: + path = _write(tmp_path, "shader.dxil", DXIL_BYTES) + + meta = detect_single_dif_from_path( + path, name="shader.dxil", debug_id=DXC_HASH, shader_group=shader_group + ) + + assert meta.data == {"type": "dxil", "features": ["symtab"]} + + +def test_detect_dxil_rejects_overlong_shader_group(tmp_path: Path) -> None: + path = _write(tmp_path, "shader.dxil", DXIL_BYTES) + + with pytest.raises(BadDif, match="Invalid shader_group"): + detect_dif_from_path( + path, + name="shader.dxil", + debug_id=DXC_HASH, + shader_group="g" * (SHADER_GROUP_MAX_LENGTH + 1), + ) + + +def test_detect_dxil_pdb_ignores_shader_group(tmp_path: Path) -> None: + path = _write(tmp_path, f"{DXC_HASH}.pdb", DXIL_PDB_BYTES) + + meta = detect_single_dif_from_path( + path, name=f"{DXC_HASH}.pdb", debug_id=DXC_HASH, shader_group="GlobalShaders" + ) + + assert meta.data == {"type": "dxilpdb", "features": ["debug"]} + + +def _create_shader_dif( + project: Project, tmp_path: Path, name: str, data: bytes, shader_group: str | None +) -> tuple[ProjectDebugFile, bool]: + path = _write(tmp_path, name, data) + meta = detect_single_dif_from_path( + path, name=name, debug_id=DXC_HASH, code_id=AFTERMATH_HASH, shader_group=shader_group + ) + session = mock.Mock() + session.put.return_value = f"difs/{name}" + with mock.patch("sentry.models.debugfile.get_session", return_value=session): + return create_dif_from_fileobj(project, meta, io.BytesIO(data)) + + +@django_db_all +def test_create_dxil_stores_shader_group(default_project: Project, tmp_path: Path) -> None: + dif, created = _create_shader_dif(default_project, tmp_path, "a.dxil", DXIL_BYTES, "Group A") + + assert created is True + dif.refresh_from_db() + assert dif.file_type == "dxil" + assert dif.data["shader_group"] == "Group A" + + +@django_db_all +def test_reupload_backfills_missing_shader_group(default_project: Project, tmp_path: Path) -> None: + first, created = _create_shader_dif(default_project, tmp_path, "a.dxil", DXIL_BYTES, None) + assert created is True + assert "shader_group" not in first.data + + second, created = _create_shader_dif(default_project, tmp_path, "a.dxil", DXIL_BYTES, "Group A") + + assert created is False + assert second.id == first.id + first.refresh_from_db() + assert first.data == {"type": "dxil", "features": ["symtab"], "shader_group": "Group A"} + assert ProjectDebugFile.objects.filter(project_id=default_project.id).count() == 1 + + +@django_db_all +def test_reupload_keeps_existing_shader_group(default_project: Project, tmp_path: Path) -> None: + first, _ = _create_shader_dif(default_project, tmp_path, "a.dxil", DXIL_BYTES, "Group A") + + second, created = _create_shader_dif(default_project, tmp_path, "a.dxil", DXIL_BYTES, "Group B") + + assert created is False + assert second.id == first.id + first.refresh_from_db() + assert first.data["shader_group"] == "Group A" + + +@django_db_all +def test_reupload_without_shader_group_is_noop(default_project: Project, tmp_path: Path) -> None: + first, _ = _create_shader_dif(default_project, tmp_path, "a.dxil", DXIL_BYTES, "Group A") + + _create_shader_dif(default_project, tmp_path, "a.dxil", DXIL_BYTES, None) + + first.refresh_from_db() + assert first.data["shader_group"] == "Group A" From 5d2a39c6e0990a34eeedfcfc709c03a6e7551bb6 Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Mon, 2 Nov 2026 17:36:41 +0200 Subject: [PATCH 05/26] Simplify shader debug file resolution Resolve shader DIFs with one presign helper and emit the teapot request entries directly, instead of a dataclass and four lookup helpers. Inline the dxil code_id and shader_group validation like the other DIF kinds and reduce the shader group backfill to its condition. --- src/sentry/lang/native/shader_difs.py | 143 +++++++----------- src/sentry/lang/native/teapot.py | 48 ++---- src/sentry/models/debugfile.py | 62 +++----- src/sentry/tasks/gpu_crash.py | 5 +- .../lang/native/test_teapot_shader_difs.py | 64 +++----- 5 files changed, 110 insertions(+), 212 deletions(-) diff --git a/src/sentry/lang/native/shader_difs.py b/src/sentry/lang/native/shader_difs.py index c4caadf03a03..b86560454d8b 100644 --- a/src/sentry/lang/native/shader_difs.py +++ b/src/sentry/lang/native/shader_difs.py @@ -1,10 +1,8 @@ """Resolve the shader debug files teapot reports missing to presigned URLs. -teapot's ``missing_difs`` names two shader kinds: ``dxil`` (the DXC shader -binary, keyed by the 16-hex Aftermath shader hash, stored as ``code_id``) and -``dxil_pdb`` (the DXC shader PDB, keyed by its DebugName, whose stem is the DXC -hash and therefore the ``debug_id``). Both are uploaded as regular project debug -files; only objectstore-backed rows can be presigned. +``missing_difs`` names two shader kinds: ``dxil`` (the DXC binary, keyed by the +16-hex Aftermath shader hash stored as ``code_id``) and ``dxil_pdb`` (the DXC PDB, +keyed by its DebugName, whose stem is the DXC hash and therefore the ``debug_id``). """ from __future__ import annotations @@ -21,7 +19,6 @@ from symbolic.exceptions import SymbolicError from sentry.debug_files.debug_files import maybe_renew_debug_files -from sentry.lang.native.teapot import ShaderBinaryRef from sentry.models.debugfile import ProjectDebugFile from sentry.models.project import Project from sentry.objectstore import get_internal_download_url @@ -29,25 +26,17 @@ logger = logging.getLogger(__name__) -MISSING_DIF_KIND_DXIL = "dxil" -MISSING_DIF_KIND_DXIL_PDB = "dxil_pdb" - -DIF_TYPE_DXIL = "dxil" -DIF_TYPE_DXIL_PDB = "dxilpdb" - SHADER_DIF_URL_VALIDITY = timedelta(minutes=15) -_CODE_ID_RE = re.compile(r"^[0-9a-fA-F]{16}$") _PDB_SUFFIX_RE = re.compile(r"\.pdb$", re.IGNORECASE) -ShaderFileRef = tuple[str, str] -"""``(key, storage_url)``: the key teapot asked for and a presigned GET URL.""" - @dataclass class ShaderDifs: - binaries: list[ShaderBinaryRef] = field(default_factory=list) - pdbs: list[ShaderFileRef] = field(default_factory=list) + """Entries for teapot's ``shader_binaries`` and ``shader_pdbs`` request arrays.""" + + binaries: list[dict[str, str]] = field(default_factory=list) + pdbs: list[dict[str, str]] = field(default_factory=list) def __bool__(self) -> bool: return bool(self.binaries or self.pdbs) @@ -63,10 +52,9 @@ def missing_shader_difs(response: Mapping[str, Any]) -> tuple[list[str], list[st debug_id = entry.get("debug_id") if not isinstance(debug_id, str) or not debug_id: continue - kind = entry.get("kind") - if kind == MISSING_DIF_KIND_DXIL: + if entry.get("kind") == "dxil": code_ids.append(debug_id) - elif kind == MISSING_DIF_KIND_DXIL_PDB: + elif entry.get("kind") == "dxil_pdb": debug_names.append(debug_id) return code_ids, debug_names @@ -79,88 +67,73 @@ def debug_id_from_debug_name(debug_name: str) -> str | None: return None -def find_dxil_by_code_id(project: Project, code_id: str) -> ProjectDebugFile | None: - if not _CODE_ID_RE.match(code_id): - return None - difs = ProjectDebugFile.objects.filter(project_id=project.id, code_id=code_id.lower()).order_by( - "-id" - ) - return next((dif for dif in difs if dif.file_type == DIF_TYPE_DXIL), None) - - -def find_dxil_pdb_by_debug_name(project: Project, debug_name: str) -> ProjectDebugFile | None: - debug_id = debug_id_from_debug_name(debug_name) - if debug_id is None: - return None - difs = ProjectDebugFile.objects.filter(project_id=project.id, debug_id=debug_id).order_by("-id") - return next((dif for dif in difs if dif.file_type == DIF_TYPE_DXIL_PDB), None) - - -def find_sibling_pdb(dxil: ProjectDebugFile) -> tuple[str, ProjectDebugFile] | None: - """The PDB uploaded next to a DXIL binary, keyed by the DebugName the decoder will ask for. - - The decoder requests a shader's PDB only after it has received the binary, so - sending the sibling PDB along with the binary saves one extra round trip. - """ - difs = ProjectDebugFile.objects.filter( - project_id=dxil.project_id, debug_id=dxil.debug_id - ).order_by("-id") - pdb = next((dif for dif in difs if dif.file_type == DIF_TYPE_DXIL_PDB), None) - if pdb is None: +def _presigned_dif( + project_id: int, dif_type: str, **filters: str +) -> tuple[ProjectDebugFile, str] | None: + """The newest debug file of ``dif_type`` matching ``filters`` with a presigned GET + URL, or None if there is none or it is not in objectstore.""" + difs = ProjectDebugFile.objects.filter(project_id=project_id, **filters).order_by("-id") + dif = next((dif for dif in difs if dif.file_type == dif_type), None) + if dif is None: return None - name = os.path.basename((pdb.object_name or "").replace("\\", "/")) - if not _PDB_SUFFIX_RE.search(name): - name = f"{dxil.debug_id.replace('-', '')[:32]}.pdb" - return name, pdb - - -def shader_group_of(dif: ProjectDebugFile) -> str | None: - """The shader group recorded at upload time (``data["shader_group"]``), or None.""" - group = (dif.data or {}).get("shader_group") - return group if isinstance(group, str) and group else None - - -def presign_dif(dif: ProjectDebugFile) -> str | None: - """Presigned GET URL for an objectstore-backed debug file, or None.""" if not dif.storage_path: logger.info( "gpu_crash.shader_dif.not_in_objectstore", - extra={"dif_id": dif.id, "debug_id": dif.debug_id, "type": dif.file_type}, + extra={"dif_id": dif.id, "debug_id": dif.debug_id, "type": dif_type}, ) return None - return get_internal_download_url( + url = get_internal_download_url( dif.get_objectstore_session(), dif.storage_path, token_validity=SHADER_DIF_URL_VALIDITY ) + return dif, url + + +def _pdb_debug_name(pdb: ProjectDebugFile) -> str: + name = os.path.basename((pdb.object_name or "").replace("\\", "/")) + if _PDB_SUFFIX_RE.search(name): + return name + return f"{pdb.debug_id.replace('-', '')[:32]}.pdb" def resolve_shader_difs(project: Project, response: Mapping[str, Any]) -> ShaderDifs: - """Look up every shader DIF teapot reported missing and presign the ones we hold.""" + """Look up every shader DIF teapot reported missing and presign the ones we hold. + + The PDB uploaded next to a resolved binary is sent along, since teapot only asks + for a shader's PDB once it has the binary. + """ code_ids, debug_names = missing_shader_difs(response) resolved = ShaderDifs() found: list[ProjectDebugFile] = [] + requested_pdbs = {name.lower() for name in debug_names} for code_id in dict.fromkeys(code_ids): - dif = find_dxil_by_code_id(project, code_id) - url = presign_dif(dif) if dif is not None else None - if dif is not None and url is not None: - found.append(dif) - resolved.binaries.append(ShaderBinaryRef(code_id.lower(), url, shader_group_of(dif))) - sibling = find_sibling_pdb(dif) - if sibling is not None: - debug_name, pdb = sibling - pdb_url = presign_dif(pdb) - if pdb_url is not None and debug_name.lower() not in { - n.lower() for n in debug_names - }: - found.append(pdb) - resolved.pdbs.append((debug_name, pdb_url)) + dxil_match = _presigned_dif(project.id, "dxil", code_id=code_id.lower()) + if dxil_match is None: + continue + dxil, url = dxil_match + found.append(dxil) + entry = {"code_id": code_id.lower(), "storage_url": url} + if group := (dxil.data or {}).get("shader_group"): + entry["group"] = group + resolved.binaries.append(entry) + + pdb_match = _presigned_dif(project.id, "dxilpdb", debug_id=dxil.debug_id) + if pdb_match is None: + continue + pdb, pdb_url = pdb_match + debug_name = _pdb_debug_name(pdb) + if debug_name.lower() not in requested_pdbs: + found.append(pdb) + resolved.pdbs.append({"debug_name": debug_name, "storage_url": pdb_url}) for debug_name in dict.fromkeys(debug_names): - dif = find_dxil_pdb_by_debug_name(project, debug_name) - url = presign_dif(dif) if dif is not None else None - if dif is not None and url is not None: - found.append(dif) - resolved.pdbs.append((debug_name, url)) + debug_id = debug_id_from_debug_name(debug_name) + pdb_match = _presigned_dif(project.id, "dxilpdb", debug_id=debug_id) if debug_id else None + if pdb_match is None: + continue + pdb, url = pdb_match + found.append(pdb) + resolved.pdbs.append({"debug_name": debug_name, "storage_url": url}) missing = len(code_ids) + len(debug_names) if missing: diff --git a/src/sentry/lang/native/teapot.py b/src/sentry/lang/native/teapot.py index 194398426619..c75a9a00edc0 100644 --- a/src/sentry/lang/native/teapot.py +++ b/src/sentry/lang/native/teapot.py @@ -14,7 +14,6 @@ import logging import re from collections.abc import Sequence -from dataclasses import dataclass from datetime import timedelta from typing import Any, Protocol @@ -58,31 +57,6 @@ class TeapotAttachment(Protocol): stored_id: str | None -ShaderFileRef = tuple[str, str] -"""``(key, storage_url)`` for a shader DIF teapot asked for in ``missing_difs``: -the dxil ``code_id`` or the dxil_pdb ``debug_name``, plus a presigned GET URL.""" - - -@dataclass(frozen=True) -class ShaderBinaryRef: - """A dxil binary teapot asked for: its Aftermath ``code_id``, a presigned GET - URL, and the shader group recorded at upload time, if any.""" - - code_id: str - storage_url: str - group: str | None = None - - def as_request_entry(self) -> dict[str, str]: - entry = {"code_id": self.code_id, "storage_url": self.storage_url} - if self.group: - entry["group"] = self.group - return entry - - -# Suffix on the idempotency/request ids of the second (shader DIF) round, so -# teapot's idempotency cache does not replay the first round's response. -SHADER_DIF_ROUND_SUFFIX = "-2" - # Dev-only inline attachment uploads live just long enough for the decode. INLINE_ATTACHMENT_TTL = timedelta(days=1) @@ -171,8 +145,8 @@ def symbolicate( self, dump: TeapotAttachment, shader_debug_info: Sequence[TeapotAttachment] | None = None, - shader_binaries: Sequence[ShaderBinaryRef] | None = None, - shader_pdbs: Sequence[ShaderFileRef] | None = None, + shader_binaries: Sequence[dict[str, str]] | None = None, + shader_pdbs: Sequence[dict[str, str]] | None = None, ) -> dict[str, Any]: shader_debug_info = shader_debug_info or [] url = f"{self.base_url}/symbolicate" @@ -193,12 +167,10 @@ def symbolicate( } request_id = self.event_id if shader_binaries or shader_pdbs: - body["shader_binaries"] = [ref.as_request_entry() for ref in shader_binaries or []] - body["shader_pdbs"] = [ - {"debug_name": debug_name, "storage_url": storage_url} - for debug_name, storage_url in shader_pdbs or [] - ] - request_id = f"{self.event_id}{SHADER_DIF_ROUND_SUFFIX}" + # Second round: a distinct id, or teapot's idempotency cache replays the first response. + body["shader_binaries"] = list(shader_binaries or []) + body["shader_pdbs"] = list(shader_pdbs or []) + request_id = f"{self.event_id}-2" data = orjson.dumps(body) headers = { "X-Teapot-Version": "1", @@ -290,8 +262,8 @@ def submit_to_teapot( event_id: str, dump: TeapotAttachment, shader_debug_info: Sequence[TeapotAttachment] | None = None, - shader_binaries: Sequence[ShaderBinaryRef] | None = None, - shader_pdbs: Sequence[ShaderFileRef] | None = None, + shader_binaries: Sequence[dict[str, str]] | None = None, + shader_pdbs: Sequence[dict[str, str]] | None = None, ) -> dict[str, Any] | None: """Best-effort teapot invocation. @@ -300,8 +272,8 @@ def submit_to_teapot( trip the circuit breaker. Raises ``TeapotUnavailable`` on a genuine outage (network, timeout, 5xx) so the caller can. - ``shader_binaries`` / ``shader_pdbs`` turn this into the second round that - hands teapot presigned URLs for the shader DIFs it reported missing. + ``shader_binaries`` / ``shader_pdbs`` make this the second round, with presigned + URLs for the shader DIFs teapot reported missing. """ try: diff --git a/src/sentry/models/debugfile.py b/src/sentry/models/debugfile.py index 56e6de2bebcf..b7fdca4b74aa 100644 --- a/src/sentry/models/debugfile.py +++ b/src/sentry/models/debugfile.py @@ -69,15 +69,14 @@ _proguard_file_re = re.compile(r"/proguard/(?:mapping-)?(.*?)\.txt$") -# DXIL shader binaries (DXC output) carry the DXBC container magic; DXC shader -# PDBs are MSF files whose stream directory names an `ILDN` (debug name) part. +# DXC shader binaries carry the DXBC container magic; DXC shader PDBs are MSF +# files whose stream directory names an `ILDN` (debug name) part. _DXIL_MAGIC = b"DXBC" _MSF_MAGIC = b"Microsoft C/C++ MSF 7.00" _DXIL_PDB_PART = b"ILDN" _DXIL_PDB_SCAN_BYTES = 64 * 1024 _SHADER_CODE_ID_RE = re.compile(r"^[0-9a-f]{16}$") -# Upper bound on the free-text shader group recorded with a dxil debug file. SHADER_GROUP_MAX_LENGTH = 512 @@ -565,21 +564,10 @@ def find_existing_dif(project: Project, meta: DifMeta, checksum: str) -> Project def _backfill_shader_group(dif: ProjectDebugFile, meta: DifMeta) -> None: - """Record the shader group on an existing ``dxil`` row that was uploaded without one. - - Re-uploading the same shader binary is a no-op for the file, but a later cook may - know the group the shader belongs to. A group already on the row is kept as-is. - """ - if meta.file_format != "dxil": - return + """A re-upload of the same dxil binary may carry the shader group an earlier cook did not.""" shader_group = (meta.data or {}).get("shader_group") - if not shader_group: - return - data = dict(dif.data or {}) - if data.get("type") != "dxil" or data.get("shader_group"): - return - data["shader_group"] = shader_group - dif.update(data=data) + if shader_group and not (dif.data or {}).get("shader_group"): + dif.update(data={**(dif.data or {}), "shader_group": shader_group}) def _get_fileobj_checksum_and_size(fileobj: IO[bytes]) -> tuple[str, int]: @@ -966,26 +954,6 @@ def _shader_debug_id(debug_id: str | None, kind: str) -> str: raise BadDif(f"Invalid debug_id for {kind}: {e}") -def _shader_code_id(code_id: str | None) -> str | None: - if code_id is None: - return None - code_id = code_id.lower() - if not _SHADER_CODE_ID_RE.match(code_id): - raise BadDif("Invalid code_id for dxil (expected 16 hex characters)") - return code_id - - -def _shader_group(shader_group: str | None) -> str | None: - if shader_group is None: - return None - shader_group = shader_group.strip() - if not shader_group: - return None - if len(shader_group) > SHADER_GROUP_MAX_LENGTH: - raise BadDif(f"Invalid shader_group for dxil (max {SHADER_GROUP_MAX_LENGTH} characters)") - return shader_group - - def detect_dif_from_path( path: str, name: str | None = None, @@ -1108,20 +1076,28 @@ def detect_dif_from_path( ) ] elif dif_kind == DifKind.Dxil: + if code_id is not None: + code_id = code_id.lower() + if not _SHADER_CODE_ID_RE.match(code_id): + raise BadDif("Invalid code_id for dxil (expected 16 hex characters)") + shader_group = (shader_group or "").strip() + if len(shader_group) > SHADER_GROUP_MAX_LENGTH: + raise BadDif( + f"Invalid shader_group for dxil (max {SHADER_GROUP_MAX_LENGTH} characters)" + ) logger.debug("File loaded as dxil: %s", path) - data = {"type": "dxil", "features": ["symtab"]} - group = _shader_group(shader_group) - if group is not None: - data["shader_group"] = group + dxil_data: dict[str, Any] = {"type": "dxil", "features": ["symtab"]} + if shader_group: + dxil_data["shader_group"] = shader_group return [ DifMeta( file_format="dxil", arch="any", debug_id=_shader_debug_id(debug_id, "dxil"), - code_id=_shader_code_id(code_id), + code_id=code_id, name=name, path=path, - data=data, + data=dxil_data, ) ] elif dif_kind == DifKind.DxilPdb: diff --git a/src/sentry/tasks/gpu_crash.py b/src/sentry/tasks/gpu_crash.py index 40fff03c8abe..f4f2b87b85e7 100644 --- a/src/sentry/tasks/gpu_crash.py +++ b/src/sentry/tasks/gpu_crash.py @@ -191,9 +191,8 @@ def _maybe_resolve_shader_difs( shaders: list[Any], response: dict[str, Any], ) -> dict[str, Any]: - """One extra teapot round with presigned URLs for the shader DIFs (dxil, - dxil_pdb) it reported missing. Best-effort: any failure keeps the first - response, and the breaker is left to the first round.""" + """Second teapot round with presigned URLs for the shader DIFs it reported missing. + Best-effort: any failure keeps the first response and does not touch the breaker.""" if response.get("status") != "partial": return response diff --git a/tests/sentry/lang/native/test_teapot_shader_difs.py b/tests/sentry/lang/native/test_teapot_shader_difs.py index 3c71eeaaaca2..ce16db9fd112 100644 --- a/tests/sentry/lang/native/test_teapot_shader_difs.py +++ b/tests/sentry/lang/native/test_teapot_shader_difs.py @@ -17,12 +17,7 @@ missing_shader_difs, resolve_shader_difs, ) -from sentry.lang.native.teapot import ( - ShaderBinaryRef, - TeapotClient, - TeapotRequestError, - TeapotUnavailable, -) +from sentry.lang.native.teapot import TeapotClient, TeapotRequestError, TeapotUnavailable from sentry.models.debugfile import ProjectDebugFile from sentry.models.project import Project from sentry.tasks.gpu_crash import _try_symbolicate @@ -91,10 +86,16 @@ def _completed() -> dict[str, Any]: return {"status": "completed", "fault_category": "shader_hang", "missing_difs": []} -def _resolved(group: str | None = SHADER_GROUP) -> ShaderDifs: +def _resolved() -> ShaderDifs: return ShaderDifs( - binaries=[ShaderBinaryRef(AFTERMATH_HASH, "http://objectstore/dxil?sig=x", group)], - pdbs=[(DEBUG_NAME, "http://objectstore/pdb?sig=x")], + binaries=[ + { + "code_id": AFTERMATH_HASH, + "storage_url": "http://objectstore/dxil?sig=x", + "group": SHADER_GROUP, + } + ], + pdbs=[{"debug_name": DEBUG_NAME, "storage_url": "http://objectstore/pdb?sig=x"}], ) @@ -134,36 +135,8 @@ def test_client_second_round_sends_shader_arrays(teapot_session: mock.Mock) -> N body = orjson.loads(kwargs["data"]) assert body["event_id"] == "abc" assert body["dump"] == {"storage_url": "http://objectstore/dump-obj-id?sig=abc"} - assert body["shader_binaries"] == [ - { - "code_id": AFTERMATH_HASH, - "storage_url": "http://objectstore/dxil?sig=x", - "group": SHADER_GROUP, - } - ] - assert body["shader_pdbs"] == [ - {"debug_name": DEBUG_NAME, "storage_url": "http://objectstore/pdb?sig=x"} - ] - - -@pytest.mark.parametrize("group", (None, "")) -def test_client_second_round_omits_group_when_absent( - teapot_session: mock.Mock, group: str | None -) -> None: - resolved = _resolved(group=group) - with mock.patch("sentry.lang.native.teapot.requests.post") as mock_post: - mock_post.return_value = _FakeResponse(200, _completed()) - TeapotClient(_FakeProject(), "abc").symbolicate( - _FakeAttachment(), - [], - shader_binaries=resolved.binaries, - shader_pdbs=resolved.pdbs, - ) - - body = orjson.loads(mock_post.call_args.kwargs["data"]) - assert body["shader_binaries"] == [ - {"code_id": AFTERMATH_HASH, "storage_url": "http://objectstore/dxil?sig=x"} - ] + assert body["shader_binaries"] == resolved.binaries + assert body["shader_pdbs"] == resolved.pdbs def test_client_first_round_omits_shader_arrays(teapot_session: mock.Mock) -> None: @@ -297,9 +270,15 @@ def test_resolve_shader_difs_presigns_objectstore_rows( assert bool(resolved) is True assert resolved.binaries == [ - ShaderBinaryRef(AFTERMATH_HASH, "http://objectstore/difs/dxil?sig=x", SHADER_GROUP) + { + "code_id": AFTERMATH_HASH, + "storage_url": "http://objectstore/difs/dxil?sig=x", + "group": SHADER_GROUP, + } + ] + assert resolved.pdbs == [ + {"debug_name": DEBUG_NAME, "storage_url": "http://objectstore/difs/pdb?sig=x"} ] - assert resolved.pdbs == [(DEBUG_NAME, "http://objectstore/difs/pdb?sig=x")] # Token validity is the longer DIF one, not the attachment default. assert dif_presign.call_args.kwargs["token_validity"] is not None @@ -314,9 +293,8 @@ def test_resolve_shader_difs_matches_code_id_case_insensitively( resolved = resolve_shader_difs(default_project, response) assert resolved.binaries == [ - ShaderBinaryRef(AFTERMATH_HASH, "http://objectstore/difs/shader?sig=x", None) + {"code_id": AFTERMATH_HASH, "storage_url": "http://objectstore/difs/shader?sig=x"} ] - assert resolved.binaries[0].group is None assert resolved.pdbs == [] From 830bf15b0d05057355d8d43ae7467dbfde7d3577 Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Mon, 2 Nov 2026 18:59:56 +0200 Subject: [PATCH 06/26] Show the shader group of a GPU crash where the UI renders it The native frame renderer shows package, function and filename only, so the group teapot puts into frame.module never reached the page. Write it to the gpu.shader_group, gpu.material and gpu.shader_class tags, the gpu_crash context, the culprit and the frame's filename when UE's sentinel or nothing fills that slot. --- src/sentry/lang/native/gpu.py | 42 ++++++++ .../lang/native/test_gpu_shader_group.py | 102 ++++++++++++++++++ 2 files changed, 144 insertions(+) create mode 100644 tests/sentry/lang/native/test_gpu_shader_group.py diff --git a/src/sentry/lang/native/gpu.py b/src/sentry/lang/native/gpu.py index 228b571b2eb1..35fea148ede7 100644 --- a/src/sentry/lang/native/gpu.py +++ b/src/sentry/lang/native/gpu.py @@ -125,6 +125,13 @@ def _tag(key: str, value: Any) -> None: gpu_tags[key] = str(value) _tag("gpu.shader_hash", primary_shader.get("shader_hash")) + shader_group = _shader_group(response) + material, shader_class = _shader_group_parts(shader_group) + _tag("gpu.shader_group", shader_group) + _tag("gpu.material", material) + _tag("gpu.shader_class", shader_class) + if shader_group: + data["culprit"] = shader_group _tag("gpu.shader_type", primary_shader.get("shader_type")) _tag("gpu.device", gpu_state.get("device_name")) _tag("gpu.generation", _generation_name(gpu_state)) @@ -154,6 +161,27 @@ def _primary_shader(response: Mapping[str, Any]) -> dict[str, Any]: return active[0] if active else {} +def _shader_group(response: Mapping[str, Any]) -> str | None: + """The UE shader group recorded with the faulting frame's dxil debug file, if any.""" + frames = response.get("frames") or [] + first = frames[0] if frames and isinstance(frames[0], dict) else {} + group = (first.get("data") or {}).get("shader_group") + return group if isinstance(group, str) and group else None + + +def _shader_group_parts(group: str | None) -> tuple[str | None, str | None]: + """``(material, shader_class)`` from ``Material/Quality/VertexFactory/ShaderClass/Permutation`` + or ``Global/ShaderClass/Permutation``.""" + if not group: + return None, None + parts = [part for part in group.split("/") if part] + if len(parts) < 2: + return None, None + if parts[0] == "Global": + return None, parts[1] + return parts[0], parts[-2] if len(parts) >= 3 else None + + def _raw_sections(response: Mapping[str, Any]) -> dict[str, Any]: raw = (response.get("shader_context") or {}).get("raw") out: dict[str, Any] = {} @@ -327,6 +355,8 @@ def _build_gpu_crash_context(response: Mapping[str, Any]) -> dict[str, Any]: sections = _raw_sections(response) faulted = _faulted_warps(sections) first_faulted = faulted[0] if faulted else {} + shader_group = _shader_group(response) + material, shader_class = _shader_group_parts(shader_group) flat: dict[str, Any] = { "type": "gpu_crash", @@ -351,6 +381,9 @@ def _build_gpu_crash_context(response: Mapping[str, Any]) -> dict[str, Any]: "resource_destroyed": resource.get("was_destroyed") or None, "shader_hash": primary_shader.get("shader_hash"), "shader_type": primary_shader.get("shader_type"), + "shader_group": shader_group, + "material": material, + "shader_class": shader_class, "shader_size": _shader_size(sections), "shader_debug_info_uid": primary_shader.get("shader_debug_info_uid"), "status": response.get("status"), @@ -445,6 +478,9 @@ def _markers_to_breadcrumbs(markers: list[Any]) -> list[dict[str, Any]]: return out +UE_FILENAME_SENTINEL = "__UE_FILENAME_SENTINEL" + + def _normalize_gpu_frames(teapot_frames: list[Any]) -> list[dict[str, Any]]: normalized: list[dict[str, Any]] = [] for raw in teapot_frames: @@ -477,6 +513,12 @@ def _normalize_gpu_frames(teapot_frames: list[Any]) -> list[dict[str, Any]]: ) if not frame.get("module") and frame.get("package"): frame["module"] = frame["package"] + group = raw_data.get("shader_group") + filename = frame.get("filename") + if isinstance(group, str) and group and filename in (None, "", UE_FILENAME_SENTINEL): + if filename: + frame.setdefault("abs_path", filename) + frame["filename"] = group frame.setdefault("data", {}) frame["data"].setdefault("symbolicator_status", "symbolicated") diff --git a/tests/sentry/lang/native/test_gpu_shader_group.py b/tests/sentry/lang/native/test_gpu_shader_group.py new file mode 100644 index 000000000000..1ac7c11e3470 --- /dev/null +++ b/tests/sentry/lang/native/test_gpu_shader_group.py @@ -0,0 +1,102 @@ +from __future__ import annotations + +from typing import Any + +import pytest + +from sentry.lang.native.gpu import apply_gpu_crash_symbolication + +MATERIAL_GROUP = "M_GpuHang_992b2d801a2e9ddf/Default/FLocalVertexFactory/FLumenCardPS/0" +GLOBAL_GROUP = "Global/FHeavyComputeLoop/0" + + +def _response(frame: dict[str, Any]) -> dict[str, Any]: + return { + "status": "completed", + "handler": "aftermath", + "fault_category": "shader_hang", + "title": "GPU hang in Main", + "fingerprint": ["gpu", "shader_hang", "49478268ce7710e2"], + "markers": [], + "fault": {"type": "Timeout"}, + "gpu_state": {"device_name": "GA102-A"}, + "shader_context": { + "active_shaders": [{"shader_hash": "49478268ce7710e2", "shader_type": "Pixel"}] + }, + "frames": [frame], + "missing_difs": [], + } + + +def _apply(frame: dict[str, Any]) -> dict[str, Any]: + data: dict[str, Any] = {"event_id": "g" * 32, "platform": "native"} + result = apply_gpu_crash_symbolication(data, _response(frame)) + assert result is not None + return result + + +def _frame(**overrides: Any) -> dict[str, Any]: + frame = { + "function": "Main", + "module": MATERIAL_GROUP, + "data": {"shader_hash": "49478268ce7710e2", "shader_group": MATERIAL_GROUP}, + } + frame.update(overrides) + return frame + + +def test_material_group_fills_tags_culprit_and_context() -> None: + data = _apply(_frame()) + + tags = dict(data["tags"]) + assert tags["gpu.shader_group"] == MATERIAL_GROUP + assert tags["gpu.material"] == "M_GpuHang_992b2d801a2e9ddf" + assert tags["gpu.shader_class"] == "FLumenCardPS" + assert data["culprit"] == MATERIAL_GROUP + + gpu_crash = data["contexts"]["gpu_crash"] + assert gpu_crash["shader_group"] == MATERIAL_GROUP + assert gpu_crash["material"] == "M_GpuHang_992b2d801a2e9ddf" + assert gpu_crash["shader_class"] == "FLumenCardPS" + + +def test_global_group_has_no_material() -> None: + data = _apply(_frame(module=GLOBAL_GROUP, data={"shader_group": GLOBAL_GROUP})) + + tags = dict(data["tags"]) + assert tags["gpu.shader_group"] == GLOBAL_GROUP + assert tags["gpu.shader_class"] == "FHeavyComputeLoop" + assert "gpu.material" not in tags + assert "material" not in data["contexts"]["gpu_crash"] + + +@pytest.mark.parametrize( + ("filename", "expected_abs_path"), + [(None, None), ("__UE_FILENAME_SENTINEL", "__UE_FILENAME_SENTINEL")], +) +def test_group_fills_the_filename_slot(filename: str | None, expected_abs_path: str | None) -> None: + frame = _frame(lineno=3477) if filename is None else _frame(filename=filename, lineno=3477) + data = _apply(frame) + + (applied,) = data["exception"]["values"][0]["stacktrace"]["frames"] + assert applied["filename"] == MATERIAL_GROUP + assert applied.get("abs_path") == expected_abs_path + assert applied["lineno"] == 3477 + + +def test_real_filename_is_kept() -> None: + data = _apply(_frame(filename="/Game/Shaders/Foo.usf", lineno=12)) + + (applied,) = data["exception"]["values"][0]["stacktrace"]["frames"] + assert applied["filename"] == "/Game/Shaders/Foo.usf" + + +def test_no_group_changes_nothing() -> None: + data = _apply({"function": "compute_01", "module": "shader_658794f3a75293d1", "data": {}}) + + tags = dict(data["tags"]) + assert "gpu.shader_group" not in tags and "gpu.material" not in tags + assert "culprit" not in data + assert "shader_group" not in data["contexts"]["gpu_crash"] + (applied,) = data["exception"]["values"][0]["stacktrace"]["frames"] + assert "filename" not in applied From a475be0621d180efbcba2b7b08e4bf2a27396540 Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Mon, 2 Nov 2026 18:59:56 +0200 Subject: [PATCH 07/26] Remove the dev-only inline attachment fallback Relay streams every attachment to objectstore before the GPU crash task runs, so an attachment without a stored_id is an error, not a case to paper over. --- src/sentry/conf/server.py | 6 --- src/sentry/lang/native/teapot.py | 30 +------------ .../lang/native/test_teapot_shader_difs.py | 44 +------------------ 3 files changed, 3 insertions(+), 77 deletions(-) diff --git a/src/sentry/conf/server.py b/src/sentry/conf/server.py index e4528bb10356..dc32bc683c7c 100644 --- a/src/sentry/conf/server.py +++ b/src/sentry/conf/server.py @@ -3417,12 +3417,6 @@ def custom_parameter_sort(parameter: _OperationParameter) -> tuple[str, int]: # Shared secret used to sign requests to teapot SENTRY_TEAPOT_SHARED_SECRET = os.getenv("SENTRY_TEAPOT_SHARED_SECRET", "") -# Dev-only: when Relay did not stream an attachment to objectstore, upload the -# cached bytes ourselves so teapot can still fetch them by presigned URL. -SENTRY_TEAPOT_INLINE_ATTACHMENT_FALLBACK = ( - os.getenv("SENTRY_TEAPOT_INLINE_ATTACHMENT_FALLBACK", "0") == "1" -) - SENTRY_REPLAYS_SERVICE_URL = "http://localhost:8090" # This is useful for testing SSO expiry flows diff --git a/src/sentry/lang/native/teapot.py b/src/sentry/lang/native/teapot.py index c75a9a00edc0..6af783f8efff 100644 --- a/src/sentry/lang/native/teapot.py +++ b/src/sentry/lang/native/teapot.py @@ -14,14 +14,12 @@ import logging import re from collections.abc import Sequence -from datetime import timedelta from typing import Any, Protocol import orjson import requests import sentry_sdk from django.conf import settings -from objectstore_client import TimeToLive from sentry import options from sentry.objectstore import UsecaseId, get_internal_download_url, get_session @@ -57,10 +55,6 @@ class TeapotAttachment(Protocol): stored_id: str | None -# Dev-only inline attachment uploads live just long enough for the decode. -INLINE_ATTACHMENT_TTL = timedelta(days=1) - - # Fallbacks for the `teapot.timeout-seconds` / `teapot.max-attempts` options. # The timeout is tight on purpose: decode is sub-second, so a slow teapot should # fail fast (the circuit breaker then takes over) rather than hold a worker. @@ -185,31 +179,9 @@ def _storage_url(self, session: Any, att: TeapotAttachment) -> str: """Short-lived self-authenticating (presigned) GET URL for the attachment — the shared internal objectstore helper (direct link, no auth header).""" if not att.stored_id: - if not getattr(settings, "SENTRY_TEAPOT_INLINE_ATTACHMENT_FALLBACK", False): - raise TeapotRequestError(f"attachment {att.name!r} is not in objectstore") - att.stored_id = self._upload_inline(session, att) + raise TeapotRequestError(f"attachment {att.name!r} is not in objectstore") return get_internal_download_url(session, att.stored_id) - def _upload_inline(self, session: Any, att: TeapotAttachment) -> str: - """Dev fallback: Relay kept the attachment in the processing cache instead - of objectstore, so store the cached bytes ourselves and presign that key.""" - load_data = getattr(att, "load_data", None) - if load_data is None: - raise TeapotRequestError(f"attachment {att.name!r} has no cached data") - payload = load_data(self.project) - key = session.put( - payload, - content_type=getattr(att, "content_type", None) or "application/octet-stream", - filename=att.name, - expiration_policy=TimeToLive(INLINE_ATTACHMENT_TTL), - ) - metrics.incr("tasks.gpu_crash.teapot_inline_attachment_upload") - logger.info( - "teapot.inline_attachment_upload", - extra={"event_id": self.event_id, "name": att.name, "size": len(payload)}, - ) - return key - def _send(self, url: str, headers: dict[str, str], data: bytes) -> dict[str, Any]: def attempt() -> dict[str, Any]: try: diff --git a/tests/sentry/lang/native/test_teapot_shader_difs.py b/tests/sentry/lang/native/test_teapot_shader_difs.py index ce16db9fd112..1ba6f51edb9a 100644 --- a/tests/sentry/lang/native/test_teapot_shader_difs.py +++ b/tests/sentry/lang/native/test_teapot_shader_difs.py @@ -17,7 +17,7 @@ missing_shader_difs, resolve_shader_difs, ) -from sentry.lang.native.teapot import TeapotClient, TeapotRequestError, TeapotUnavailable +from sentry.lang.native.teapot import TeapotClient, TeapotUnavailable from sentry.models.debugfile import ProjectDebugFile from sentry.models.project import Project from sentry.tasks.gpu_crash import _try_symbolicate @@ -44,19 +44,9 @@ def __init__(self, id: int = 42, organization_id: int = 7) -> None: class _FakeAttachment: - def __init__( - self, - name: str = "dump.nv-gpudmp", - stored_id: str | None = "dump-obj-id", - data: bytes = b"dump", - ) -> None: + def __init__(self, name: str = "dump.nv-gpudmp", stored_id: str | None = "dump-obj-id") -> None: self.name = name self.stored_id = stored_id - self.content_type = "application/octet-stream" - self._data = data - - def load_data(self, project: Any = None) -> bytes: - return self._data class _FakeResponse: @@ -102,7 +92,6 @@ def _resolved() -> ShaderDifs: @pytest.fixture def teapot_session() -> Iterator[mock.Mock]: session = mock.Mock() - session.put.return_value = "inline-key" get_url = mock.Mock(side_effect=lambda _session, key: f"http://objectstore/{key}?sig=abc") with ( mock.patch.object(settings, "SENTRY_TEAPOT_URL", "http://teapot.test", create=True), @@ -152,35 +141,6 @@ def test_client_first_round_omits_shader_arrays(teapot_session: mock.Mock) -> No assert "shader_pdbs" not in body -def test_client_inline_fallback_uploads_uncached_attachment(teapot_session: mock.Mock) -> None: - dump = _FakeAttachment(stored_id=None, data=b"gpu-dump-bytes") - with ( - mock.patch.object(settings, "SENTRY_TEAPOT_INLINE_ATTACHMENT_FALLBACK", True, create=True), - mock.patch("sentry.lang.native.teapot.requests.post") as mock_post, - ): - mock_post.return_value = _FakeResponse(200, _completed()) - TeapotClient(_FakeProject(), "abc").symbolicate(dump, []) - - teapot_session.put.assert_called_once() - assert teapot_session.put.call_args.args[0] == b"gpu-dump-bytes" - assert teapot_session.put.call_args.kwargs["filename"] == "dump.nv-gpudmp" - assert dump.stored_id == "inline-key" - body = orjson.loads(mock_post.call_args.kwargs["data"]) - assert body["dump"] == {"storage_url": "http://objectstore/inline-key?sig=abc"} - - -def test_client_inline_fallback_off_by_default(teapot_session: mock.Mock) -> None: - with ( - mock.patch.object(settings, "SENTRY_TEAPOT_INLINE_ATTACHMENT_FALLBACK", False, create=True), - mock.patch("sentry.lang.native.teapot.requests.post") as mock_post, - ): - with pytest.raises(TeapotRequestError): - TeapotClient(_FakeProject(), "abc").symbolicate(_FakeAttachment(stored_id=None), []) - - teapot_session.put.assert_not_called() - assert mock_post.call_count == 0 - - # --------------------------------------------------------------------------- # shader_difs — resolving missing_difs against ProjectDebugFile # --------------------------------------------------------------------------- From e75c6f93e93d0ce70754e0a36a68f55426c7b033 Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Mon, 2 Nov 2026 18:59:56 +0200 Subject: [PATCH 08/26] Backfill the shader group when assemble finds the checksum The assemble endpoint answers from the existing row for a known checksum and never runs the task, so a group that a later cook supplies for an already uploaded dxil binary was dropped. --- src/sentry/api/endpoints/debug_files.py | 8 ++++++ tests/sentry/models/test_debugfile_shader.py | 27 ++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/src/sentry/api/endpoints/debug_files.py b/src/sentry/api/endpoints/debug_files.py index 5f00c41d6fe9..513e17328538 100644 --- a/src/sentry/api/endpoints/debug_files.py +++ b/src/sentry/api/endpoints/debug_files.py @@ -738,6 +738,14 @@ def batch_assemble( ) continue + shader_group = file.get("shader_group") + if ( + shader_group + and debug_file.file_type == "dxil" + and not (debug_file.data or {}).get("shader_group") + ): + debug_file.update(data={**(debug_file.data or {}), "shader_group": shader_group}) + file_response[checksum] = { "state": ChunkFileState.OK, "detail": None, diff --git a/tests/sentry/models/test_debugfile_shader.py b/tests/sentry/models/test_debugfile_shader.py index 51768f0e5302..faa80505fa04 100644 --- a/tests/sentry/models/test_debugfile_shader.py +++ b/tests/sentry/models/test_debugfile_shader.py @@ -6,6 +6,7 @@ import pytest +from sentry.api.endpoints.debug_files import batch_assemble from sentry.constants import KNOWN_DIF_FORMATS from sentry.models.debugfile import ( DIF_MIMETYPES, @@ -21,6 +22,7 @@ determine_dif_kind, ) from sentry.models.project import Project +from sentry.tasks.assemble import ChunkFileState from sentry.testutils.pytest.fixtures import django_db_all DXC_HASH = "731322bf682bdc554eefdd89098b7383" @@ -227,3 +229,28 @@ def test_reupload_without_shader_group_is_noop(default_project: Project, tmp_pat first.refresh_from_db() assert first.data["shader_group"] == "Group A" + + +@django_db_all +def test_assemble_backfills_shader_group_for_known_checksum( + default_project: Project, tmp_path: Path +) -> None: + first, _ = _create_shader_dif(default_project, tmp_path, "a.dxil", DXIL_BYTES, None) + assert first.checksum is not None + + response = batch_assemble( + default_project, + { + first.checksum: { + "name": "a.dxil", + "debug_id": DEBUG_ID, + "chunks": [first.checksum], + "code_id": AFTERMATH_HASH, + "shader_group": "Group A", + } + }, + ) + + assert response[first.checksum]["state"] == ChunkFileState.OK + first.refresh_from_db() + assert first.data["shader_group"] == "Group A" From 6a12c84da47319820b91111cf6820cd94c9e6e6c Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Mon, 2 Nov 2026 20:21:27 +0200 Subject: [PATCH 09/26] Drop the shader PDB debug file kind The Linux decoder never reads a shader PDB, so only the DXIL binary is stored, resolved and sent to teapot. The dxilpdb kind, the sibling PDB lookup and the shader_pdbs request field return with line numbers. --- src/sentry/constants.py | 1 - src/sentry/lang/native/shader_difs.py | 81 +++------------- src/sentry/lang/native/teapot.py | 12 +-- src/sentry/models/debugfile.py | 40 +------- src/sentry/tasks/gpu_crash.py | 1 - .../lang/native/test_teapot_shader_difs.py | 81 ++++------------ tests/sentry/models/test_debugfile_shader.py | 97 +++++-------------- 7 files changed, 65 insertions(+), 248 deletions(-) diff --git a/src/sentry/constants.py b/src/sentry/constants.py index 3a0c06b8d497..c6a39bff51d5 100644 --- a/src/sentry/constants.py +++ b/src/sentry/constants.py @@ -375,7 +375,6 @@ def parse_log_level(level: str | None) -> LogLevel | None: "application/x-portable-pdb": "portablepdb", "application/x-dartsymbolmap+json": "dartsymbolmap", "application/x-dxil": "dxil", - "application/x-dxil-pdb": "dxilpdb", } NATIVE_UNKNOWN_STRING = "" diff --git a/src/sentry/lang/native/shader_difs.py b/src/sentry/lang/native/shader_difs.py index b86560454d8b..7dff0e7e09a7 100644 --- a/src/sentry/lang/native/shader_difs.py +++ b/src/sentry/lang/native/shader_difs.py @@ -1,23 +1,17 @@ -"""Resolve the shader debug files teapot reports missing to presigned URLs. +"""Resolve the shader binaries teapot reports missing to presigned URLs. -``missing_difs`` names two shader kinds: ``dxil`` (the DXC binary, keyed by the -16-hex Aftermath shader hash stored as ``code_id``) and ``dxil_pdb`` (the DXC PDB, -keyed by its DebugName, whose stem is the DXC hash and therefore the ``debug_id``). +``missing_difs`` names them by kind ``dxil`` and the 16-hex Aftermath shader hash +stored as ``code_id``. """ from __future__ import annotations import logging -import os -import re from collections.abc import Mapping from dataclasses import dataclass, field from datetime import timedelta from typing import Any -from symbolic.debuginfo import normalize_debug_id -from symbolic.exceptions import SymbolicError - from sentry.debug_files.debug_files import maybe_renew_debug_files from sentry.models.debugfile import ProjectDebugFile from sentry.models.project import Project @@ -28,43 +22,27 @@ SHADER_DIF_URL_VALIDITY = timedelta(minutes=15) -_PDB_SUFFIX_RE = re.compile(r"\.pdb$", re.IGNORECASE) - @dataclass class ShaderDifs: - """Entries for teapot's ``shader_binaries`` and ``shader_pdbs`` request arrays.""" + """Entries for teapot's ``shader_binaries`` request array.""" binaries: list[dict[str, str]] = field(default_factory=list) - pdbs: list[dict[str, str]] = field(default_factory=list) def __bool__(self) -> bool: - return bool(self.binaries or self.pdbs) + return bool(self.binaries) -def missing_shader_difs(response: Mapping[str, Any]) -> tuple[list[str], list[str]]: - """Split teapot's ``missing_difs`` into dxil code ids and dxil_pdb debug names.""" +def missing_shader_difs(response: Mapping[str, Any]) -> list[str]: + """The dxil code ids in teapot's ``missing_difs``.""" code_ids: list[str] = [] - debug_names: list[str] = [] for entry in response.get("missing_difs") or []: - if not isinstance(entry, dict): + if not isinstance(entry, dict) or entry.get("kind") != "dxil": continue debug_id = entry.get("debug_id") - if not isinstance(debug_id, str) or not debug_id: - continue - if entry.get("kind") == "dxil": + if isinstance(debug_id, str) and debug_id: code_ids.append(debug_id) - elif entry.get("kind") == "dxil_pdb": - debug_names.append(debug_id) - return code_ids, debug_names - - -def debug_id_from_debug_name(debug_name: str) -> str | None: - stem = _PDB_SUFFIX_RE.sub("", os.path.basename(debug_name.replace("\\", "/"))) - try: - return normalize_debug_id(stem) - except SymbolicError: - return None + return code_ids def _presigned_dif( @@ -88,23 +66,11 @@ def _presigned_dif( return dif, url -def _pdb_debug_name(pdb: ProjectDebugFile) -> str: - name = os.path.basename((pdb.object_name or "").replace("\\", "/")) - if _PDB_SUFFIX_RE.search(name): - return name - return f"{pdb.debug_id.replace('-', '')[:32]}.pdb" - - def resolve_shader_difs(project: Project, response: Mapping[str, Any]) -> ShaderDifs: - """Look up every shader DIF teapot reported missing and presign the ones we hold. - - The PDB uploaded next to a resolved binary is sent along, since teapot only asks - for a shader's PDB once it has the binary. - """ - code_ids, debug_names = missing_shader_difs(response) + """Look up every shader binary teapot reported missing and presign the ones we hold.""" + code_ids = missing_shader_difs(response) resolved = ShaderDifs() found: list[ProjectDebugFile] = [] - requested_pdbs = {name.lower() for name in debug_names} for code_id in dict.fromkeys(code_ids): dxil_match = _presigned_dif(project.id, "dxil", code_id=code_id.lower()) @@ -117,27 +83,8 @@ def resolve_shader_difs(project: Project, response: Mapping[str, Any]) -> Shader entry["group"] = group resolved.binaries.append(entry) - pdb_match = _presigned_dif(project.id, "dxilpdb", debug_id=dxil.debug_id) - if pdb_match is None: - continue - pdb, pdb_url = pdb_match - debug_name = _pdb_debug_name(pdb) - if debug_name.lower() not in requested_pdbs: - found.append(pdb) - resolved.pdbs.append({"debug_name": debug_name, "storage_url": pdb_url}) - - for debug_name in dict.fromkeys(debug_names): - debug_id = debug_id_from_debug_name(debug_name) - pdb_match = _presigned_dif(project.id, "dxilpdb", debug_id=debug_id) if debug_id else None - if pdb_match is None: - continue - pdb, url = pdb_match - found.append(pdb) - resolved.pdbs.append({"debug_name": debug_name, "storage_url": url}) - - missing = len(code_ids) + len(debug_names) - if missing: - metrics.incr("tasks.gpu_crash.shader_difs.missing", amount=missing) + if code_ids: + metrics.incr("tasks.gpu_crash.shader_difs.missing", amount=len(code_ids)) if found: maybe_renew_debug_files(found) metrics.incr("tasks.gpu_crash.shader_difs.resolved", amount=len(found)) diff --git a/src/sentry/lang/native/teapot.py b/src/sentry/lang/native/teapot.py index 6af783f8efff..918c1489119e 100644 --- a/src/sentry/lang/native/teapot.py +++ b/src/sentry/lang/native/teapot.py @@ -140,7 +140,6 @@ def symbolicate( dump: TeapotAttachment, shader_debug_info: Sequence[TeapotAttachment] | None = None, shader_binaries: Sequence[dict[str, str]] | None = None, - shader_pdbs: Sequence[dict[str, str]] | None = None, ) -> dict[str, Any]: shader_debug_info = shader_debug_info or [] url = f"{self.base_url}/symbolicate" @@ -160,10 +159,9 @@ def symbolicate( ], } request_id = self.event_id - if shader_binaries or shader_pdbs: + if shader_binaries: # Second round: a distinct id, or teapot's idempotency cache replays the first response. - body["shader_binaries"] = list(shader_binaries or []) - body["shader_pdbs"] = list(shader_pdbs or []) + body["shader_binaries"] = list(shader_binaries) request_id = f"{self.event_id}-2" data = orjson.dumps(body) headers = { @@ -235,7 +233,6 @@ def submit_to_teapot( dump: TeapotAttachment, shader_debug_info: Sequence[TeapotAttachment] | None = None, shader_binaries: Sequence[dict[str, str]] | None = None, - shader_pdbs: Sequence[dict[str, str]] | None = None, ) -> dict[str, Any] | None: """Best-effort teapot invocation. @@ -244,8 +241,8 @@ def submit_to_teapot( trip the circuit breaker. Raises ``TeapotUnavailable`` on a genuine outage (network, timeout, 5xx) so the caller can. - ``shader_binaries`` / ``shader_pdbs`` make this the second round, with presigned - URLs for the shader DIFs teapot reported missing. + ``shader_binaries`` makes this the second round, with presigned URLs for the + shader binaries teapot reported missing. """ try: @@ -254,7 +251,6 @@ def submit_to_teapot( dump, shader_debug_info or [], shader_binaries=shader_binaries, - shader_pdbs=shader_pdbs, ) except TeapotRequestError as e: logger.info("teapot.request_error", extra={"event_id": event_id, "error": str(e)}) diff --git a/src/sentry/models/debugfile.py b/src/sentry/models/debugfile.py index b7fdca4b74aa..c20b8c8a8eb2 100644 --- a/src/sentry/models/debugfile.py +++ b/src/sentry/models/debugfile.py @@ -69,12 +69,7 @@ _proguard_file_re = re.compile(r"/proguard/(?:mapping-)?(.*?)\.txt$") -# DXC shader binaries carry the DXBC container magic; DXC shader PDBs are MSF -# files whose stream directory names an `ILDN` (debug name) part. _DXIL_MAGIC = b"DXBC" -_MSF_MAGIC = b"Microsoft C/C++ MSF 7.00" -_DXIL_PDB_PART = b"ILDN" -_DXIL_PDB_SCAN_BYTES = 64 * 1024 _SHADER_CODE_ID_RE = re.compile(r"^[0-9a-f]{16}$") SHADER_GROUP_MAX_LENGTH = 512 @@ -107,8 +102,6 @@ def _dif_file_extension(file_format: str, file_type: str | None) -> str: return ".json" if file_format == "dxil": return ".dxil" - if file_format == "dxilpdb": - return ".pdb" return "" @@ -467,7 +460,6 @@ def clean_redundant_difs(project: Project, debug_id: str) -> None: uuidmap_seen = False il2cpp_seen = False dxil_seen = False - dxilpdb_seen = False for i, dif in enumerate(difs): mime_type = dif.get_content_type() if mime_type == DIF_MIMETYPES["bcsymbolmap"]: @@ -490,11 +482,6 @@ def clean_redundant_difs(project: Project, debug_id: str) -> None: dxil_seen = True else: dif.delete() - elif mime_type == DIF_MIMETYPES["dxilpdb"]: - if not dxilpdb_seen: - dxilpdb_seen = True - else: - dif.delete() else: # We always keep the latest file. If it has no features, likely the # previous files did not have features either and will be removed, or we @@ -539,7 +526,6 @@ def _get_dif_object_name(meta: DifMeta) -> str: "il2cpp", "dartsymbolmap", "dxil", - "dxilpdb", ): return meta.name if meta.file_format == "breakpad": @@ -914,7 +900,6 @@ class DifKind(enum.Enum): Il2Cpp = enum.auto() DartSymbolMap = enum.auto() Dxil = enum.auto() - DxilPdb = enum.auto() # TODO(flub): should we include proguard? The tradeoff is that we'd be matching the # regex of it twice. That cost is probably not too great to worry about. @@ -936,22 +921,17 @@ def determine_dif_kind(path: str) -> DifKind: return DifKind.DartSymbolMap elif data.startswith(_DXIL_MAGIC): return DifKind.Dxil - elif _MSF_MAGIC.startswith(data): - data += fp.read(_DXIL_PDB_SCAN_BYTES - len(data)) - if data.startswith(_MSF_MAGIC) and _DXIL_PDB_PART in data: - return DifKind.DxilPdb - return DifKind.Object else: return DifKind.Object -def _shader_debug_id(debug_id: str | None, kind: str) -> str: +def _shader_debug_id(debug_id: str | None) -> str: if debug_id is None: - raise BadDif(f"Missing debug_id for {kind}") + raise BadDif("Missing debug_id for dxil") try: return normalize_debug_id(debug_id) except SymbolicError as e: - raise BadDif(f"Invalid debug_id for {kind}: {e}") + raise BadDif(f"Invalid debug_id for dxil: {e}") def detect_dif_from_path( @@ -1093,25 +1073,13 @@ def detect_dif_from_path( DifMeta( file_format="dxil", arch="any", - debug_id=_shader_debug_id(debug_id, "dxil"), + debug_id=_shader_debug_id(debug_id), code_id=code_id, name=name, path=path, data=dxil_data, ) ] - elif dif_kind == DifKind.DxilPdb: - logger.debug("File loaded as dxilpdb: %s", path) - return [ - DifMeta( - file_format="dxilpdb", - arch="any", - debug_id=_shader_debug_id(debug_id, "dxilpdb"), - name=name, - path=path, - data={"type": "dxilpdb", "features": ["debug"]}, - ) - ] else: # native debug information files (MachO, ELF or Breakpad) try: diff --git a/src/sentry/tasks/gpu_crash.py b/src/sentry/tasks/gpu_crash.py index f4f2b87b85e7..86ed128c9afd 100644 --- a/src/sentry/tasks/gpu_crash.py +++ b/src/sentry/tasks/gpu_crash.py @@ -208,7 +208,6 @@ def _maybe_resolve_shader_difs( dump, shaders, shader_binaries=difs.binaries, - shader_pdbs=difs.pdbs, ) except TeapotUnavailable: metrics.incr("tasks.gpu_crash.shader_difs", tags={"result": "unavailable"}) diff --git a/tests/sentry/lang/native/test_teapot_shader_difs.py b/tests/sentry/lang/native/test_teapot_shader_difs.py index 1ba6f51edb9a..f5c18ba6c982 100644 --- a/tests/sentry/lang/native/test_teapot_shader_difs.py +++ b/tests/sentry/lang/native/test_teapot_shader_difs.py @@ -11,22 +11,15 @@ from django.conf import settings from django.utils import timezone -from sentry.lang.native.shader_difs import ( - ShaderDifs, - debug_id_from_debug_name, - missing_shader_difs, - resolve_shader_difs, -) +from sentry.lang.native.shader_difs import ShaderDifs, missing_shader_difs, resolve_shader_difs from sentry.lang.native.teapot import TeapotClient, TeapotUnavailable -from sentry.models.debugfile import ProjectDebugFile +from sentry.models.debugfile import DIF_MIMETYPES, ProjectDebugFile from sentry.models.project import Project from sentry.tasks.gpu_crash import _try_symbolicate from sentry.testutils.helpers.options import override_options from sentry.testutils.pytest.fixtures import django_db_all -DXC_HASH = "731322bf682bdc554eefdd89098b7383" DEBUG_ID = "731322bf-682b-dc55-4eef-dd89098b7383" -DEBUG_NAME = f"{DXC_HASH}.pdb" AFTERMATH_HASH = "0123456789abcdef" SHADER_GROUP = "GlobalShaders" @@ -63,10 +56,7 @@ def _partial(**overrides: Any) -> dict[str, Any]: base: dict[str, Any] = { "status": "partial", "fault_category": "shader_hang", - "missing_difs": [ - {"debug_id": AFTERMATH_HASH, "kind": "dxil"}, - {"debug_id": DEBUG_NAME, "kind": "dxil_pdb"}, - ], + "missing_difs": [{"debug_id": AFTERMATH_HASH, "kind": "dxil"}], } base.update(overrides) return base @@ -84,8 +74,7 @@ def _resolved() -> ShaderDifs: "storage_url": "http://objectstore/dxil?sig=x", "group": SHADER_GROUP, } - ], - pdbs=[{"debug_name": DEBUG_NAME, "storage_url": "http://objectstore/pdb?sig=x"}], + ] ) @@ -106,15 +95,12 @@ def teapot_session() -> Iterator[mock.Mock]: # --------------------------------------------------------------------------- -def test_client_second_round_sends_shader_arrays(teapot_session: mock.Mock) -> None: +def test_client_second_round_sends_shader_binaries(teapot_session: mock.Mock) -> None: resolved = _resolved() with mock.patch("sentry.lang.native.teapot.requests.post") as mock_post: mock_post.return_value = _FakeResponse(200, _completed()) TeapotClient(_FakeProject(), "abc").symbolicate( - _FakeAttachment(), - [], - shader_binaries=resolved.binaries, - shader_pdbs=resolved.pdbs, + _FakeAttachment(), [], shader_binaries=resolved.binaries ) kwargs = mock_post.call_args.kwargs @@ -125,10 +111,9 @@ def test_client_second_round_sends_shader_arrays(teapot_session: mock.Mock) -> N assert body["event_id"] == "abc" assert body["dump"] == {"storage_url": "http://objectstore/dump-obj-id?sig=abc"} assert body["shader_binaries"] == resolved.binaries - assert body["shader_pdbs"] == resolved.pdbs -def test_client_first_round_omits_shader_arrays(teapot_session: mock.Mock) -> None: +def test_client_first_round_omits_shader_binaries(teapot_session: mock.Mock) -> None: with mock.patch("sentry.lang.native.teapot.requests.post") as mock_post: mock_post.return_value = _FakeResponse(200, _completed()) TeapotClient(_FakeProject(), "abc").symbolicate(_FakeAttachment(), []) @@ -138,7 +123,6 @@ def test_client_first_round_omits_shader_arrays(teapot_session: mock.Mock) -> No assert kwargs["headers"]["X-Request-Id"] == "abc" body = orjson.loads(kwargs["data"]) assert "shader_binaries" not in body - assert "shader_pdbs" not in body # --------------------------------------------------------------------------- @@ -150,40 +134,23 @@ def test_missing_shader_difs_parsing() -> None: response = _partial( missing_difs=[ {"debug_id": AFTERMATH_HASH, "kind": "dxil"}, - {"debug_id": DEBUG_NAME, "kind": "dxil_pdb"}, - {"debug_id": "deadbeef", "kind": "pdb"}, + {"debug_id": "deadbeef", "kind": "unknown"}, {"kind": "dxil"}, "garbage", ] ) - assert missing_shader_difs(response) == ([AFTERMATH_HASH], [DEBUG_NAME]) - assert missing_shader_difs({}) == ([], []) - - -@pytest.mark.parametrize( - "debug_name,expected", - ( - (DEBUG_NAME, DEBUG_ID), - (DXC_HASH.upper() + ".PDB", DEBUG_ID), - ("C:\\build\\shaders\\" + DEBUG_NAME, DEBUG_ID), - (DXC_HASH, DEBUG_ID), - ("not-a-hash.pdb", None), - ), -) -def test_debug_id_from_debug_name(debug_name: str, expected: str | None) -> None: - assert debug_id_from_debug_name(debug_name) == expected + assert missing_shader_difs(response) == [AFTERMATH_HASH] + assert missing_shader_difs({}) == [] def _create_dif( project: Project, - dif_type: str, + dif_type: str = "dxil", code_id: str | None = None, storage_path: str | None = "difs/shader", shader_group: str | None = None, ) -> ProjectDebugFile: - content_type = "application/x-dxil" if dif_type == "dxil" else "application/x-dxil-pdb" - features = ["symtab"] if dif_type == "dxil" else ["debug"] - data: dict[str, Any] = {"type": dif_type, "features": features} + data: dict[str, Any] = {"type": dif_type, "features": ["symtab"]} if shader_group is not None: data["shader_group"] = shader_group return ProjectDebugFile.objects.create( @@ -195,7 +162,7 @@ def _create_dif( checksum="0" * 40, data=data, storage_path=storage_path, - content_type=content_type, + content_type=DIF_MIMETYPES[dif_type], file_size=64, date_created=timezone.now(), ) @@ -219,12 +186,10 @@ def test_resolve_shader_difs_presigns_objectstore_rows( ) -> None: _create_dif( default_project, - "dxil", code_id=AFTERMATH_HASH, storage_path="difs/dxil", shader_group=SHADER_GROUP, ) - _create_dif(default_project, "dxilpdb", storage_path="difs/pdb") resolved = resolve_shader_difs(default_project, _partial()) @@ -236,9 +201,6 @@ def test_resolve_shader_difs_presigns_objectstore_rows( "group": SHADER_GROUP, } ] - assert resolved.pdbs == [ - {"debug_name": DEBUG_NAME, "storage_url": "http://objectstore/difs/pdb?sig=x"} - ] # Token validity is the longer DIF one, not the attachment default. assert dif_presign.call_args.kwargs["token_validity"] is not None @@ -247,7 +209,7 @@ def test_resolve_shader_difs_presigns_objectstore_rows( def test_resolve_shader_difs_matches_code_id_case_insensitively( default_project: Project, dif_presign: mock.Mock ) -> None: - _create_dif(default_project, "dxil", code_id=AFTERMATH_HASH) + _create_dif(default_project, code_id=AFTERMATH_HASH) response = _partial(missing_difs=[{"debug_id": AFTERMATH_HASH.upper(), "kind": "dxil"}]) resolved = resolve_shader_difs(default_project, response) @@ -255,28 +217,26 @@ def test_resolve_shader_difs_matches_code_id_case_insensitively( assert resolved.binaries == [ {"code_id": AFTERMATH_HASH, "storage_url": "http://objectstore/difs/shader?sig=x"} ] - assert resolved.pdbs == [] @django_db_all def test_resolve_shader_difs_ignores_other_types( default_project: Project, dif_presign: mock.Mock ) -> None: - # Same debug_id, but the row is the binary; teapot asked for the PDB. - _create_dif(default_project, "dxil", code_id=AFTERMATH_HASH) - response = _partial(missing_difs=[{"debug_id": DEBUG_NAME, "kind": "dxil_pdb"}]) + # Same code_id, but the row is not a shader binary. + _create_dif(default_project, dif_type="pe", code_id=AFTERMATH_HASH) - resolved = resolve_shader_difs(default_project, response) + resolved = resolve_shader_difs(default_project, _partial()) assert bool(resolved) is False + dif_presign.assert_not_called() @django_db_all def test_resolve_shader_difs_skips_legacy_rows( default_project: Project, dif_presign: mock.Mock ) -> None: - _create_dif(default_project, "dxil", code_id=AFTERMATH_HASH, storage_path=None) - _create_dif(default_project, "dxilpdb", storage_path=None) + _create_dif(default_project, code_id=AFTERMATH_HASH, storage_path=None) resolved = resolve_shader_difs(default_project, _partial()) @@ -322,8 +282,7 @@ def test_task_runs_second_round_with_resolved_difs( assert resolve.call_args.args[1] is partial assert teapot.call_count == 2 second = teapot.call_args_list[1] - assert second.kwargs["shader_binaries"] == _resolved().binaries - assert second.kwargs["shader_pdbs"] == _resolved().pdbs + assert second.kwargs == {"shader_binaries": _resolved().binaries} assert apply.call_args.args[1] is completed circuit_breaker.record_success.assert_called_once() circuit_breaker.record_error.assert_not_called() diff --git a/tests/sentry/models/test_debugfile_shader.py b/tests/sentry/models/test_debugfile_shader.py index faa80505fa04..29a11be0ed78 100644 --- a/tests/sentry/models/test_debugfile_shader.py +++ b/tests/sentry/models/test_debugfile_shader.py @@ -30,9 +30,6 @@ AFTERMATH_HASH = "0123456789abcdef" DXIL_BYTES = b"DXBC" + b"\x00" * 60 -MSF_MAGIC = b"Microsoft C/C++ MSF 7.00\r\n\x1aDS\x00\x00\x00" -DXIL_PDB_BYTES = MSF_MAGIC + b"\x00" * 4096 + b"ILDN" + b"\x00" * 64 -PLAIN_PDB_BYTES = MSF_MAGIC + b"\x00" * 4096 def _write(tmp_path: Path, name: str, data: bytes) -> str: @@ -41,33 +38,16 @@ def _write(tmp_path: Path, name: str, data: bytes) -> str: return str(path) -def test_shader_formats_registered() -> None: +def test_dxil_format_registered() -> None: assert KNOWN_DIF_FORMATS["application/x-dxil"] == "dxil" - assert KNOWN_DIF_FORMATS["application/x-dxil-pdb"] == "dxilpdb" assert DIF_MIMETYPES["dxil"] == "application/x-dxil" - assert DIF_MIMETYPES["dxilpdb"] == "application/x-dxil-pdb" assert _dif_file_extension("dxil", "dxil") == ".dxil" - assert _dif_file_extension("dxilpdb", "dxilpdb") == ".pdb" def test_determine_dif_kind_dxil(tmp_path: Path) -> None: assert determine_dif_kind(_write(tmp_path, "a.dxil", DXIL_BYTES)) == DifKind.Dxil -def test_determine_dif_kind_dxil_pdb(tmp_path: Path) -> None: - assert determine_dif_kind(_write(tmp_path, "a.pdb", DXIL_PDB_BYTES)) == DifKind.DxilPdb - - -def test_determine_dif_kind_plain_pdb_is_object(tmp_path: Path) -> None: - # A regular MSF PDB has no ILDN part and stays with symbolic. - assert determine_dif_kind(_write(tmp_path, "a.pdb", PLAIN_PDB_BYTES)) == DifKind.Object - - -def test_determine_dif_kind_ildn_outside_scan_window_is_object(tmp_path: Path) -> None: - data = MSF_MAGIC + b"\x00" * (64 * 1024) + b"ILDN" - assert determine_dif_kind(_write(tmp_path, "a.pdb", data)) == DifKind.Object - - def test_detect_dxil(tmp_path: Path) -> None: path = _write(tmp_path, "shader.dxil", DXIL_BYTES) @@ -93,28 +73,14 @@ def test_detect_dxil_without_code_id(tmp_path: Path) -> None: assert meta.code_id is None -def test_detect_dxil_pdb(tmp_path: Path) -> None: - path = _write(tmp_path, f"{DXC_HASH}.pdb", DXIL_PDB_BYTES) - - meta = detect_single_dif_from_path(path, name=f"{DXC_HASH}.pdb", debug_id=DXC_HASH) - - assert meta.file_format == "dxilpdb" - assert meta.arch == "any" - assert meta.debug_id == DEBUG_ID - assert meta.code_id is None - assert meta.data == {"type": "dxilpdb", "features": ["debug"]} - assert _get_dif_object_name(meta) == f"{DXC_HASH}.pdb" - - -@pytest.mark.parametrize("data", (DXIL_BYTES, DXIL_PDB_BYTES)) -def test_detect_shader_requires_debug_id(tmp_path: Path, data: bytes) -> None: - path = _write(tmp_path, "shader.bin", data) +def test_detect_dxil_requires_debug_id(tmp_path: Path) -> None: + path = _write(tmp_path, "shader.dxil", DXIL_BYTES) with pytest.raises(BadDif, match="Missing debug_id"): - detect_dif_from_path(path, name="shader.bin") + detect_dif_from_path(path, name="shader.dxil") -def test_detect_shader_rejects_invalid_debug_id(tmp_path: Path) -> None: +def test_detect_dxil_rejects_invalid_debug_id(tmp_path: Path) -> None: path = _write(tmp_path, "shader.dxil", DXIL_BYTES) with pytest.raises(BadDif, match="Invalid debug_id"): @@ -128,25 +94,23 @@ def test_detect_dxil_rejects_invalid_code_id(tmp_path: Path) -> None: detect_dif_from_path(path, name="shader.dxil", debug_id=DXC_HASH, code_id="abc") -def test_detect_dxil_stores_shader_group(tmp_path: Path) -> None: - path = _write(tmp_path, "shader.dxil", DXIL_BYTES) - - meta = detect_single_dif_from_path( - path, name="shader.dxil", debug_id=DXC_HASH, shader_group=" GlobalShaders " - ) - - assert meta.data == {"type": "dxil", "features": ["symtab"], "shader_group": "GlobalShaders"} - - -@pytest.mark.parametrize("shader_group", (None, "", " ")) -def test_detect_dxil_drops_empty_shader_group(tmp_path: Path, shader_group: str | None) -> None: +@pytest.mark.parametrize( + "shader_group,stored", + ((" GlobalShaders ", "GlobalShaders"), (None, None), ("", None), (" ", None)), +) +def test_detect_dxil_stores_shader_group( + tmp_path: Path, shader_group: str | None, stored: str | None +) -> None: path = _write(tmp_path, "shader.dxil", DXIL_BYTES) meta = detect_single_dif_from_path( path, name="shader.dxil", debug_id=DXC_HASH, shader_group=shader_group ) - assert meta.data == {"type": "dxil", "features": ["symtab"]} + expected: dict[str, object] = {"type": "dxil", "features": ["symtab"]} + if stored is not None: + expected["shader_group"] = stored + assert meta.data == expected def test_detect_dxil_rejects_overlong_shader_group(tmp_path: Path) -> None: @@ -161,16 +125,6 @@ def test_detect_dxil_rejects_overlong_shader_group(tmp_path: Path) -> None: ) -def test_detect_dxil_pdb_ignores_shader_group(tmp_path: Path) -> None: - path = _write(tmp_path, f"{DXC_HASH}.pdb", DXIL_PDB_BYTES) - - meta = detect_single_dif_from_path( - path, name=f"{DXC_HASH}.pdb", debug_id=DXC_HASH, shader_group="GlobalShaders" - ) - - assert meta.data == {"type": "dxilpdb", "features": ["debug"]} - - def _create_shader_dif( project: Project, tmp_path: Path, name: str, data: bytes, shader_group: str | None ) -> tuple[ProjectDebugFile, bool]: @@ -209,11 +163,16 @@ def test_reupload_backfills_missing_shader_group(default_project: Project, tmp_p assert ProjectDebugFile.objects.filter(project_id=default_project.id).count() == 1 +@pytest.mark.parametrize("reupload_group", ("Group B", None)) @django_db_all -def test_reupload_keeps_existing_shader_group(default_project: Project, tmp_path: Path) -> None: +def test_reupload_keeps_existing_shader_group( + default_project: Project, tmp_path: Path, reupload_group: str | None +) -> None: first, _ = _create_shader_dif(default_project, tmp_path, "a.dxil", DXIL_BYTES, "Group A") - second, created = _create_shader_dif(default_project, tmp_path, "a.dxil", DXIL_BYTES, "Group B") + second, created = _create_shader_dif( + default_project, tmp_path, "a.dxil", DXIL_BYTES, reupload_group + ) assert created is False assert second.id == first.id @@ -221,16 +180,6 @@ def test_reupload_keeps_existing_shader_group(default_project: Project, tmp_path assert first.data["shader_group"] == "Group A" -@django_db_all -def test_reupload_without_shader_group_is_noop(default_project: Project, tmp_path: Path) -> None: - first, _ = _create_shader_dif(default_project, tmp_path, "a.dxil", DXIL_BYTES, "Group A") - - _create_shader_dif(default_project, tmp_path, "a.dxil", DXIL_BYTES, None) - - first.refresh_from_db() - assert first.data["shader_group"] == "Group A" - - @django_db_all def test_assemble_backfills_shader_group_for_known_checksum( default_project: Project, tmp_path: Path From 01f0fe83c39bc880dd24d85bba5384ac01b6c260 Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Tue, 3 Nov 2026 14:39:03 +0200 Subject: [PATCH 10/26] Drop the dxil object kind from debug file data --- src/sentry/api/endpoints/debug_files.py | 2 +- src/sentry/lang/native/shader_difs.py | 21 +++++++------------ src/sentry/models/debugfile.py | 2 +- .../lang/native/test_teapot_shader_difs.py | 2 +- tests/sentry/models/test_debugfile_shader.py | 8 +++---- 5 files changed, 14 insertions(+), 21 deletions(-) diff --git a/src/sentry/api/endpoints/debug_files.py b/src/sentry/api/endpoints/debug_files.py index 513e17328538..429d52c9db3a 100644 --- a/src/sentry/api/endpoints/debug_files.py +++ b/src/sentry/api/endpoints/debug_files.py @@ -741,7 +741,7 @@ def batch_assemble( shader_group = file.get("shader_group") if ( shader_group - and debug_file.file_type == "dxil" + and debug_file.file_format == "dxil" and not (debug_file.data or {}).get("shader_group") ): debug_file.update(data={**(debug_file.data or {}), "shader_group": shader_group}) diff --git a/src/sentry/lang/native/shader_difs.py b/src/sentry/lang/native/shader_difs.py index 7dff0e7e09a7..165973db32aa 100644 --- a/src/sentry/lang/native/shader_difs.py +++ b/src/sentry/lang/native/shader_difs.py @@ -6,7 +6,6 @@ from __future__ import annotations -import logging from collections.abc import Mapping from dataclasses import dataclass, field from datetime import timedelta @@ -18,8 +17,6 @@ from sentry.objectstore import get_internal_download_url from sentry.utils import metrics -logger = logging.getLogger(__name__) - SHADER_DIF_URL_VALIDITY = timedelta(minutes=15) @@ -48,17 +45,13 @@ def missing_shader_difs(response: Mapping[str, Any]) -> list[str]: def _presigned_dif( project_id: int, dif_type: str, **filters: str ) -> tuple[ProjectDebugFile, str] | None: - """The newest debug file of ``dif_type`` matching ``filters`` with a presigned GET - URL, or None if there is none or it is not in objectstore.""" - difs = ProjectDebugFile.objects.filter(project_id=project_id, **filters).order_by("-id") - dif = next((dif for dif in difs if dif.file_type == dif_type), None) - if dif is None: - return None - if not dif.storage_path: - logger.info( - "gpu_crash.shader_dif.not_in_objectstore", - extra={"dif_id": dif.id, "debug_id": dif.debug_id, "type": dif_type}, - ) + """The newest objectstore-backed debug file of ``dif_type`` matching ``filters`` + with a presigned GET URL, or None if there is none.""" + difs = ProjectDebugFile.objects.filter( + project_id=project_id, storage_path__isnull=False, **filters + ).order_by("-id") + dif = next((dif for dif in difs if dif.file_format == dif_type), None) + if dif is None or not dif.storage_path: return None url = get_internal_download_url( dif.get_objectstore_session(), dif.storage_path, token_validity=SHADER_DIF_URL_VALIDITY diff --git a/src/sentry/models/debugfile.py b/src/sentry/models/debugfile.py index c20b8c8a8eb2..0f4c722fe034 100644 --- a/src/sentry/models/debugfile.py +++ b/src/sentry/models/debugfile.py @@ -1066,7 +1066,7 @@ def detect_dif_from_path( f"Invalid shader_group for dxil (max {SHADER_GROUP_MAX_LENGTH} characters)" ) logger.debug("File loaded as dxil: %s", path) - dxil_data: dict[str, Any] = {"type": "dxil", "features": ["symtab"]} + dxil_data: dict[str, Any] = {"features": ["symtab"]} if shader_group: dxil_data["shader_group"] = shader_group return [ diff --git a/tests/sentry/lang/native/test_teapot_shader_difs.py b/tests/sentry/lang/native/test_teapot_shader_difs.py index f5c18ba6c982..18f7c3741df7 100644 --- a/tests/sentry/lang/native/test_teapot_shader_difs.py +++ b/tests/sentry/lang/native/test_teapot_shader_difs.py @@ -150,7 +150,7 @@ def _create_dif( storage_path: str | None = "difs/shader", shader_group: str | None = None, ) -> ProjectDebugFile: - data: dict[str, Any] = {"type": dif_type, "features": ["symtab"]} + data: dict[str, Any] = {"features": ["symtab"]} if shader_group is not None: data["shader_group"] = shader_group return ProjectDebugFile.objects.create( diff --git a/tests/sentry/models/test_debugfile_shader.py b/tests/sentry/models/test_debugfile_shader.py index 29a11be0ed78..b7e1a7663960 100644 --- a/tests/sentry/models/test_debugfile_shader.py +++ b/tests/sentry/models/test_debugfile_shader.py @@ -60,7 +60,7 @@ def test_detect_dxil(tmp_path: Path) -> None: assert meta.debug_id == DEBUG_ID assert meta.code_id == AFTERMATH_HASH assert meta.name == "shader.dxil" - assert meta.data == {"type": "dxil", "features": ["symtab"]} + assert meta.data == {"features": ["symtab"]} assert _get_dif_object_name(meta) == "shader.dxil" @@ -107,7 +107,7 @@ def test_detect_dxil_stores_shader_group( path, name="shader.dxil", debug_id=DXC_HASH, shader_group=shader_group ) - expected: dict[str, object] = {"type": "dxil", "features": ["symtab"]} + expected: dict[str, object] = {"features": ["symtab"]} if stored is not None: expected["shader_group"] = stored assert meta.data == expected @@ -144,7 +144,7 @@ def test_create_dxil_stores_shader_group(default_project: Project, tmp_path: Pat assert created is True dif.refresh_from_db() - assert dif.file_type == "dxil" + assert dif.file_format == "dxil" assert dif.data["shader_group"] == "Group A" @@ -159,7 +159,7 @@ def test_reupload_backfills_missing_shader_group(default_project: Project, tmp_p assert created is False assert second.id == first.id first.refresh_from_db() - assert first.data == {"type": "dxil", "features": ["symtab"], "shader_group": "Group A"} + assert first.data == {"features": ["symtab"], "shader_group": "Group A"} assert ProjectDebugFile.objects.filter(project_id=default_project.id).count() == 1 From dd3cc2b2651d9c1de5d70f2f03b2585d8f6ee11e Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Tue, 3 Nov 2026 14:50:32 +0200 Subject: [PATCH 11/26] Return the resolved shader binaries as a plain list --- src/sentry/lang/native/shader_difs.py | 51 +++++++------------ src/sentry/tasks/gpu_crash.py | 2 +- .../lang/native/test_teapot_shader_difs.py | 32 ++++++------ 3 files changed, 34 insertions(+), 51 deletions(-) diff --git a/src/sentry/lang/native/shader_difs.py b/src/sentry/lang/native/shader_difs.py index 165973db32aa..af7a217dab45 100644 --- a/src/sentry/lang/native/shader_difs.py +++ b/src/sentry/lang/native/shader_difs.py @@ -7,7 +7,6 @@ from __future__ import annotations from collections.abc import Mapping -from dataclasses import dataclass, field from datetime import timedelta from typing import Any @@ -20,16 +19,6 @@ SHADER_DIF_URL_VALIDITY = timedelta(minutes=15) -@dataclass -class ShaderDifs: - """Entries for teapot's ``shader_binaries`` request array.""" - - binaries: list[dict[str, str]] = field(default_factory=list) - - def __bool__(self) -> bool: - return bool(self.binaries) - - def missing_shader_difs(response: Mapping[str, Any]) -> list[str]: """The dxil code ids in teapot's ``missing_difs``.""" code_ids: list[str] = [] @@ -42,39 +31,35 @@ def missing_shader_difs(response: Mapping[str, Any]) -> list[str]: return code_ids -def _presigned_dif( - project_id: int, dif_type: str, **filters: str -) -> tuple[ProjectDebugFile, str] | None: - """The newest objectstore-backed debug file of ``dif_type`` matching ``filters`` - with a presigned GET URL, or None if there is none.""" +def _find_dxil(project_id: int, code_id: str) -> ProjectDebugFile | None: + """The newest objectstore-backed dxil debug file with ``code_id``.""" difs = ProjectDebugFile.objects.filter( - project_id=project_id, storage_path__isnull=False, **filters + project_id=project_id, code_id=code_id, storage_path__isnull=False ).order_by("-id") - dif = next((dif for dif in difs if dif.file_format == dif_type), None) - if dif is None or not dif.storage_path: - return None - url = get_internal_download_url( - dif.get_objectstore_session(), dif.storage_path, token_validity=SHADER_DIF_URL_VALIDITY - ) - return dif, url + return next((dif for dif in difs if dif.file_format == "dxil"), None) -def resolve_shader_difs(project: Project, response: Mapping[str, Any]) -> ShaderDifs: - """Look up every shader binary teapot reported missing and presign the ones we hold.""" +def resolve_shader_difs(project: Project, response: Mapping[str, Any]) -> list[dict[str, str]]: + """Entries for teapot's ``shader_binaries`` request array: every shader binary it + reported missing that we hold, with a presigned URL.""" code_ids = missing_shader_difs(response) - resolved = ShaderDifs() + resolved: list[dict[str, str]] = [] found: list[ProjectDebugFile] = [] - for code_id in dict.fromkeys(code_ids): - dxil_match = _presigned_dif(project.id, "dxil", code_id=code_id.lower()) - if dxil_match is None: + for code_id in dict.fromkeys(code_id.lower() for code_id in code_ids): + dxil = _find_dxil(project.id, code_id) + if dxil is None or not dxil.storage_path: continue - dxil, url = dxil_match found.append(dxil) - entry = {"code_id": code_id.lower(), "storage_url": url} + url = get_internal_download_url( + dxil.get_objectstore_session(), + dxil.storage_path, + token_validity=SHADER_DIF_URL_VALIDITY, + ) + entry = {"code_id": code_id, "storage_url": url} if group := (dxil.data or {}).get("shader_group"): entry["group"] = group - resolved.binaries.append(entry) + resolved.append(entry) if code_ids: metrics.incr("tasks.gpu_crash.shader_difs.missing", amount=len(code_ids)) diff --git a/src/sentry/tasks/gpu_crash.py b/src/sentry/tasks/gpu_crash.py index 86ed128c9afd..8ddbde231de6 100644 --- a/src/sentry/tasks/gpu_crash.py +++ b/src/sentry/tasks/gpu_crash.py @@ -207,7 +207,7 @@ def _maybe_resolve_shader_difs( event_id, dump, shaders, - shader_binaries=difs.binaries, + shader_binaries=difs, ) except TeapotUnavailable: metrics.incr("tasks.gpu_crash.shader_difs", tags={"result": "unavailable"}) diff --git a/tests/sentry/lang/native/test_teapot_shader_difs.py b/tests/sentry/lang/native/test_teapot_shader_difs.py index 18f7c3741df7..7b0e690343d3 100644 --- a/tests/sentry/lang/native/test_teapot_shader_difs.py +++ b/tests/sentry/lang/native/test_teapot_shader_difs.py @@ -11,7 +11,7 @@ from django.conf import settings from django.utils import timezone -from sentry.lang.native.shader_difs import ShaderDifs, missing_shader_difs, resolve_shader_difs +from sentry.lang.native.shader_difs import missing_shader_difs, resolve_shader_difs from sentry.lang.native.teapot import TeapotClient, TeapotUnavailable from sentry.models.debugfile import DIF_MIMETYPES, ProjectDebugFile from sentry.models.project import Project @@ -66,16 +66,14 @@ def _completed() -> dict[str, Any]: return {"status": "completed", "fault_category": "shader_hang", "missing_difs": []} -def _resolved() -> ShaderDifs: - return ShaderDifs( - binaries=[ - { - "code_id": AFTERMATH_HASH, - "storage_url": "http://objectstore/dxil?sig=x", - "group": SHADER_GROUP, - } - ] - ) +def _resolved() -> list[dict[str, str]]: + return [ + { + "code_id": AFTERMATH_HASH, + "storage_url": "http://objectstore/dxil?sig=x", + "group": SHADER_GROUP, + } + ] @pytest.fixture @@ -100,7 +98,7 @@ def test_client_second_round_sends_shader_binaries(teapot_session: mock.Mock) -> with mock.patch("sentry.lang.native.teapot.requests.post") as mock_post: mock_post.return_value = _FakeResponse(200, _completed()) TeapotClient(_FakeProject(), "abc").symbolicate( - _FakeAttachment(), [], shader_binaries=resolved.binaries + _FakeAttachment(), [], shader_binaries=resolved ) kwargs = mock_post.call_args.kwargs @@ -110,7 +108,7 @@ def test_client_second_round_sends_shader_binaries(teapot_session: mock.Mock) -> body = orjson.loads(kwargs["data"]) assert body["event_id"] == "abc" assert body["dump"] == {"storage_url": "http://objectstore/dump-obj-id?sig=abc"} - assert body["shader_binaries"] == resolved.binaries + assert body["shader_binaries"] == resolved def test_client_first_round_omits_shader_binaries(teapot_session: mock.Mock) -> None: @@ -194,7 +192,7 @@ def test_resolve_shader_difs_presigns_objectstore_rows( resolved = resolve_shader_difs(default_project, _partial()) assert bool(resolved) is True - assert resolved.binaries == [ + assert resolved == [ { "code_id": AFTERMATH_HASH, "storage_url": "http://objectstore/difs/dxil?sig=x", @@ -214,7 +212,7 @@ def test_resolve_shader_difs_matches_code_id_case_insensitively( resolved = resolve_shader_difs(default_project, response) - assert resolved.binaries == [ + assert resolved == [ {"code_id": AFTERMATH_HASH, "storage_url": "http://objectstore/difs/shader?sig=x"} ] @@ -282,7 +280,7 @@ def test_task_runs_second_round_with_resolved_difs( assert resolve.call_args.args[1] is partial assert teapot.call_count == 2 second = teapot.call_args_list[1] - assert second.kwargs == {"shader_binaries": _resolved().binaries} + assert second.kwargs == {"shader_binaries": _resolved()} assert apply.call_args.args[1] is completed circuit_breaker.record_success.assert_called_once() circuit_breaker.record_error.assert_not_called() @@ -296,7 +294,7 @@ def test_task_keeps_first_response_when_nothing_resolved(default_project: Projec override_options({"teapot.enabled": True}), mock.patch(FIND_DUMP, return_value=_FakeAttachment()), mock.patch(FIND_SHADERS, return_value=[]), - mock.patch(RESOLVE, return_value=ShaderDifs()), + mock.patch(RESOLVE, return_value=[]), mock.patch(SUBMIT_TEAPOT, return_value=partial) as teapot, mock.patch(APPLY, return_value=data) as apply, ): From b60ff07acd480a1f5f6033e6ed731834b75a5dba Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Tue, 3 Nov 2026 14:50:32 +0200 Subject: [PATCH 12/26] Backfill the shader group only in the assemble endpoint --- src/sentry/models/debugfile.py | 9 ------ tests/sentry/models/test_debugfile_shader.py | 32 -------------------- 2 files changed, 41 deletions(-) diff --git a/src/sentry/models/debugfile.py b/src/sentry/models/debugfile.py index 0f4c722fe034..0de2e3569779 100644 --- a/src/sentry/models/debugfile.py +++ b/src/sentry/models/debugfile.py @@ -549,13 +549,6 @@ def find_existing_dif(project: Project, meta: DifMeta, checksum: str) -> Project ) -def _backfill_shader_group(dif: ProjectDebugFile, meta: DifMeta) -> None: - """A re-upload of the same dxil binary may carry the shader group an earlier cook did not.""" - shader_group = (meta.data or {}).get("shader_group") - if shader_group and not (dif.data or {}).get("shader_group"): - dif.update(data={**(dif.data or {}), "shader_group": shader_group}) - - def _get_fileobj_checksum_and_size(fileobj: IO[bytes]) -> tuple[str, int]: file_size = 0 hasher = hashlib.sha1() @@ -590,7 +583,6 @@ def create_dif_from_file( dif = find_existing_dif(project, meta, checksum) if dif is not None: - _backfill_shader_group(dif, meta) return dif, False object_name = _get_dif_object_name(meta) @@ -677,7 +669,6 @@ def create_dif_from_fileobj( dif = find_existing_dif(project, meta, checksum) if dif is not None: - _backfill_shader_group(dif, meta) return dif, False object_name = _get_dif_object_name(meta) diff --git a/tests/sentry/models/test_debugfile_shader.py b/tests/sentry/models/test_debugfile_shader.py index b7e1a7663960..5704339d682f 100644 --- a/tests/sentry/models/test_debugfile_shader.py +++ b/tests/sentry/models/test_debugfile_shader.py @@ -148,38 +148,6 @@ def test_create_dxil_stores_shader_group(default_project: Project, tmp_path: Pat assert dif.data["shader_group"] == "Group A" -@django_db_all -def test_reupload_backfills_missing_shader_group(default_project: Project, tmp_path: Path) -> None: - first, created = _create_shader_dif(default_project, tmp_path, "a.dxil", DXIL_BYTES, None) - assert created is True - assert "shader_group" not in first.data - - second, created = _create_shader_dif(default_project, tmp_path, "a.dxil", DXIL_BYTES, "Group A") - - assert created is False - assert second.id == first.id - first.refresh_from_db() - assert first.data == {"features": ["symtab"], "shader_group": "Group A"} - assert ProjectDebugFile.objects.filter(project_id=default_project.id).count() == 1 - - -@pytest.mark.parametrize("reupload_group", ("Group B", None)) -@django_db_all -def test_reupload_keeps_existing_shader_group( - default_project: Project, tmp_path: Path, reupload_group: str | None -) -> None: - first, _ = _create_shader_dif(default_project, tmp_path, "a.dxil", DXIL_BYTES, "Group A") - - second, created = _create_shader_dif( - default_project, tmp_path, "a.dxil", DXIL_BYTES, reupload_group - ) - - assert created is False - assert second.id == first.id - first.refresh_from_db() - assert first.data["shader_group"] == "Group A" - - @django_db_all def test_assemble_backfills_shader_group_for_known_checksum( default_project: Project, tmp_path: Path From 99d2c5738c8bc5987958cee65b0d419020a3ac50 Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Tue, 3 Nov 2026 14:50:32 +0200 Subject: [PATCH 13/26] Drop the UE filename sentinel handling for shader frames --- src/sentry/lang/native/gpu.py | 8 +------- tests/sentry/lang/native/test_gpu_shader_group.py | 13 ++----------- 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/src/sentry/lang/native/gpu.py b/src/sentry/lang/native/gpu.py index 35fea148ede7..49abf3538821 100644 --- a/src/sentry/lang/native/gpu.py +++ b/src/sentry/lang/native/gpu.py @@ -478,9 +478,6 @@ def _markers_to_breadcrumbs(markers: list[Any]) -> list[dict[str, Any]]: return out -UE_FILENAME_SENTINEL = "__UE_FILENAME_SENTINEL" - - def _normalize_gpu_frames(teapot_frames: list[Any]) -> list[dict[str, Any]]: normalized: list[dict[str, Any]] = [] for raw in teapot_frames: @@ -514,10 +511,7 @@ def _normalize_gpu_frames(teapot_frames: list[Any]) -> list[dict[str, Any]]: if not frame.get("module") and frame.get("package"): frame["module"] = frame["package"] group = raw_data.get("shader_group") - filename = frame.get("filename") - if isinstance(group, str) and group and filename in (None, "", UE_FILENAME_SENTINEL): - if filename: - frame.setdefault("abs_path", filename) + if isinstance(group, str) and group and not frame.get("filename"): frame["filename"] = group frame.setdefault("data", {}) diff --git a/tests/sentry/lang/native/test_gpu_shader_group.py b/tests/sentry/lang/native/test_gpu_shader_group.py index 1ac7c11e3470..eeb02ebe76a8 100644 --- a/tests/sentry/lang/native/test_gpu_shader_group.py +++ b/tests/sentry/lang/native/test_gpu_shader_group.py @@ -2,8 +2,6 @@ from typing import Any -import pytest - from sentry.lang.native.gpu import apply_gpu_crash_symbolication MATERIAL_GROUP = "M_GpuHang_992b2d801a2e9ddf/Default/FLocalVertexFactory/FLumenCardPS/0" @@ -70,18 +68,11 @@ def test_global_group_has_no_material() -> None: assert "material" not in data["contexts"]["gpu_crash"] -@pytest.mark.parametrize( - ("filename", "expected_abs_path"), - [(None, None), ("__UE_FILENAME_SENTINEL", "__UE_FILENAME_SENTINEL")], -) -def test_group_fills_the_filename_slot(filename: str | None, expected_abs_path: str | None) -> None: - frame = _frame(lineno=3477) if filename is None else _frame(filename=filename, lineno=3477) - data = _apply(frame) +def test_group_fills_the_empty_filename_slot() -> None: + data = _apply(_frame()) (applied,) = data["exception"]["values"][0]["stacktrace"]["frames"] assert applied["filename"] == MATERIAL_GROUP - assert applied.get("abs_path") == expected_abs_path - assert applied["lineno"] == 3477 def test_real_filename_is_kept() -> None: From b7dd604f2eab0eec75569471e220a8eada86b046 Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Tue, 3 Nov 2026 16:31:29 +0200 Subject: [PATCH 14/26] Keep the assemble task kwargs unchanged for files without shader fields --- src/sentry/api/endpoints/debug_files.py | 24 +++++++++---------- .../lang/native/test_gpu_shader_group.py | 3 ++- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/sentry/api/endpoints/debug_files.py b/src/sentry/api/endpoints/debug_files.py index 429d52c9db3a..0f1c17062970 100644 --- a/src/sentry/api/endpoints/debug_files.py +++ b/src/sentry/api/endpoints/debug_files.py @@ -5,7 +5,7 @@ from collections.abc import Iterable, Iterator, Mapping, Sequence, Set from contextlib import closing from datetime import datetime -from typing import TYPE_CHECKING, NotRequired, TypedDict, TypeGuard, cast +from typing import TYPE_CHECKING, Any, NotRequired, TypedDict, TypeGuard, cast import jsonschema import orjson @@ -795,17 +795,17 @@ def batch_assemble( set_assemble_status(AssembleTask.DIF, project.id, checksum, ChunkFileState.CREATED) name, debug_id, chunks = get_file_info(file) - assemble_dif.apply_async( - kwargs={ - "project_id": project.id, - "name": name, - "debug_id": debug_id, - "code_id": file.get("code_id"), - "shader_group": file.get("shader_group"), - "checksum": checksum, - "chunks": chunks, - } - ) + kwargs: dict[str, Any] = { + "project_id": project.id, + "name": name, + "debug_id": debug_id, + "checksum": checksum, + "chunks": chunks, + } + for key in ("code_id", "shader_group"): + if key in file: + kwargs[key] = file[key] + assemble_dif.apply_async(kwargs=kwargs) file_response[checksum] = {"state": ChunkFileState.CREATED, "missingChunks": []} diff --git a/tests/sentry/lang/native/test_gpu_shader_group.py b/tests/sentry/lang/native/test_gpu_shader_group.py index eeb02ebe76a8..597f9c81ce2d 100644 --- a/tests/sentry/lang/native/test_gpu_shader_group.py +++ b/tests/sentry/lang/native/test_gpu_shader_group.py @@ -1,5 +1,6 @@ from __future__ import annotations +from collections.abc import MutableMapping from typing import Any from sentry.lang.native.gpu import apply_gpu_crash_symbolication @@ -26,7 +27,7 @@ def _response(frame: dict[str, Any]) -> dict[str, Any]: } -def _apply(frame: dict[str, Any]) -> dict[str, Any]: +def _apply(frame: dict[str, Any]) -> MutableMapping[str, Any]: data: dict[str, Any] = {"event_id": "g" * 32, "platform": "native"} result = apply_gpu_crash_symbolication(data, _response(frame)) assert result is not None From 57d7064fb4c98e19f3328e36e64123e974e6f0a2 Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Tue, 3 Nov 2026 17:47:19 +0200 Subject: [PATCH 15/26] Type the second teapot round's attachments like the first call --- src/sentry/tasks/gpu_crash.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/sentry/tasks/gpu_crash.py b/src/sentry/tasks/gpu_crash.py index 8ddbde231de6..9032897afdea 100644 --- a/src/sentry/tasks/gpu_crash.py +++ b/src/sentry/tasks/gpu_crash.py @@ -143,14 +143,11 @@ def _try_symbolicate(data: MutableMapping[str, Any], project_id: int, event_id: metrics.incr("tasks.gpu_crash.skipped", tags={"reason": "project_missing"}) return False + teapot_dump = cast(TeapotAttachment, dump) + teapot_shaders = cast(Sequence[TeapotAttachment], shaders) try: with metrics.timer("tasks.gpu_crash.teapot"): - response = submit_to_teapot( - project, - event_id, - cast(TeapotAttachment, dump), - cast(Sequence[TeapotAttachment], shaders), - ) + response = submit_to_teapot(project, event_id, teapot_dump, teapot_shaders) except TeapotUnavailable: # A genuine outage — feed the breaker so repeated failures trip it. breaker.record_error() @@ -164,7 +161,9 @@ def _try_symbolicate(data: MutableMapping[str, Any], project_id: int, event_id: return False breaker.record_success() - response = _maybe_resolve_shader_difs(project, event_id, dump, shaders, response) + response = _maybe_resolve_shader_difs( + project, event_id, teapot_dump, teapot_shaders, response + ) applied = apply_gpu_crash_symbolication(data, response) metrics.incr( @@ -188,7 +187,7 @@ def _maybe_resolve_shader_difs( project: Project, event_id: str, dump: TeapotAttachment, - shaders: list[Any], + shaders: Sequence[TeapotAttachment], response: dict[str, Any], ) -> dict[str, Any]: """Second teapot round with presigned URLs for the shader DIFs it reported missing. From 36d5b740007c84933cc12958f61b9b10a5c15484 Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Wed, 4 Nov 2026 10:27:55 +0200 Subject: [PATCH 16/26] Truncate the shader group for the culprit and tag --- src/sentry/lang/native/gpu.py | 6 ++++-- tests/sentry/lang/native/test_gpu_shader_group.py | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/sentry/lang/native/gpu.py b/src/sentry/lang/native/gpu.py index 49abf3538821..f7efa5c76d70 100644 --- a/src/sentry/lang/native/gpu.py +++ b/src/sentry/lang/native/gpu.py @@ -8,7 +8,9 @@ from typing import Any from xml.sax.saxutils import unescape as _xml_unescape +from sentry.constants import MAX_CULPRIT_LENGTH, MAX_TAG_VALUE_LENGTH from sentry.utils import metrics +from sentry.utils.strings import truncatechars logger = logging.getLogger(__name__) @@ -127,11 +129,11 @@ def _tag(key: str, value: Any) -> None: _tag("gpu.shader_hash", primary_shader.get("shader_hash")) shader_group = _shader_group(response) material, shader_class = _shader_group_parts(shader_group) - _tag("gpu.shader_group", shader_group) + _tag("gpu.shader_group", truncatechars(shader_group, MAX_TAG_VALUE_LENGTH)) _tag("gpu.material", material) _tag("gpu.shader_class", shader_class) if shader_group: - data["culprit"] = shader_group + data["culprit"] = truncatechars(shader_group, MAX_CULPRIT_LENGTH) _tag("gpu.shader_type", primary_shader.get("shader_type")) _tag("gpu.device", gpu_state.get("device_name")) _tag("gpu.generation", _generation_name(gpu_state)) diff --git a/tests/sentry/lang/native/test_gpu_shader_group.py b/tests/sentry/lang/native/test_gpu_shader_group.py index 597f9c81ce2d..dbf3622ec9ea 100644 --- a/tests/sentry/lang/native/test_gpu_shader_group.py +++ b/tests/sentry/lang/native/test_gpu_shader_group.py @@ -3,6 +3,7 @@ from collections.abc import MutableMapping from typing import Any +from sentry.constants import MAX_CULPRIT_LENGTH, MAX_TAG_VALUE_LENGTH from sentry.lang.native.gpu import apply_gpu_crash_symbolication MATERIAL_GROUP = "M_GpuHang_992b2d801a2e9ddf/Default/FLocalVertexFactory/FLumenCardPS/0" @@ -59,6 +60,20 @@ def test_material_group_fills_tags_culprit_and_context() -> None: assert gpu_crash["shader_class"] == "FLumenCardPS" +def test_long_group_is_truncated_for_culprit_and_tag() -> None: + group = "M_" + "x" * 600 + "/Default/FLocalVertexFactory/FLumenCardPS/0" + data = _apply( + _frame(module=group, data={"shader_hash": "49478268ce7710e2", "shader_group": group}) + ) + + assert len(data["culprit"]) <= MAX_CULPRIT_LENGTH + assert data["culprit"].startswith("M_xxx") + tags = dict(data["tags"]) + assert len(tags["gpu.shader_group"]) <= MAX_TAG_VALUE_LENGTH + assert tags["gpu.shader_class"] == "FLumenCardPS" + assert data["contexts"]["gpu_crash"]["shader_group"] == group + + def test_global_group_has_no_material() -> None: data = _apply(_frame(module=GLOBAL_GROUP, data={"shader_group": GLOBAL_GROUP})) From bcd32c6b7321b2b496cfcf5b9e01c9ee531f6263 Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Wed, 4 Nov 2026 10:27:56 +0200 Subject: [PATCH 17/26] Store the shader group even when the assemble status is cached --- src/sentry/api/endpoints/debug_files.py | 14 ++++++--- tests/sentry/models/test_debugfile_shader.py | 30 +++++++++++++++++++- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/src/sentry/api/endpoints/debug_files.py b/src/sentry/api/endpoints/debug_files.py index 0f1c17062970..7889068c06aa 100644 --- a/src/sentry/api/endpoints/debug_files.py +++ b/src/sentry/api/endpoints/debug_files.py @@ -693,10 +693,16 @@ def batch_assemble( requested_debug_id = _get_requested_debug_id(file) cached_debug_id = detail.get("uuid") if isinstance(detail, Mapping) else None - if state == ChunkFileState.OK and not _is_proguard_reupload_clone_request( - file=file, - requested_debug_id=requested_debug_id, - selected_debug_id=cached_debug_id, + # A request carrying a shader group goes through the database lookup + # below so the group can be stored on an already assembled file. + if ( + state == ChunkFileState.OK + and not file.get("shader_group") + and not _is_proguard_reupload_clone_request( + file=file, + requested_debug_id=requested_debug_id, + selected_debug_id=cached_debug_id, + ) ): file_response[checksum] = { "state": state, diff --git a/tests/sentry/models/test_debugfile_shader.py b/tests/sentry/models/test_debugfile_shader.py index 5704339d682f..035f042e42fb 100644 --- a/tests/sentry/models/test_debugfile_shader.py +++ b/tests/sentry/models/test_debugfile_shader.py @@ -22,7 +22,7 @@ determine_dif_kind, ) from sentry.models.project import Project -from sentry.tasks.assemble import ChunkFileState +from sentry.tasks.assemble import AssembleTask, ChunkFileState, set_assemble_status from sentry.testutils.pytest.fixtures import django_db_all DXC_HASH = "731322bf682bdc554eefdd89098b7383" @@ -171,3 +171,31 @@ def test_assemble_backfills_shader_group_for_known_checksum( assert response[first.checksum]["state"] == ChunkFileState.OK first.refresh_from_db() assert first.data["shader_group"] == "Group A" + + +@django_db_all +def test_assemble_backfills_shader_group_despite_cached_ok_status( + default_project: Project, tmp_path: Path +) -> None: + first, _ = _create_shader_dif(default_project, tmp_path, "a.dxil", DXIL_BYTES, None) + assert first.checksum is not None + set_assemble_status( + AssembleTask.DIF, default_project.id, first.checksum, ChunkFileState.OK, detail=None + ) + + response = batch_assemble( + default_project, + { + first.checksum: { + "name": "a.dxil", + "debug_id": DEBUG_ID, + "chunks": [first.checksum], + "code_id": AFTERMATH_HASH, + "shader_group": "Group A", + } + }, + ) + + assert response[first.checksum]["state"] == ChunkFileState.OK + first.refresh_from_db() + assert first.data["shader_group"] == "Group A" From ecb75495a209c6e63acc8cd55b885e8e1e01d5e2 Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Fri, 6 Nov 2026 16:26:29 +0200 Subject: [PATCH 18/26] Let Teapot fetch shader binaries from the project's Sentry source --- src/sentry/api/endpoints/debug_files.py | 6 +- src/sentry/lang/native/shader_difs.py | 69 ---- src/sentry/lang/native/teapot.py | 28 +- src/sentry/tasks/gpu_crash.py | 63 +-- .../sentry/api/endpoints/test_debug_files.py | 38 ++ tests/sentry/lang/native/test_teapot.py | 17 + .../lang/native/test_teapot_shader_difs.py | 366 ------------------ tests/sentry/tasks/test_gpu_crash.py | 7 + 8 files changed, 85 insertions(+), 509 deletions(-) delete mode 100644 src/sentry/lang/native/shader_difs.py delete mode 100644 tests/sentry/lang/native/test_teapot_shader_difs.py diff --git a/src/sentry/api/endpoints/debug_files.py b/src/sentry/api/endpoints/debug_files.py index 7889068c06aa..2222c5e73f52 100644 --- a/src/sentry/api/endpoints/debug_files.py +++ b/src/sentry/api/endpoints/debug_files.py @@ -483,8 +483,10 @@ def get( def on_results( difs: Sequence[ProjectDebugFile], ) -> list[DebugFileSerializerResponse]: - # NOTE: we are only refreshing files if there is direct query for specific files - if debug_id and not query and not file_formats: + # NOTE: we are only refreshing files if there is direct query for specific files. + # Symbolicator looks files up by debug id, Teapot looks shader binaries up by + # code id; both keep the files they use from expiring. + if (debug_id or code_id) and not query and not file_formats: maybe_renew_debug_files(difs) serialized: list[DebugFileSerializerResponse] = serialize(difs, request.user) diff --git a/src/sentry/lang/native/shader_difs.py b/src/sentry/lang/native/shader_difs.py deleted file mode 100644 index af7a217dab45..000000000000 --- a/src/sentry/lang/native/shader_difs.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Resolve the shader binaries teapot reports missing to presigned URLs. - -``missing_difs`` names them by kind ``dxil`` and the 16-hex Aftermath shader hash -stored as ``code_id``. -""" - -from __future__ import annotations - -from collections.abc import Mapping -from datetime import timedelta -from typing import Any - -from sentry.debug_files.debug_files import maybe_renew_debug_files -from sentry.models.debugfile import ProjectDebugFile -from sentry.models.project import Project -from sentry.objectstore import get_internal_download_url -from sentry.utils import metrics - -SHADER_DIF_URL_VALIDITY = timedelta(minutes=15) - - -def missing_shader_difs(response: Mapping[str, Any]) -> list[str]: - """The dxil code ids in teapot's ``missing_difs``.""" - code_ids: list[str] = [] - for entry in response.get("missing_difs") or []: - if not isinstance(entry, dict) or entry.get("kind") != "dxil": - continue - debug_id = entry.get("debug_id") - if isinstance(debug_id, str) and debug_id: - code_ids.append(debug_id) - return code_ids - - -def _find_dxil(project_id: int, code_id: str) -> ProjectDebugFile | None: - """The newest objectstore-backed dxil debug file with ``code_id``.""" - difs = ProjectDebugFile.objects.filter( - project_id=project_id, code_id=code_id, storage_path__isnull=False - ).order_by("-id") - return next((dif for dif in difs if dif.file_format == "dxil"), None) - - -def resolve_shader_difs(project: Project, response: Mapping[str, Any]) -> list[dict[str, str]]: - """Entries for teapot's ``shader_binaries`` request array: every shader binary it - reported missing that we hold, with a presigned URL.""" - code_ids = missing_shader_difs(response) - resolved: list[dict[str, str]] = [] - found: list[ProjectDebugFile] = [] - - for code_id in dict.fromkeys(code_id.lower() for code_id in code_ids): - dxil = _find_dxil(project.id, code_id) - if dxil is None or not dxil.storage_path: - continue - found.append(dxil) - url = get_internal_download_url( - dxil.get_objectstore_session(), - dxil.storage_path, - token_validity=SHADER_DIF_URL_VALIDITY, - ) - entry = {"code_id": code_id, "storage_url": url} - if group := (dxil.data or {}).get("shader_group"): - entry["group"] = group - resolved.append(entry) - - if code_ids: - metrics.incr("tasks.gpu_crash.shader_difs.missing", amount=len(code_ids)) - if found: - maybe_renew_debug_files(found) - metrics.incr("tasks.gpu_crash.shader_difs.resolved", amount=len(found)) - return resolved diff --git a/src/sentry/lang/native/teapot.py b/src/sentry/lang/native/teapot.py index 918c1489119e..60211d55f058 100644 --- a/src/sentry/lang/native/teapot.py +++ b/src/sentry/lang/native/teapot.py @@ -13,7 +13,7 @@ import hmac import logging import re -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from typing import Any, Protocol import orjson @@ -139,7 +139,7 @@ def symbolicate( self, dump: TeapotAttachment, shader_debug_info: Sequence[TeapotAttachment] | None = None, - shader_binaries: Sequence[dict[str, str]] | None = None, + sources: Sequence[Mapping[str, Any]] | None = None, ) -> dict[str, Any]: shader_debug_info = shader_debug_info or [] url = f"{self.base_url}/symbolicate" @@ -158,17 +158,16 @@ def symbolicate( for att in shader_debug_info ], } - request_id = self.event_id - if shader_binaries: - # Second round: a distinct id, or teapot's idempotency cache replays the first response. - body["shader_binaries"] = list(shader_binaries) - request_id = f"{self.event_id}-2" + if sources: + # Where teapot looks up the shader binaries the dump references, as for + # Symbolicator. Carries the system token; never log the body. + body["sources"] = list(sources) data = orjson.dumps(body) headers = { "X-Teapot-Version": "1", - "X-Request-Id": request_id, + "X-Request-Id": self.event_id, "Content-Type": "application/json", - "Idempotency-Key": request_id, + "Idempotency-Key": self.event_id, **_auth_headers(data), } return self._send(url, headers=headers, data=data) @@ -232,7 +231,7 @@ def submit_to_teapot( event_id: str, dump: TeapotAttachment, shader_debug_info: Sequence[TeapotAttachment] | None = None, - shader_binaries: Sequence[dict[str, str]] | None = None, + sources: Sequence[Mapping[str, Any]] | None = None, ) -> dict[str, Any] | None: """Best-effort teapot invocation. @@ -241,17 +240,12 @@ def submit_to_teapot( trip the circuit breaker. Raises ``TeapotUnavailable`` on a genuine outage (network, timeout, 5xx) so the caller can. - ``shader_binaries`` makes this the second round, with presigned URLs for the - shader binaries teapot reported missing. + ``sources`` tells teapot where to fetch the shader binaries itself. """ try: client = TeapotClient(project=project, event_id=event_id) - return client.symbolicate( - dump, - shader_debug_info or [], - shader_binaries=shader_binaries, - ) + return client.symbolicate(dump, shader_debug_info or [], sources=sources) except TeapotRequestError as e: logger.info("teapot.request_error", extra={"event_id": event_id, "error": str(e)}) return None diff --git a/src/sentry/tasks/gpu_crash.py b/src/sentry/tasks/gpu_crash.py index 9032897afdea..f0b7c4d66cf8 100644 --- a/src/sentry/tasks/gpu_crash.py +++ b/src/sentry/tasks/gpu_crash.py @@ -19,7 +19,7 @@ from sentry.ingest.event_payload import load_event_payload from sentry.killswitches import killswitch_matches_context from sentry.lang.native.gpu import apply_gpu_crash_symbolication -from sentry.lang.native.shader_difs import resolve_shader_difs +from sentry.lang.native.sources import get_internal_source from sentry.lang.native.teapot import TeapotAttachment, TeapotUnavailable, submit_to_teapot from sentry.lang.native.utils import ( find_all_shader_debug_attachments, @@ -143,11 +143,15 @@ def _try_symbolicate(data: MutableMapping[str, Any], project_id: int, event_id: metrics.incr("tasks.gpu_crash.skipped", tags={"reason": "project_missing"}) return False - teapot_dump = cast(TeapotAttachment, dump) - teapot_shaders = cast(Sequence[TeapotAttachment], shaders) try: with metrics.timer("tasks.gpu_crash.teapot"): - response = submit_to_teapot(project, event_id, teapot_dump, teapot_shaders) + response = submit_to_teapot( + project, + event_id, + cast(TeapotAttachment, dump), + cast(Sequence[TeapotAttachment], shaders), + sources=[get_internal_source(project)], + ) except TeapotUnavailable: # A genuine outage — feed the breaker so repeated failures trip it. breaker.record_error() @@ -161,10 +165,6 @@ def _try_symbolicate(data: MutableMapping[str, Any], project_id: int, event_id: return False breaker.record_success() - response = _maybe_resolve_shader_difs( - project, event_id, teapot_dump, teapot_shaders, response - ) - applied = apply_gpu_crash_symbolication(data, response) metrics.incr( "tasks.gpu_crash.completed", @@ -181,50 +181,3 @@ def _try_symbolicate(data: MutableMapping[str, Any], project_id: int, event_id: ) sentry_sdk.capture_exception(e) return False - - -def _maybe_resolve_shader_difs( - project: Project, - event_id: str, - dump: TeapotAttachment, - shaders: Sequence[TeapotAttachment], - response: dict[str, Any], -) -> dict[str, Any]: - """Second teapot round with presigned URLs for the shader DIFs it reported missing. - Best-effort: any failure keeps the first response and does not touch the breaker.""" - if response.get("status") != "partial": - return response - - try: - difs = resolve_shader_difs(project, response) - if not difs: - metrics.incr("tasks.gpu_crash.shader_difs", tags={"result": "unresolved"}) - return response - with metrics.timer("tasks.gpu_crash.teapot_shader_difs"): - second = submit_to_teapot( - project, - event_id, - dump, - shaders, - shader_binaries=difs, - ) - except TeapotUnavailable: - metrics.incr("tasks.gpu_crash.shader_difs", tags={"result": "unavailable"}) - return response - except Exception as e: - metrics.incr("tasks.gpu_crash.shader_difs", tags={"result": "error"}) - logger.warning( - "tasks.gpu_crash.shader_difs_error", extra={"event_id": event_id, "error": repr(e)} - ) - sentry_sdk.capture_exception(e) - return response - - if second is None: - metrics.incr("tasks.gpu_crash.shader_difs", tags={"result": "request_error"}) - return response - - metrics.incr( - "tasks.gpu_crash.shader_difs", - tags={"result": second.get("status") or "unknown"}, - ) - return second diff --git a/tests/sentry/api/endpoints/test_debug_files.py b/tests/sentry/api/endpoints/test_debug_files.py index 17b73148449a..f28615bc06fc 100644 --- a/tests/sentry/api/endpoints/test_debug_files.py +++ b/tests/sentry/api/endpoints/test_debug_files.py @@ -1,5 +1,6 @@ import os import zipfile +from datetime import timedelta from io import BytesIO from unittest.mock import patch from uuid import uuid4 @@ -8,6 +9,7 @@ from django.core.files.uploadedfile import SimpleUploadedFile from django.http import HttpResponse from django.urls import reverse +from django.utils import timezone from rest_framework.response import Response from symbolic.debuginfo import Archive @@ -168,6 +170,42 @@ def test_dsyms_search(self) -> None: dsyms = response.data assert len(dsyms) == 20 + def test_dsyms_lookup_by_id_renews_files(self) -> None: + old = timezone.now() - timedelta(days=60) + by_debug_id = self.create_dif_file( + debug_id="dfb8e43a-f242-3d73-a453-aeb6a777ef75", code_id="ae0459704fc7256" + ) + by_code_id = self.create_dif_file( + debug_id="a223507a-29b5-0f71-32d4-4901c2228311", code_id="658794f3a75293d1" + ) + ProjectDebugFile.objects.filter(id__in=[by_debug_id.id, by_code_id.id]).update( + date_accessed=old + ) + + for query in ("debug_id=dfb8e43a-f242-3d73-a453-aeb6a777ef75", "code_id=658794f3a75293d1"): + response = self.client.get(f"{self.url}?{query}") + assert response.status_code == 200, response.content + assert len(response.data) == 1 + + by_debug_id.refresh_from_db() + by_code_id.refresh_from_db() + assert by_debug_id.date_accessed > old + assert by_code_id.date_accessed > old + + def test_dsyms_search_and_format_filter_do_not_renew(self) -> None: + old = timezone.now() - timedelta(days=60) + dif = self.create_dif_file( + debug_id="a223507a-29b5-0f71-32d4-4901c2228311", code_id="658794f3a75293d1" + ) + ProjectDebugFile.objects.filter(id=dif.id).update(date_accessed=old) + + for query in ("query=658794f3a75293d1", "code_id=658794f3a75293d1&file_formats=pe"): + response = self.client.get(f"{self.url}?{query}") + assert response.status_code == 200, response.content + + dif.refresh_from_db() + assert dif.date_accessed == old + def test_dsyms_debugid_codeid_full_match(self) -> None: self._do_test_dsyms_by_debugid_and_codeid( ("dfb8e43a-f242-3d73-a453-aeb6a777ef75", "ae0459704fc7256"), diff --git a/tests/sentry/lang/native/test_teapot.py b/tests/sentry/lang/native/test_teapot.py index bfa0327fec5d..2909d2ddb53f 100644 --- a/tests/sentry/lang/native/test_teapot.py +++ b/tests/sentry/lang/native/test_teapot.py @@ -540,6 +540,23 @@ def test_client_sends_presigned_urls(mock_objectstore: mock.Mock) -> None: } +def test_client_sends_sources_only_when_given(mock_objectstore: mock.Mock) -> None: + source = {"type": "sentry", "id": "sentry:project", "url": "http://s/dsyms/", "token": "t"} + with ( + _configured_teapot(), + mock.patch("sentry.lang.native.teapot.requests.post") as mock_post, + ): + mock_post.return_value = _FakeResponse(200, _completed_response()) + TeapotClient(_FakeProject(), "abc").symbolicate(_FakeAttachment(stored_id="dump")) + TeapotClient(_FakeProject(), "abc").symbolicate( + _FakeAttachment(stored_id="dump"), sources=[source] + ) + + without, with_sources = (orjson.loads(c.kwargs["data"]) for c in mock_post.call_args_list) + assert "sources" not in without + assert with_sources["sources"] == [source] + + def test_client_signs_request_when_secret_set(mock_objectstore: mock.Mock) -> None: from django.conf import settings diff --git a/tests/sentry/lang/native/test_teapot_shader_difs.py b/tests/sentry/lang/native/test_teapot_shader_difs.py deleted file mode 100644 index 7b0e690343d3..000000000000 --- a/tests/sentry/lang/native/test_teapot_shader_difs.py +++ /dev/null @@ -1,366 +0,0 @@ -"""Unit tests for the second (shader DIF) teapot round.""" - -from __future__ import annotations - -from collections.abc import Iterator -from typing import Any -from unittest import mock - -import orjson -import pytest -from django.conf import settings -from django.utils import timezone - -from sentry.lang.native.shader_difs import missing_shader_difs, resolve_shader_difs -from sentry.lang.native.teapot import TeapotClient, TeapotUnavailable -from sentry.models.debugfile import DIF_MIMETYPES, ProjectDebugFile -from sentry.models.project import Project -from sentry.tasks.gpu_crash import _try_symbolicate -from sentry.testutils.helpers.options import override_options -from sentry.testutils.pytest.fixtures import django_db_all - -DEBUG_ID = "731322bf-682b-dc55-4eef-dd89098b7383" -AFTERMATH_HASH = "0123456789abcdef" -SHADER_GROUP = "GlobalShaders" - -FIND_DUMP = "sentry.tasks.gpu_crash.find_gpu_crash_dump_attachment" -FIND_SHADERS = "sentry.tasks.gpu_crash.find_all_shader_debug_attachments" -SUBMIT_TEAPOT = "sentry.tasks.gpu_crash.submit_to_teapot" -APPLY = "sentry.tasks.gpu_crash.apply_gpu_crash_symbolication" -RESOLVE = "sentry.tasks.gpu_crash.resolve_shader_difs" - - -class _FakeProject: - def __init__(self, id: int = 42, organization_id: int = 7) -> None: - self.id = id - self.organization_id = organization_id - - -class _FakeAttachment: - def __init__(self, name: str = "dump.nv-gpudmp", stored_id: str | None = "dump-obj-id") -> None: - self.name = name - self.stored_id = stored_id - - -class _FakeResponse: - def __init__(self, status_code: int, body: Any = None) -> None: - self.status_code = status_code - self._body = body if body is not None else {} - self.text = "" - - def json(self) -> Any: - return self._body - - -def _partial(**overrides: Any) -> dict[str, Any]: - base: dict[str, Any] = { - "status": "partial", - "fault_category": "shader_hang", - "missing_difs": [{"debug_id": AFTERMATH_HASH, "kind": "dxil"}], - } - base.update(overrides) - return base - - -def _completed() -> dict[str, Any]: - return {"status": "completed", "fault_category": "shader_hang", "missing_difs": []} - - -def _resolved() -> list[dict[str, str]]: - return [ - { - "code_id": AFTERMATH_HASH, - "storage_url": "http://objectstore/dxil?sig=x", - "group": SHADER_GROUP, - } - ] - - -@pytest.fixture -def teapot_session() -> Iterator[mock.Mock]: - session = mock.Mock() - get_url = mock.Mock(side_effect=lambda _session, key: f"http://objectstore/{key}?sig=abc") - with ( - mock.patch.object(settings, "SENTRY_TEAPOT_URL", "http://teapot.test", create=True), - mock.patch("sentry.lang.native.teapot.get_session", return_value=session), - mock.patch("sentry.lang.native.teapot.get_internal_download_url", get_url), - ): - yield session - - -# --------------------------------------------------------------------------- -# TeapotClient — second round wire format -# --------------------------------------------------------------------------- - - -def test_client_second_round_sends_shader_binaries(teapot_session: mock.Mock) -> None: - resolved = _resolved() - with mock.patch("sentry.lang.native.teapot.requests.post") as mock_post: - mock_post.return_value = _FakeResponse(200, _completed()) - TeapotClient(_FakeProject(), "abc").symbolicate( - _FakeAttachment(), [], shader_binaries=resolved - ) - - kwargs = mock_post.call_args.kwargs - assert kwargs["headers"]["Idempotency-Key"] == "abc-2" - assert kwargs["headers"]["X-Request-Id"] == "abc-2" - - body = orjson.loads(kwargs["data"]) - assert body["event_id"] == "abc" - assert body["dump"] == {"storage_url": "http://objectstore/dump-obj-id?sig=abc"} - assert body["shader_binaries"] == resolved - - -def test_client_first_round_omits_shader_binaries(teapot_session: mock.Mock) -> None: - with mock.patch("sentry.lang.native.teapot.requests.post") as mock_post: - mock_post.return_value = _FakeResponse(200, _completed()) - TeapotClient(_FakeProject(), "abc").symbolicate(_FakeAttachment(), []) - - kwargs = mock_post.call_args.kwargs - assert kwargs["headers"]["Idempotency-Key"] == "abc" - assert kwargs["headers"]["X-Request-Id"] == "abc" - body = orjson.loads(kwargs["data"]) - assert "shader_binaries" not in body - - -# --------------------------------------------------------------------------- -# shader_difs — resolving missing_difs against ProjectDebugFile -# --------------------------------------------------------------------------- - - -def test_missing_shader_difs_parsing() -> None: - response = _partial( - missing_difs=[ - {"debug_id": AFTERMATH_HASH, "kind": "dxil"}, - {"debug_id": "deadbeef", "kind": "unknown"}, - {"kind": "dxil"}, - "garbage", - ] - ) - assert missing_shader_difs(response) == [AFTERMATH_HASH] - assert missing_shader_difs({}) == [] - - -def _create_dif( - project: Project, - dif_type: str = "dxil", - code_id: str | None = None, - storage_path: str | None = "difs/shader", - shader_group: str | None = None, -) -> ProjectDebugFile: - data: dict[str, Any] = {"features": ["symtab"]} - if shader_group is not None: - data["shader_group"] = shader_group - return ProjectDebugFile.objects.create( - project_id=project.id, - debug_id=DEBUG_ID, - code_id=code_id, - object_name="shader", - cpu_name="any", - checksum="0" * 40, - data=data, - storage_path=storage_path, - content_type=DIF_MIMETYPES[dif_type], - file_size=64, - date_created=timezone.now(), - ) - - -@pytest.fixture -def dif_presign() -> Iterator[mock.Mock]: - get_url = mock.Mock( - side_effect=lambda _session, key, token_validity=None: f"http://objectstore/{key}?sig=x" - ) - with ( - mock.patch.object(ProjectDebugFile, "get_objectstore_session", return_value=mock.Mock()), - mock.patch("sentry.lang.native.shader_difs.get_internal_download_url", get_url), - ): - yield get_url - - -@django_db_all -def test_resolve_shader_difs_presigns_objectstore_rows( - default_project: Project, dif_presign: mock.Mock -) -> None: - _create_dif( - default_project, - code_id=AFTERMATH_HASH, - storage_path="difs/dxil", - shader_group=SHADER_GROUP, - ) - - resolved = resolve_shader_difs(default_project, _partial()) - - assert bool(resolved) is True - assert resolved == [ - { - "code_id": AFTERMATH_HASH, - "storage_url": "http://objectstore/difs/dxil?sig=x", - "group": SHADER_GROUP, - } - ] - # Token validity is the longer DIF one, not the attachment default. - assert dif_presign.call_args.kwargs["token_validity"] is not None - - -@django_db_all -def test_resolve_shader_difs_matches_code_id_case_insensitively( - default_project: Project, dif_presign: mock.Mock -) -> None: - _create_dif(default_project, code_id=AFTERMATH_HASH) - response = _partial(missing_difs=[{"debug_id": AFTERMATH_HASH.upper(), "kind": "dxil"}]) - - resolved = resolve_shader_difs(default_project, response) - - assert resolved == [ - {"code_id": AFTERMATH_HASH, "storage_url": "http://objectstore/difs/shader?sig=x"} - ] - - -@django_db_all -def test_resolve_shader_difs_ignores_other_types( - default_project: Project, dif_presign: mock.Mock -) -> None: - # Same code_id, but the row is not a shader binary. - _create_dif(default_project, dif_type="pe", code_id=AFTERMATH_HASH) - - resolved = resolve_shader_difs(default_project, _partial()) - - assert bool(resolved) is False - dif_presign.assert_not_called() - - -@django_db_all -def test_resolve_shader_difs_skips_legacy_rows( - default_project: Project, dif_presign: mock.Mock -) -> None: - _create_dif(default_project, code_id=AFTERMATH_HASH, storage_path=None) - - resolved = resolve_shader_difs(default_project, _partial()) - - assert bool(resolved) is False - dif_presign.assert_not_called() - - -@django_db_all -def test_resolve_shader_difs_nothing_missing(default_project: Project) -> None: - assert bool(resolve_shader_difs(default_project, _completed())) is False - - -# --------------------------------------------------------------------------- -# Task — second round orchestration -# --------------------------------------------------------------------------- - - -@pytest.fixture(autouse=True) -def circuit_breaker() -> Iterator[mock.Mock]: - breaker = mock.Mock() - breaker.should_allow_request.return_value = True - with mock.patch("sentry.tasks.gpu_crash._teapot_circuit_breaker", return_value=breaker): - yield breaker - - -@django_db_all -def test_task_runs_second_round_with_resolved_difs( - default_project: Project, circuit_breaker: mock.Mock -) -> None: - data: dict[str, Any] = {"event_id": "e", "project": default_project.id} - partial, completed = _partial(), _completed() - with ( - override_options({"teapot.enabled": True}), - mock.patch(FIND_DUMP, return_value=_FakeAttachment()), - mock.patch(FIND_SHADERS, return_value=[]), - mock.patch(RESOLVE, return_value=_resolved()) as resolve, - mock.patch(SUBMIT_TEAPOT, side_effect=[partial, completed]) as teapot, - mock.patch(APPLY, return_value=data) as apply, - ): - changed = _try_symbolicate(data, default_project.id, "e") - - assert changed is True - assert resolve.call_args.args[1] is partial - assert teapot.call_count == 2 - second = teapot.call_args_list[1] - assert second.kwargs == {"shader_binaries": _resolved()} - assert apply.call_args.args[1] is completed - circuit_breaker.record_success.assert_called_once() - circuit_breaker.record_error.assert_not_called() - - -@django_db_all -def test_task_keeps_first_response_when_nothing_resolved(default_project: Project) -> None: - data: dict[str, Any] = {"event_id": "e", "project": default_project.id} - partial = _partial() - with ( - override_options({"teapot.enabled": True}), - mock.patch(FIND_DUMP, return_value=_FakeAttachment()), - mock.patch(FIND_SHADERS, return_value=[]), - mock.patch(RESOLVE, return_value=[]), - mock.patch(SUBMIT_TEAPOT, return_value=partial) as teapot, - mock.patch(APPLY, return_value=data) as apply, - ): - _try_symbolicate(data, default_project.id, "e") - - assert teapot.call_count == 1 - assert apply.call_args.args[1] is partial - - -@django_db_all -def test_task_skips_second_round_when_completed(default_project: Project) -> None: - data: dict[str, Any] = {"event_id": "e", "project": default_project.id} - with ( - override_options({"teapot.enabled": True}), - mock.patch(FIND_DUMP, return_value=_FakeAttachment()), - mock.patch(FIND_SHADERS, return_value=[]), - mock.patch(RESOLVE) as resolve, - mock.patch(SUBMIT_TEAPOT, return_value=_completed()) as teapot, - mock.patch(APPLY, return_value=data), - ): - _try_symbolicate(data, default_project.id, "e") - - assert teapot.call_count == 1 - resolve.assert_not_called() - - -@pytest.mark.parametrize("second_outcome", (TeapotUnavailable("down"), None, RuntimeError("x"))) -@django_db_all -def test_task_second_round_failure_falls_back_to_first_response( - default_project: Project, circuit_breaker: mock.Mock, second_outcome: Any -) -> None: - data: dict[str, Any] = {"event_id": "e", "project": default_project.id} - partial = _partial() - with ( - override_options({"teapot.enabled": True}), - mock.patch(FIND_DUMP, return_value=_FakeAttachment()), - mock.patch(FIND_SHADERS, return_value=[]), - mock.patch(RESOLVE, return_value=_resolved()), - mock.patch(SUBMIT_TEAPOT, side_effect=[partial, second_outcome]) as teapot, - mock.patch(APPLY, return_value=data) as apply, - mock.patch("sentry.tasks.gpu_crash.sentry_sdk.capture_exception"), - ): - changed = _try_symbolicate(data, default_project.id, "e") - - assert changed is True - assert teapot.call_count == 2 - assert apply.call_args.args[1] is partial - circuit_breaker.record_success.assert_called_once() - circuit_breaker.record_error.assert_not_called() - - -@django_db_all -def test_task_resolver_error_falls_back_to_first_response(default_project: Project) -> None: - data: dict[str, Any] = {"event_id": "e", "project": default_project.id} - partial = _partial() - with ( - override_options({"teapot.enabled": True}), - mock.patch(FIND_DUMP, return_value=_FakeAttachment()), - mock.patch(FIND_SHADERS, return_value=[]), - mock.patch(RESOLVE, side_effect=RuntimeError("db down")), - mock.patch(SUBMIT_TEAPOT, return_value=partial) as teapot, - mock.patch(APPLY, return_value=data) as apply, - mock.patch("sentry.tasks.gpu_crash.sentry_sdk.capture_exception"), - ): - changed = _try_symbolicate(data, default_project.id, "e") - - assert changed is True - assert teapot.call_count == 1 - assert apply.call_args.args[1] is partial diff --git a/tests/sentry/tasks/test_gpu_crash.py b/tests/sentry/tasks/test_gpu_crash.py index d460179b79ce..bd020ce1c937 100644 --- a/tests/sentry/tasks/test_gpu_crash.py +++ b/tests/sentry/tasks/test_gpu_crash.py @@ -59,6 +59,13 @@ def test_try_symbolicate_happy_path(default_project: Project, circuit_breaker: m assert changed is True assert teapot.call_count == 1 assert apply.call_count == 1 + (source,) = teapot.call_args.kwargs["sources"] + assert source["type"] == "sentry" + assert ( + f"/projects/{default_project.organization.slug}/{default_project.slug}/files/dsyms/" + in (source["url"]) + ) + assert source["token"] # A successful decode closes the loop on the breaker. circuit_breaker.record_success.assert_called_once() From 249a41ba5e949f6f60f2c5eb2dec1b23614bd8fc Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Fri, 6 Nov 2026 17:33:58 +0200 Subject: [PATCH 19/26] Trim comments and name the DXIL header --- src/sentry/api/endpoints/debug_files.py | 4 +--- src/sentry/lang/native/teapot.py | 3 +-- src/sentry/models/debugfile.py | 4 ++-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/src/sentry/api/endpoints/debug_files.py b/src/sentry/api/endpoints/debug_files.py index 2222c5e73f52..9e00be3f9916 100644 --- a/src/sentry/api/endpoints/debug_files.py +++ b/src/sentry/api/endpoints/debug_files.py @@ -483,9 +483,7 @@ def get( def on_results( difs: Sequence[ProjectDebugFile], ) -> list[DebugFileSerializerResponse]: - # NOTE: we are only refreshing files if there is direct query for specific files. - # Symbolicator looks files up by debug id, Teapot looks shader binaries up by - # code id; both keep the files they use from expiring. + # NOTE: we are only refreshing files if there is direct query for specific files if (debug_id or code_id) and not query and not file_formats: maybe_renew_debug_files(difs) diff --git a/src/sentry/lang/native/teapot.py b/src/sentry/lang/native/teapot.py index 60211d55f058..92be52484949 100644 --- a/src/sentry/lang/native/teapot.py +++ b/src/sentry/lang/native/teapot.py @@ -159,8 +159,7 @@ def symbolicate( ], } if sources: - # Where teapot looks up the shader binaries the dump references, as for - # Symbolicator. Carries the system token; never log the body. + # Carries the system token; never log the body. body["sources"] = list(sources) data = orjson.dumps(body) headers = { diff --git a/src/sentry/models/debugfile.py b/src/sentry/models/debugfile.py index 0de2e3569779..04dab16712ff 100644 --- a/src/sentry/models/debugfile.py +++ b/src/sentry/models/debugfile.py @@ -69,7 +69,7 @@ _proguard_file_re = re.compile(r"/proguard/(?:mapping-)?(.*?)\.txt$") -_DXIL_MAGIC = b"DXBC" +_DXIL_HEADER = b"DXBC" _SHADER_CODE_ID_RE = re.compile(r"^[0-9a-f]{16}$") SHADER_GROUP_MAX_LENGTH = 512 @@ -910,7 +910,7 @@ def determine_dif_kind(path: str) -> DifKind: return DifKind.Il2Cpp elif data.startswith(b"["): return DifKind.DartSymbolMap - elif data.startswith(_DXIL_MAGIC): + elif data.startswith(_DXIL_HEADER): return DifKind.Dxil else: return DifKind.Object From ea989913ee2f7a4a47eae3ec596a9d7ddbadfc68 Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Mon, 9 Nov 2026 09:28:19 +0200 Subject: [PATCH 20/26] Reuse the cached assemble result when the shader group is already stored --- src/sentry/api/endpoints/debug_files.py | 13 ++++++-- tests/sentry/models/test_debugfile_shader.py | 34 ++++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/src/sentry/api/endpoints/debug_files.py b/src/sentry/api/endpoints/debug_files.py index be2768ccf971..4a0ee9e87f33 100644 --- a/src/sentry/api/endpoints/debug_files.py +++ b/src/sentry/api/endpoints/debug_files.py @@ -687,6 +687,13 @@ def get_file_info(file: AssembleRequestFile) -> tuple[str, str | None, list[str] return name, debug_id, chunks +def _cached_dif_lacks_shader_group(file: AssembleRequestFile, detail: object) -> bool: + if not file.get("shader_group"): + return False + data = detail.get("data") if isinstance(detail, Mapping) else None + return not (isinstance(data, Mapping) and data.get("shader_group")) + + def batch_assemble( project: Project, files: AssembleRequestPayload ) -> dict[str, DifAssembleFileResponse]: @@ -705,11 +712,11 @@ def batch_assemble( requested_debug_id = _get_requested_debug_id(file) cached_debug_id = detail.get("uuid") if isinstance(detail, Mapping) else None - # A request carrying a shader group goes through the database lookup - # below so the group can be stored on an already assembled file. + # A shader group the cached file lacks goes through the database lookup + # below, so it can be stored on the already assembled file. if ( state == ChunkFileState.OK - and not file.get("shader_group") + and not _cached_dif_lacks_shader_group(file, detail) and not _is_proguard_reupload_clone_request( file=file, requested_debug_id=requested_debug_id, diff --git a/tests/sentry/models/test_debugfile_shader.py b/tests/sentry/models/test_debugfile_shader.py index 035f042e42fb..703368fd9493 100644 --- a/tests/sentry/models/test_debugfile_shader.py +++ b/tests/sentry/models/test_debugfile_shader.py @@ -6,7 +6,9 @@ import pytest +from sentry.api.endpoints import debug_files from sentry.api.endpoints.debug_files import batch_assemble +from sentry.api.serializers import serialize from sentry.constants import KNOWN_DIF_FORMATS from sentry.models.debugfile import ( DIF_MIMETYPES, @@ -199,3 +201,35 @@ def test_assemble_backfills_shader_group_despite_cached_ok_status( assert response[first.checksum]["state"] == ChunkFileState.OK first.refresh_from_db() assert first.data["shader_group"] == "Group A" + + +@django_db_all +def test_assemble_reuses_cached_ok_status_when_group_is_stored( + default_project: Project, tmp_path: Path +) -> None: + first, _ = _create_shader_dif(default_project, tmp_path, "a.dxil", DXIL_BYTES, "Group A") + assert first.checksum is not None + set_assemble_status( + AssembleTask.DIF, default_project.id, first.checksum, ChunkFileState.OK, serialize(first) + ) + + with mock.patch.object( + debug_files, + "_find_existing_debug_files", + wraps=debug_files._find_existing_debug_files, + ) as find_existing: + response = batch_assemble( + default_project, + { + first.checksum: { + "name": "a.dxil", + "debug_id": DEBUG_ID, + "chunks": [first.checksum], + "code_id": AFTERMATH_HASH, + "shader_group": "Group A", + } + }, + ) + + assert response[first.checksum]["state"] == ChunkFileState.OK + assert list(find_existing.call_args.kwargs["checksums"]) == [] From 25b322abecb64cd653e3c6028bd0704e1a80628e Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Mon, 9 Nov 2026 11:52:56 +0200 Subject: [PATCH 21/26] Cap the material and shader class tags and redact source tokens from Teapot errors --- src/sentry/lang/native/gpu.py | 4 ++-- src/sentry/lang/native/teapot.py | 20 +++++++++++++--- .../lang/native/test_gpu_shader_group.py | 4 +++- tests/sentry/lang/native/test_teapot.py | 24 +++++++++++++++++++ 4 files changed, 46 insertions(+), 6 deletions(-) diff --git a/src/sentry/lang/native/gpu.py b/src/sentry/lang/native/gpu.py index f7efa5c76d70..d8b46d57a306 100644 --- a/src/sentry/lang/native/gpu.py +++ b/src/sentry/lang/native/gpu.py @@ -130,8 +130,8 @@ def _tag(key: str, value: Any) -> None: shader_group = _shader_group(response) material, shader_class = _shader_group_parts(shader_group) _tag("gpu.shader_group", truncatechars(shader_group, MAX_TAG_VALUE_LENGTH)) - _tag("gpu.material", material) - _tag("gpu.shader_class", shader_class) + _tag("gpu.material", truncatechars(material, MAX_TAG_VALUE_LENGTH)) + _tag("gpu.shader_class", truncatechars(shader_class, MAX_TAG_VALUE_LENGTH)) if shader_group: data["culprit"] = truncatechars(shader_group, MAX_CULPRIT_LENGTH) _tag("gpu.shader_type", primary_shader.get("shader_type")) diff --git a/src/sentry/lang/native/teapot.py b/src/sentry/lang/native/teapot.py index 92be52484949..09d848c01188 100644 --- a/src/sentry/lang/native/teapot.py +++ b/src/sentry/lang/native/teapot.py @@ -109,6 +109,13 @@ def _redact_urls(text: str) -> str: return _URL_QUERY_RE.sub(r"\1?", text) +def _redact(text: str, secrets: Sequence[str]) -> str: + """Presigned URL tokens and the given secrets (the sources' tokens) removed.""" + for secret in secrets: + text = text.replace(secret, "") + return _redact_urls(text) + + def _resolve_url() -> str | None: base = getattr(settings, "SENTRY_TEAPOT_URL", None) return base.rstrip("/") if base else None @@ -158,9 +165,13 @@ def symbolicate( for att in shader_debug_info ], } + secrets: list[str] = [] if sources: # Carries the system token; never log the body. body["sources"] = list(sources) + secrets = [ + s["token"] for s in sources if isinstance(s.get("token"), str) and s["token"] + ] data = orjson.dumps(body) headers = { "X-Teapot-Version": "1", @@ -169,7 +180,7 @@ def symbolicate( "Idempotency-Key": self.event_id, **_auth_headers(data), } - return self._send(url, headers=headers, data=data) + return self._send(url, headers=headers, data=data, secrets=secrets) def _storage_url(self, session: Any, att: TeapotAttachment) -> str: """Short-lived self-authenticating (presigned) GET URL for the attachment — @@ -178,7 +189,9 @@ def _storage_url(self, session: Any, att: TeapotAttachment) -> str: raise TeapotRequestError(f"attachment {att.name!r} is not in objectstore") return get_internal_download_url(session, att.stored_id) - def _send(self, url: str, headers: dict[str, str], data: bytes) -> dict[str, Any]: + def _send( + self, url: str, headers: dict[str, str], data: bytes, secrets: Sequence[str] = () + ) -> dict[str, Any]: def attempt() -> dict[str, Any]: try: resp = requests.post(url, data=data, headers=headers, timeout=_timeout()) @@ -189,7 +202,8 @@ def attempt() -> dict[str, Any]: raise _RetryableTeapotError(f"teapot returned {resp.status_code}") if resp.status_code >= 400: - safe_body = _redact_urls(resp.text[:512]) + # Redact before truncating, so a cut can't leave part of a secret. + safe_body = _redact(resp.text, secrets)[:512] logger.warning( "teapot.request_failed", extra={ diff --git a/tests/sentry/lang/native/test_gpu_shader_group.py b/tests/sentry/lang/native/test_gpu_shader_group.py index dbf3622ec9ea..ccee1425e968 100644 --- a/tests/sentry/lang/native/test_gpu_shader_group.py +++ b/tests/sentry/lang/native/test_gpu_shader_group.py @@ -60,7 +60,7 @@ def test_material_group_fills_tags_culprit_and_context() -> None: assert gpu_crash["shader_class"] == "FLumenCardPS" -def test_long_group_is_truncated_for_culprit_and_tag() -> None: +def test_long_group_is_truncated_for_culprit_and_tags() -> None: group = "M_" + "x" * 600 + "/Default/FLocalVertexFactory/FLumenCardPS/0" data = _apply( _frame(module=group, data={"shader_hash": "49478268ce7710e2", "shader_group": group}) @@ -70,6 +70,8 @@ def test_long_group_is_truncated_for_culprit_and_tag() -> None: assert data["culprit"].startswith("M_xxx") tags = dict(data["tags"]) assert len(tags["gpu.shader_group"]) <= MAX_TAG_VALUE_LENGTH + assert len(tags["gpu.material"]) <= MAX_TAG_VALUE_LENGTH + assert tags["gpu.material"].startswith("M_xxx") assert tags["gpu.shader_class"] == "FLumenCardPS" assert data["contexts"]["gpu_crash"]["shader_group"] == group diff --git a/tests/sentry/lang/native/test_teapot.py b/tests/sentry/lang/native/test_teapot.py index 2909d2ddb53f..5d71acbe93be 100644 --- a/tests/sentry/lang/native/test_teapot.py +++ b/tests/sentry/lang/native/test_teapot.py @@ -7,6 +7,7 @@ import contextlib import hashlib import hmac +import logging from collections.abc import Iterator from typing import Any from unittest import mock @@ -557,6 +558,29 @@ def test_client_sends_sources_only_when_given(mock_objectstore: mock.Mock) -> No assert with_sources["sources"] == [source] +def test_client_redacts_source_tokens_from_errors( + mock_objectstore: mock.Mock, caplog: pytest.LogCaptureFixture +) -> None: + token = "system-token-0123456789abcdef0123456789abcdef" + source = {"type": "sentry", "id": "sentry:project", "url": "http://s/dsyms/", "token": token} + echoed = f'{{"status": "failed", "error": {{"message": "bad JSON body; last read: {token}"}}}}' + with ( + _configured_teapot(), + mock.patch("sentry.lang.native.teapot.requests.post") as mock_post, + caplog.at_level(logging.WARNING, logger="sentry.lang.native.teapot"), + ): + mock_post.return_value = _FakeResponse(400, echoed) + with pytest.raises(TeapotRequestError) as exc: + TeapotClient(_FakeProject(), "abc").symbolicate( + _FakeAttachment(stored_id="dump"), sources=[source] + ) + + assert token not in str(exc.value) + assert "" in str(exc.value) + (record,) = [r for r in caplog.records if r.getMessage() == "teapot.request_failed"] + assert token not in str(record.__dict__) + + def test_client_signs_request_when_secret_set(mock_objectstore: mock.Mock) -> None: from django.conf import settings From b212d0effdd570b85d71b82b205bae16b6a7627b Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Tue, 10 Nov 2026 16:23:38 +0200 Subject: [PATCH 22/26] Give Teapot a decode budget below the request timeout Teapot looks up shader binaries in Sentry after the decode. Send max_decode_ms two seconds below the request timeout, so Teapot answers with what it has before the client gives up. --- src/sentry/lang/native/teapot.py | 10 ++++++++++ tests/sentry/lang/native/test_teapot.py | 17 +++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/src/sentry/lang/native/teapot.py b/src/sentry/lang/native/teapot.py index 09d848c01188..c561b30cc55d 100644 --- a/src/sentry/lang/native/teapot.py +++ b/src/sentry/lang/native/teapot.py @@ -61,6 +61,11 @@ class TeapotAttachment(Protocol): DEFAULT_TIMEOUT = 5 DEFAULT_MAX_ATTEMPTS = 2 +# Teapot's budget ends before the timeout, so it answers with what it has +# instead of being cut off. `MIN_DECODE_BUDGET_MS` is teapot's lower limit. +DECODE_BUDGET_MARGIN_MS = 2000 +MIN_DECODE_BUDGET_MS = 1000 + RETRYABLE_STATUS = (502, 503, 504) # Bounded exponential backoff between transient retries. @@ -75,6 +80,10 @@ def _timeout() -> int: return DEFAULT_TIMEOUT +def _max_decode_ms() -> int: + return max(MIN_DECODE_BUDGET_MS, _timeout() * 1000 - DECODE_BUDGET_MARGIN_MS) + + def _max_attempts() -> int: try: return max(1, int(options.get("teapot.max-attempts"))) @@ -164,6 +173,7 @@ def symbolicate( } for att in shader_debug_info ], + "options": {"max_decode_ms": _max_decode_ms()}, } secrets: list[str] = [] if sources: diff --git a/tests/sentry/lang/native/test_teapot.py b/tests/sentry/lang/native/test_teapot.py index 5d71acbe93be..8cf6da6d0cf3 100644 --- a/tests/sentry/lang/native/test_teapot.py +++ b/tests/sentry/lang/native/test_teapot.py @@ -558,6 +558,23 @@ def test_client_sends_sources_only_when_given(mock_objectstore: mock.Mock) -> No assert with_sources["sources"] == [source] +@pytest.mark.parametrize(("timeout", "max_decode_ms"), [(5, 3000), (2, 1000)]) +def test_client_gives_teapot_a_budget_below_its_timeout( + mock_objectstore: mock.Mock, timeout: int, max_decode_ms: int +) -> None: + with ( + _configured_teapot(), + mock.patch("sentry.lang.native.teapot._timeout", return_value=timeout), + mock.patch("sentry.lang.native.teapot.requests.post") as mock_post, + ): + mock_post.return_value = _FakeResponse(200, _completed_response()) + TeapotClient(_FakeProject(), "abc").symbolicate(_FakeAttachment(stored_id="dump")) + + kwargs = mock_post.call_args.kwargs + assert kwargs["timeout"] == timeout + assert orjson.loads(kwargs["data"])["options"] == {"max_decode_ms": max_decode_ms} + + def test_client_redacts_source_tokens_from_errors( mock_objectstore: mock.Mock, caplog: pytest.LogCaptureFixture ) -> None: From 1560f934b2191e005ea68f99cc80d0f183a47ab3 Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Fri, 13 Nov 2026 16:57:13 +0200 Subject: [PATCH 23/26] Give Teapot a 15 s timeout and a 13 s decode budget A page fault dump can take a few seconds to decode, more than the 3 s budget that the 5 s timeout left, and a decode over the budget gets a 504 that is retried and counts toward the circuit breaker. Two attempts of 15 s still fit the task's 120 s deadline. The budget comment now says that it stops the shader binary lookups early. --- src/sentry/lang/native/teapot.py | 13 ++++++++----- src/sentry/options/defaults.py | 5 +++-- tests/sentry/lang/native/test_teapot.py | 2 +- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/src/sentry/lang/native/teapot.py b/src/sentry/lang/native/teapot.py index c561b30cc55d..bd53ffd3faf0 100644 --- a/src/sentry/lang/native/teapot.py +++ b/src/sentry/lang/native/teapot.py @@ -56,13 +56,16 @@ class TeapotAttachment(Protocol): # Fallbacks for the `teapot.timeout-seconds` / `teapot.max-attempts` options. -# The timeout is tight on purpose: decode is sub-second, so a slow teapot should -# fail fast (the circuit breaker then takes over) rather than hold a worker. -DEFAULT_TIMEOUT = 5 +# Most decodes take milliseconds, but a page fault dump can take a few seconds, +# and the request also covers teapot's shader binary lookups. A teapot that is +# down still fails well inside the task's deadline, and the breaker takes over. +DEFAULT_TIMEOUT = 15 DEFAULT_MAX_ATTEMPTS = 2 -# Teapot's budget ends before the timeout, so it answers with what it has -# instead of being cut off. `MIN_DECODE_BUDGET_MS` is teapot's lower limit. +# Teapot's budget for the decode and the shader binary lookups ends before the +# timeout: it stops the lookups early and answers with what it has, instead of +# being cut off. A decode that alone outlasts the budget gets a 504. +# `MIN_DECODE_BUDGET_MS` is teapot's lower limit. DECODE_BUDGET_MARGIN_MS = 2000 MIN_DECODE_BUDGET_MS = 1000 diff --git a/src/sentry/options/defaults.py b/src/sentry/options/defaults.py index eaa32402b347..303784417ae1 100644 --- a/src/sentry/options/defaults.py +++ b/src/sentry/options/defaults.py @@ -537,10 +537,11 @@ default=False, flags=FLAG_ALLOW_EMPTY | FLAG_PRIORITIZE_DISK | FLAG_AUTOMATOR_MODIFIABLE, ) -# Tight timeout: decode is sub-second, so a slow teapot should fail fast. +# Covers the decode (seconds for a page fault dump) and the shader binary lookups; +# teapot gets this minus 2 s as its budget (`sentry.lang.native.teapot`). register( "teapot.timeout-seconds", - default=5, + default=15, flags=FLAG_AUTOMATOR_MODIFIABLE, ) # Retries only on transient 5xx; low so a slow teapot can't pile up work. diff --git a/tests/sentry/lang/native/test_teapot.py b/tests/sentry/lang/native/test_teapot.py index 8cf6da6d0cf3..b2d713c67c28 100644 --- a/tests/sentry/lang/native/test_teapot.py +++ b/tests/sentry/lang/native/test_teapot.py @@ -558,7 +558,7 @@ def test_client_sends_sources_only_when_given(mock_objectstore: mock.Mock) -> No assert with_sources["sources"] == [source] -@pytest.mark.parametrize(("timeout", "max_decode_ms"), [(5, 3000), (2, 1000)]) +@pytest.mark.parametrize(("timeout", "max_decode_ms"), [(15, 13000), (2, 1000)]) def test_client_gives_teapot_a_budget_below_its_timeout( mock_objectstore: mock.Mock, timeout: int, max_decode_ms: int ) -> None: From 98b6652af5a1bccb31776cdb6be6fc6e2d52dbcd Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Fri, 13 Nov 2026 16:57:13 +0200 Subject: [PATCH 24/26] Send Teapot the debug-files source only when an option allows it With the source, Teapot looks up the shader binaries of every GPU crash with the system token. The option teapot.shader-sources.enabled, off by default, turns that on and off without a Teapot deploy. --- src/sentry/options/defaults.py | 7 +++++++ src/sentry/tasks/gpu_crash.py | 5 ++++- tests/sentry/tasks/test_gpu_crash.py | 19 ++++++++++++++++++- 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/sentry/options/defaults.py b/src/sentry/options/defaults.py index 303784417ae1..65cb63767d72 100644 --- a/src/sentry/options/defaults.py +++ b/src/sentry/options/defaults.py @@ -544,6 +544,13 @@ default=15, flags=FLAG_AUTOMATOR_MODIFIABLE, ) +# Sends teapot the project's debug-files source, so that it looks up the shader +# binaries of a GPU crash itself (with the system token). +register( + "teapot.shader-sources.enabled", + default=False, + flags=FLAG_AUTOMATOR_MODIFIABLE, +) # Retries only on transient 5xx; low so a slow teapot can't pile up work. register( "teapot.max-attempts", diff --git a/src/sentry/tasks/gpu_crash.py b/src/sentry/tasks/gpu_crash.py index f0b7c4d66cf8..998a561ef7dc 100644 --- a/src/sentry/tasks/gpu_crash.py +++ b/src/sentry/tasks/gpu_crash.py @@ -143,6 +143,9 @@ def _try_symbolicate(data: MutableMapping[str, Any], project_id: int, event_id: metrics.incr("tasks.gpu_crash.skipped", tags={"reason": "project_missing"}) return False + sources = ( + [get_internal_source(project)] if options.get("teapot.shader-sources.enabled") else None + ) try: with metrics.timer("tasks.gpu_crash.teapot"): response = submit_to_teapot( @@ -150,7 +153,7 @@ def _try_symbolicate(data: MutableMapping[str, Any], project_id: int, event_id: event_id, cast(TeapotAttachment, dump), cast(Sequence[TeapotAttachment], shaders), - sources=[get_internal_source(project)], + sources=sources, ) except TeapotUnavailable: # A genuine outage — feed the breaker so repeated failures trip it. diff --git a/tests/sentry/tasks/test_gpu_crash.py b/tests/sentry/tasks/test_gpu_crash.py index bd020ce1c937..979f7931b16a 100644 --- a/tests/sentry/tasks/test_gpu_crash.py +++ b/tests/sentry/tasks/test_gpu_crash.py @@ -48,7 +48,7 @@ def circuit_breaker() -> Iterator[mock.Mock]: def test_try_symbolicate_happy_path(default_project: Project, circuit_breaker: mock.Mock) -> None: data: dict[str, Any] = {"event_id": "e", "project": default_project.id} with ( - override_options({"teapot.enabled": True}), + override_options({"teapot.enabled": True, "teapot.shader-sources.enabled": True}), mock.patch(FIND_DUMP, return_value=_FakeAttachment()), mock.patch(FIND_SHADERS, return_value=[]), mock.patch(SUBMIT_TEAPOT, return_value=_completed()) as teapot, @@ -72,6 +72,23 @@ def test_try_symbolicate_happy_path(default_project: Project, circuit_breaker: m circuit_breaker.record_error.assert_not_called() +@django_db_all +def test_try_symbolicate_sends_no_sources_by_default( + default_project: Project, circuit_breaker: mock.Mock +) -> None: + data: dict[str, Any] = {"event_id": "e", "project": default_project.id} + with ( + override_options({"teapot.enabled": True}), + mock.patch(FIND_DUMP, return_value=_FakeAttachment()), + mock.patch(FIND_SHADERS, return_value=[]), + mock.patch(SUBMIT_TEAPOT, return_value=_completed()) as teapot, + mock.patch(APPLY, return_value=data), + ): + _try_symbolicate(data, default_project.id, "e") + + assert teapot.call_args.kwargs["sources"] is None + + @django_db_all def test_try_symbolicate_skips_when_circuit_open( default_project: Project, circuit_breaker: mock.Mock From 0f146fadc5e8242186305fa358798a35df7d1acd Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Fri, 13 Nov 2026 16:57:13 +0200 Subject: [PATCH 25/26] Fill in a missing shader code id when a known file is uploaded again A dxil binary that went up without a manifest entry has no code_id, so Teapot can never find it. Uploading the same file again with a code_id now stores it, as it already did for the shader group, also when the assemble status of the file is cached. --- src/sentry/api/endpoints/debug_files.py | 43 +++++++++++------- src/sentry/models/debugfile.py | 13 +++++- tests/sentry/models/test_debugfile_shader.py | 46 +++++++++++++++++++- 3 files changed, 83 insertions(+), 19 deletions(-) diff --git a/src/sentry/api/endpoints/debug_files.py b/src/sentry/api/endpoints/debug_files.py index 4a0ee9e87f33..0d18d06ada5e 100644 --- a/src/sentry/api/endpoints/debug_files.py +++ b/src/sentry/api/endpoints/debug_files.py @@ -64,6 +64,7 @@ find_existing_dif, get_debug_id_from_dif_request, get_dif_download_filename, + normalize_shader_code_id, wasm_legacy_code_id, wasm_legacy_debug_id, ) @@ -687,11 +688,17 @@ def get_file_info(file: AssembleRequestFile) -> tuple[str, str | None, list[str] return name, debug_id, chunks -def _cached_dif_lacks_shader_group(file: AssembleRequestFile, detail: object) -> bool: - if not file.get("shader_group"): - return False - data = detail.get("data") if isinstance(detail, Mapping) else None - return not (isinstance(data, Mapping) and data.get("shader_group")) +def _cached_dif_lacks_shader_metadata(file: AssembleRequestFile, detail: object) -> bool: + """The request has a shader group or a shader code id that the cached file does not.""" + cached = detail if isinstance(detail, Mapping) else {} + data = cached.get("data") + lacks_group = bool(file.get("shader_group")) and not ( + isinstance(data, Mapping) and data.get("shader_group") + ) + lacks_code_id = normalize_shader_code_id(file.get("code_id")) is not None and not cached.get( + "codeId" + ) + return lacks_group or lacks_code_id def batch_assemble( @@ -712,11 +719,11 @@ def batch_assemble( requested_debug_id = _get_requested_debug_id(file) cached_debug_id = detail.get("uuid") if isinstance(detail, Mapping) else None - # A shader group the cached file lacks goes through the database lookup - # below, so it can be stored on the already assembled file. + # A shader group or code id the cached file lacks goes through the database + # lookup below, so it can be stored on the already assembled file. if ( state == ChunkFileState.OK - and not _cached_dif_lacks_shader_group(file, detail) + and not _cached_dif_lacks_shader_metadata(file, detail) and not _is_proguard_reupload_clone_request( file=file, requested_debug_id=requested_debug_id, @@ -763,13 +770,19 @@ def batch_assemble( ) continue - shader_group = file.get("shader_group") - if ( - shader_group - and debug_file.file_format == "dxil" - and not (debug_file.data or {}).get("shader_group") - ): - debug_file.update(data={**(debug_file.data or {}), "shader_group": shader_group}) + # A file first uploaded without a manifest entry gets its shader group and + # code id when it is uploaded again with one; without the code id the + # shader binary lookup can never find it. + if debug_file.file_format == "dxil": + updates: dict[str, Any] = {} + shader_group = file.get("shader_group") + if shader_group and not (debug_file.data or {}).get("shader_group"): + updates["data"] = {**(debug_file.data or {}), "shader_group": shader_group} + code_id = normalize_shader_code_id(file.get("code_id")) + if code_id and not debug_file.code_id: + updates["code_id"] = code_id + if updates: + debug_file.update(**updates) file_response[checksum] = { "state": ChunkFileState.OK, diff --git a/src/sentry/models/debugfile.py b/src/sentry/models/debugfile.py index 04dab16712ff..47095ab031b3 100644 --- a/src/sentry/models/debugfile.py +++ b/src/sentry/models/debugfile.py @@ -72,6 +72,15 @@ _DXIL_HEADER = b"DXBC" _SHADER_CODE_ID_RE = re.compile(r"^[0-9a-f]{16}$") + +def normalize_shader_code_id(code_id: str | None) -> str | None: + """The `code_id` of a `dxil` file in lower case, or None unless it is 16 hex characters.""" + if code_id is None: + return None + code_id = code_id.lower() + return code_id if _SHADER_CODE_ID_RE.match(code_id) else None + + SHADER_GROUP_MAX_LENGTH = 512 @@ -1048,8 +1057,8 @@ def detect_dif_from_path( ] elif dif_kind == DifKind.Dxil: if code_id is not None: - code_id = code_id.lower() - if not _SHADER_CODE_ID_RE.match(code_id): + code_id = normalize_shader_code_id(code_id) + if code_id is None: raise BadDif("Invalid code_id for dxil (expected 16 hex characters)") shader_group = (shader_group or "").strip() if len(shader_group) > SHADER_GROUP_MAX_LENGTH: diff --git a/tests/sentry/models/test_debugfile_shader.py b/tests/sentry/models/test_debugfile_shader.py index 703368fd9493..ca5ddde348df 100644 --- a/tests/sentry/models/test_debugfile_shader.py +++ b/tests/sentry/models/test_debugfile_shader.py @@ -128,11 +128,16 @@ def test_detect_dxil_rejects_overlong_shader_group(tmp_path: Path) -> None: def _create_shader_dif( - project: Project, tmp_path: Path, name: str, data: bytes, shader_group: str | None + project: Project, + tmp_path: Path, + name: str, + data: bytes, + shader_group: str | None, + code_id: str | None = AFTERMATH_HASH, ) -> tuple[ProjectDebugFile, bool]: path = _write(tmp_path, name, data) meta = detect_single_dif_from_path( - path, name=name, debug_id=DXC_HASH, code_id=AFTERMATH_HASH, shader_group=shader_group + path, name=name, debug_id=DXC_HASH, code_id=code_id, shader_group=shader_group ) session = mock.Mock() session.put.return_value = f"difs/{name}" @@ -203,6 +208,43 @@ def test_assemble_backfills_shader_group_despite_cached_ok_status( assert first.data["shader_group"] == "Group A" +@pytest.mark.parametrize("cached", [False, True]) +@django_db_all +def test_assemble_backfills_code_id_for_known_checksum( + default_project: Project, tmp_path: Path, cached: bool +) -> None: + first, _ = _create_shader_dif( + default_project, tmp_path, "a.dxil", DXIL_BYTES, "Group A", code_id=None + ) + assert first.checksum is not None + if cached: + set_assemble_status( + AssembleTask.DIF, + default_project.id, + first.checksum, + ChunkFileState.OK, + serialize(first), + ) + + response = batch_assemble( + default_project, + { + first.checksum: { + "name": "a.dxil", + "debug_id": DEBUG_ID, + "chunks": [first.checksum], + "code_id": AFTERMATH_HASH.upper(), + "shader_group": "Group A", + } + }, + ) + + assert response[first.checksum]["state"] == ChunkFileState.OK + first.refresh_from_db() + assert first.code_id == AFTERMATH_HASH + assert list(ProjectDebugFile.objects.filter(code_id=AFTERMATH_HASH)) == [first] + + @django_db_all def test_assemble_reuses_cached_ok_status_when_group_is_stored( default_project: Project, tmp_path: Path From 05a6a5917d60329f19d137a767af857382f16934 Mon Sep 17 00:00:00 2001 From: Ivan Tustanivskyi Date: Fri, 13 Nov 2026 16:57:14 +0200 Subject: [PATCH 26/26] Take the shader of a GPU crash from the frame that faulted Teapot's frames follow the order of the active shaders, so the first frame is not always the shader that faulted. The shader group, culprit, material, shader class, shader hash and faulting PC now come from the frame that Teapot marks with data.faulted. Without a mark the first frame is used, except on a page fault, where it can be any running shader and no shader is named. The decoder's own name of a renamed frame is kept in raw_function. --- src/sentry/lang/native/gpu.py | 39 +++++++++++--- .../lang/native/test_gpu_shader_group.py | 51 +++++++++++++++++++ 2 files changed, 83 insertions(+), 7 deletions(-) diff --git a/src/sentry/lang/native/gpu.py b/src/sentry/lang/native/gpu.py index d8b46d57a306..3eba003c59fd 100644 --- a/src/sentry/lang/native/gpu.py +++ b/src/sentry/lang/native/gpu.py @@ -158,16 +158,41 @@ def _tag(key: str, value: Any) -> None: return data +def _faulted_frame(response: Mapping[str, Any]) -> dict[str, Any]: + """The frame of the shader that faulted, which teapot marks with `data.faulted`. + + The frames follow the order of the active shaders. Without a mark, the first + frame is the faulted one on a hang, but on a page fault it can be any shader + that was running, so there is none. + """ + frames = [frame for frame in response.get("frames") or [] if isinstance(frame, dict)] + for frame in frames: + if (frame.get("data") or {}).get("faulted") is True: + return frame + if not frames or response.get("fault_category") == "page_fault": + return {} + return frames[0] + + def _primary_shader(response: Mapping[str, Any]) -> dict[str, Any]: - active = (response.get("shader_context") or {}).get("active_shaders") or [] + """The active shader of the faulted frame. When that frame has no hash, the first + active shader, except on a page fault (see `_faulted_frame`).""" + active = [ + s + for s in (response.get("shader_context") or {}).get("active_shaders") or [] + if isinstance(s, dict) + ] + shader_hash = (_faulted_frame(response).get("data") or {}).get("shader_hash") + if shader_hash: + return next((s for s in active if s.get("shader_hash") == shader_hash), {}) + if response.get("fault_category") == "page_fault": + return {} return active[0] if active else {} def _shader_group(response: Mapping[str, Any]) -> str | None: - """The UE shader group recorded with the faulting frame's dxil debug file, if any.""" - frames = response.get("frames") or [] - first = frames[0] if frames and isinstance(frames[0], dict) else {} - group = (first.get("data") or {}).get("shader_group") + """The UE shader group recorded with the faulted frame's dxil debug file, if any.""" + group = (_faulted_frame(response).get("data") or {}).get("shader_group") return group if isinstance(group, str) and group else None @@ -351,8 +376,7 @@ def _build_gpu_crash_context(response: Mapping[str, Any]) -> dict[str, Any]: fault = response.get("fault") or {} gpu_state = response.get("gpu_state") or {} primary_shader = _primary_shader(response) - frames = response.get("frames") or [] - primary_frame = frames[0] if frames and isinstance(frames[0], dict) else {} + primary_frame = _faulted_frame(response) resource = _faulting_resource(fault) sections = _raw_sections(response) faulted = _faulted_warps(sections) @@ -488,6 +512,7 @@ def _normalize_gpu_frames(teapot_frames: list[Any]) -> list[dict[str, Any]]: frame: dict[str, Any] = {} for field in ( "function", + "raw_function", "module", "filename", "abs_path", diff --git a/tests/sentry/lang/native/test_gpu_shader_group.py b/tests/sentry/lang/native/test_gpu_shader_group.py index ccee1425e968..4c653e1411b5 100644 --- a/tests/sentry/lang/native/test_gpu_shader_group.py +++ b/tests/sentry/lang/native/test_gpu_shader_group.py @@ -109,3 +109,54 @@ def test_no_group_changes_nothing() -> None: assert "shader_group" not in data["contexts"]["gpu_crash"] (applied,) = data["exception"]["values"][0]["stacktrace"]["frames"] assert "filename" not in applied + + +def _apply_page_fault(frames: list[dict[str, Any]]) -> MutableMapping[str, Any]: + response = _response(frames[0]) + response.update( + fault_category="page_fault", + frames=frames, + shader_context={ + "active_shaders": [ + {"shader_hash": frame["data"]["shader_hash"], "shader_type": "Compute"} + for frame in frames + ] + }, + ) + data: dict[str, Any] = {"event_id": "g" * 32, "platform": "native"} + result = apply_gpu_crash_symbolication(data, response) + assert result is not None + return result + + +def test_page_fault_takes_the_group_of_the_faulted_frame() -> None: + running = _frame(data={"shader_hash": "1111111111111111", "shader_group": GLOBAL_GROUP}) + faulted = _frame( + instruction_addr="0x50", + data={"shader_hash": "49478268ce7710e2", "shader_group": MATERIAL_GROUP, "faulted": True}, + ) + data = _apply_page_fault([running, faulted]) + + tags = dict(data["tags"]) + assert tags["gpu.shader_group"] == MATERIAL_GROUP + assert tags["gpu.shader_hash"] == "49478268ce7710e2" + assert data["culprit"] == MATERIAL_GROUP + assert data["contexts"]["gpu_crash"]["faulting_pc"] == "0x50" + + +def test_page_fault_without_a_faulted_frame_names_no_shader() -> None: + running = _frame(data={"shader_hash": "49478268ce7710e2", "shader_group": MATERIAL_GROUP}) + data = _apply_page_fault([running]) + + tags = dict(data["tags"]) + assert "gpu.shader_group" not in tags and "gpu.shader_hash" not in tags + assert "culprit" not in data + assert "shader_group" not in data["contexts"]["gpu_crash"] + + +def test_decoder_name_is_kept_as_raw_function() -> None: + data = _apply(_frame(raw_function="fragment_01")) + + (applied,) = data["exception"]["values"][0]["stacktrace"]["frames"] + assert applied["function"] == "Main" + assert applied["raw_function"] == "fragment_01"