feat(teapot): Name the faulting shader of a GPU crash from uploaded DXIL binaries

src/sentry/api/endpoints/debug_files.py

@@ -5,7 +5,7 @@
 from collections.abc import Iterable, Iterator, Mapping, Sequence, Set
 from contextlib import closing
 from datetime import datetime
-from typing import TYPE_CHECKING, Annotated, NotRequired, TypedDict, TypeGuard, cast
+from typing import TYPE_CHECKING, Annotated, Any, NotRequired, TypedDict, TypeGuard, cast
 
 import jsonschema
 import orjson
@@ -55,6 +55,7 @@
 from sentry.debug_files.upload import find_missing_chunks
 from sentry.lang.native.sources import record_last_upload
 from sentry.models.debugfile import (
+    SHADER_GROUP_MAX_LENGTH,
     ProguardArtifactRelease,
     ProjectDebugFile,
     build_proguard_reupload_dif_meta,
@@ -63,6 +64,7 @@
     find_existing_dif,
     get_debug_id_from_dif_request,
     get_dif_download_filename,
+    normalize_shader_code_id,
     wasm_legacy_code_id,
     wasm_legacy_debug_id,
 )
@@ -484,7 +486,7 @@ def on_results(
             difs: Sequence[ProjectDebugFile],
         ) -> list[DebugFileSerializerResponse]:
             # NOTE: we are only refreshing files if there is direct query for specific files
-            if debug_id and not query and not file_formats:
+            if (debug_id or code_id) and not query and not file_formats:
                 maybe_renew_debug_files(difs)
 
             serialized: list[DebugFileSerializerResponse] = serialize(difs, request.user)
@@ -588,6 +590,8 @@ class AssembleRequestFile(TypedDict):
     name: str
     chunks: list[str]
     debug_id: NotRequired[str]
+    code_id: NotRequired[str]
+    shader_group: NotRequired[str]
 
 
 AssembleRequestPayload = dict[str, AssembleRequestFile]
@@ -610,6 +614,16 @@ class AssembleRequestFile(TypedDict):
             "pattern": "^[A-Fa-f0-9-]+$",
             "description": "The debug ID to store the file under, overriding the one read from the file.",
         },
+        "code_id": {
+            "type": "string",
+            "pattern": "^[A-Fa-f0-9]+$",
+            "description": "The code ID to store the file under, for file types the server cannot read it from.",
+        },
+        "shader_group": {
+            "type": "string",
+            "maxLength": SHADER_GROUP_MAX_LENGTH,
+            "description": "For `dxil` files: the shader group the binary belongs to, as recorded by the engine's shader symbol export.",
+        },
         "chunks": {
             "type": "array",
             "items": {"type": "string", "pattern": _SHA1_PATTERN},
@@ -674,6 +688,19 @@ def get_file_info(file: AssembleRequestFile) -> tuple[str, str | None, list[str]
     return name, debug_id, chunks
 
 
+def _cached_dif_lacks_shader_metadata(file: AssembleRequestFile, detail: object) -> bool:
+    """The request has a shader group or a shader code id that the cached file does not."""
+    cached = detail if isinstance(detail, Mapping) else {}
+    data = cached.get("data")
+    lacks_group = bool(file.get("shader_group")) and not (
+        isinstance(data, Mapping) and data.get("shader_group")
+    )
+    lacks_code_id = normalize_shader_code_id(file.get("code_id")) is not None and not cached.get(
+        "codeId"
+    )
+    return lacks_group or lacks_code_id
+
+
 def batch_assemble(
     project: Project, files: AssembleRequestPayload
 ) -> dict[str, DifAssembleFileResponse]:
@@ -692,10 +719,16 @@ def batch_assemble(
         requested_debug_id = _get_requested_debug_id(file)
         cached_debug_id = detail.get("uuid") if isinstance(detail, Mapping) else None
 
-        if state == ChunkFileState.OK and not _is_proguard_reupload_clone_request(
-            file=file,
-            requested_debug_id=requested_debug_id,
-            selected_debug_id=cached_debug_id,
+        # A shader group or code id the cached file lacks goes through the database
+        # lookup below, so it can be stored on the already assembled file.
+        if (
+            state == ChunkFileState.OK
+            and not _cached_dif_lacks_shader_metadata(file, detail)
+            and not _is_proguard_reupload_clone_request(
+                file=file,
+                requested_debug_id=requested_debug_id,
+                selected_debug_id=cached_debug_id,
+            )
         ):
             file_response[checksum] = {
                 "state": state,
@@ -737,6 +770,20 @@ def batch_assemble(
             )
             continue
 
+        # A file first uploaded without a manifest entry gets its shader group and
+        # code id when it is uploaded again with one; without the code id the
+        # shader binary lookup can never find it.
+        if debug_file.file_format == "dxil":
+            updates: dict[str, Any] = {}
+            shader_group = file.get("shader_group")
+            if shader_group and not (debug_file.data or {}).get("shader_group"):
+                updates["data"] = {**(debug_file.data or {}), "shader_group": shader_group}
+            code_id = normalize_shader_code_id(file.get("code_id"))
+            if code_id and not debug_file.code_id:
+                updates["code_id"] = code_id
+            if updates:
+                debug_file.update(**updates)
+
         file_response[checksum] = {
             "state": ChunkFileState.OK,
             "detail": None,
@@ -786,15 +833,17 @@ def batch_assemble(
         set_assemble_status(AssembleTask.DIF, project.id, checksum, ChunkFileState.CREATED)
 
         name, debug_id, chunks = get_file_info(file)
-        assemble_dif.apply_async(
-            kwargs={
-                "project_id": project.id,
-                "name": name,
-                "debug_id": debug_id,
-                "checksum": checksum,
-                "chunks": chunks,
-            }
-        )
+        kwargs: dict[str, Any] = {
+            "project_id": project.id,
+            "name": name,
+            "debug_id": debug_id,
+            "checksum": checksum,
+            "chunks": chunks,
+        }
+        for key in ("code_id", "shader_group"):
+            if key in file:
+                kwargs[key] = file[key]
+        assemble_dif.apply_async(kwargs=kwargs)
 
         file_response[checksum] = {"state": ChunkFileState.CREATED, "missingChunks": []}
 

src/sentry/constants.py

@@ -375,6 +375,7 @@ def parse_log_level(level: str | None) -> LogLevel | None:
     "application/x-il2cpp-json": "il2cpp",
     "application/x-portable-pdb": "portablepdb",
     "application/x-dartsymbolmap+json": "dartsymbolmap",
+    "application/x-dxil": "dxil",
 }
 
 NATIVE_UNKNOWN_STRING = "<unknown>"

src/sentry/lang/native/gpu.py

@@ -8,7 +8,9 @@
 from typing import Any
 from xml.sax.saxutils import unescape as _xml_unescape
 
+from sentry.constants import MAX_CULPRIT_LENGTH, MAX_TAG_VALUE_LENGTH
 from sentry.utils import metrics
+from sentry.utils.strings import truncatechars
 
 logger = logging.getLogger(__name__)
 
@@ -125,6 +127,13 @@ def _tag(key: str, value: Any) -> None:
             gpu_tags[key] = str(value)
 
     _tag("gpu.shader_hash", primary_shader.get("shader_hash"))
+    shader_group = _shader_group(response)
+    material, shader_class = _shader_group_parts(shader_group)
+    _tag("gpu.shader_group", truncatechars(shader_group, MAX_TAG_VALUE_LENGTH))
+    _tag("gpu.material", truncatechars(material, MAX_TAG_VALUE_LENGTH))
+    _tag("gpu.shader_class", truncatechars(shader_class, MAX_TAG_VALUE_LENGTH))
+    if shader_group:
+        data["culprit"] = truncatechars(shader_group, MAX_CULPRIT_LENGTH)
     _tag("gpu.shader_type", primary_shader.get("shader_type"))
     _tag("gpu.device", gpu_state.get("device_name"))
     _tag("gpu.generation", _generation_name(gpu_state))
@@ -149,11 +158,57 @@ def _tag(key: str, value: Any) -> None:
     return data
 
 
+def _faulted_frame(response: Mapping[str, Any]) -> dict[str, Any]:
+    """The frame of the shader that faulted, which teapot marks with `data.faulted`.
+
+    The frames follow the order of the active shaders. Without a mark, the first
+    frame is the faulted one on a hang, but on a page fault it can be any shader
+    that was running, so there is none.
+    """
+    frames = [frame for frame in response.get("frames") or [] if isinstance(frame, dict)]
+    for frame in frames:
+        if (frame.get("data") or {}).get("faulted") is True:
+            return frame
+    if not frames or response.get("fault_category") == "page_fault":
+        return {}
+    return frames[0]
+
+
 def _primary_shader(response: Mapping[str, Any]) -> dict[str, Any]:
-    active = (response.get("shader_context") or {}).get("active_shaders") or []
+    """The active shader of the faulted frame. When that frame has no hash, the first
+    active shader, except on a page fault (see `_faulted_frame`)."""
+    active = [
+        s
+        for s in (response.get("shader_context") or {}).get("active_shaders") or []
+        if isinstance(s, dict)
+    ]
+    shader_hash = (_faulted_frame(response).get("data") or {}).get("shader_hash")
+    if shader_hash:
+        return next((s for s in active if s.get("shader_hash") == shader_hash), {})
+    if response.get("fault_category") == "page_fault":
+        return {}
     return active[0] if active else {}
 
 
+def _shader_group(response: Mapping[str, Any]) -> str | None:
+    """The UE shader group recorded with the faulted frame's dxil debug file, if any."""
+    group = (_faulted_frame(response).get("data") or {}).get("shader_group")
+    return group if isinstance(group, str) and group else None
+
+
+def _shader_group_parts(group: str | None) -> tuple[str | None, str | None]:
+    """``(material, shader_class)`` from ``Material/Quality/VertexFactory/ShaderClass/Permutation``
+    or ``Global/ShaderClass/Permutation``."""
+    if not group:
+        return None, None
+    parts = [part for part in group.split("/") if part]
+    if len(parts) < 2:
+        return None, None
+    if parts[0] == "Global":
+        return None, parts[1]
+    return parts[0], parts[-2] if len(parts) >= 3 else None
+
+
 def _raw_sections(response: Mapping[str, Any]) -> dict[str, Any]:
     raw = (response.get("shader_context") or {}).get("raw")
     out: dict[str, Any] = {}
@@ -321,12 +376,13 @@ def _build_gpu_crash_context(response: Mapping[str, Any]) -> dict[str, Any]:
     fault = response.get("fault") or {}
     gpu_state = response.get("gpu_state") or {}
     primary_shader = _primary_shader(response)
-    frames = response.get("frames") or []
-    primary_frame = frames[0] if frames and isinstance(frames[0], dict) else {}
+    primary_frame = _faulted_frame(response)
     resource = _faulting_resource(fault)
     sections = _raw_sections(response)
     faulted = _faulted_warps(sections)
     first_faulted = faulted[0] if faulted else {}
+    shader_group = _shader_group(response)
+    material, shader_class = _shader_group_parts(shader_group)
 
     flat: dict[str, Any] = {
         "type": "gpu_crash",
@@ -351,6 +407,9 @@ def _build_gpu_crash_context(response: Mapping[str, Any]) -> dict[str, Any]:
         "resource_destroyed": resource.get("was_destroyed") or None,
         "shader_hash": primary_shader.get("shader_hash"),
         "shader_type": primary_shader.get("shader_type"),
+        "shader_group": shader_group,
+        "material": material,
+        "shader_class": shader_class,
         "shader_size": _shader_size(sections),
         "shader_debug_info_uid": primary_shader.get("shader_debug_info_uid"),
         "status": response.get("status"),
@@ -453,6 +512,7 @@ def _normalize_gpu_frames(teapot_frames: list[Any]) -> list[dict[str, Any]]:
         frame: dict[str, Any] = {}
         for field in (
             "function",
+            "raw_function",
             "module",
             "filename",
             "abs_path",
@@ -477,6 +537,9 @@ def _normalize_gpu_frames(teapot_frames: list[Any]) -> list[dict[str, Any]]:
             )
         if not frame.get("module") and frame.get("package"):
             frame["module"] = frame["package"]
+        group = raw_data.get("shader_group")
+        if isinstance(group, str) and group and not frame.get("filename"):
+            frame["filename"] = group
 
         frame.setdefault("data", {})
         frame["data"].setdefault("symbolicator_status", "symbolicated")