diff --git a/src/sentry/api/endpoints/debug_files.py b/src/sentry/api/endpoints/debug_files.py index d09bd5d80ca6..0d18d06ada5e 100644 --- a/src/sentry/api/endpoints/debug_files.py +++ b/src/sentry/api/endpoints/debug_files.py @@ -5,7 +5,7 @@ from collections.abc import Iterable, Iterator, Mapping, Sequence, Set from contextlib import closing from datetime import datetime -from typing import TYPE_CHECKING, Annotated, NotRequired, TypedDict, TypeGuard, cast +from typing import TYPE_CHECKING, Annotated, Any, NotRequired, TypedDict, TypeGuard, cast import jsonschema import orjson @@ -55,6 +55,7 @@ from sentry.debug_files.upload import find_missing_chunks from sentry.lang.native.sources import record_last_upload from sentry.models.debugfile import ( + SHADER_GROUP_MAX_LENGTH, ProguardArtifactRelease, ProjectDebugFile, build_proguard_reupload_dif_meta, @@ -63,6 +64,7 @@ find_existing_dif, get_debug_id_from_dif_request, get_dif_download_filename, + normalize_shader_code_id, wasm_legacy_code_id, wasm_legacy_debug_id, ) @@ -484,7 +486,7 @@ def on_results( difs: Sequence[ProjectDebugFile], ) -> list[DebugFileSerializerResponse]: # NOTE: we are only refreshing files if there is direct query for specific files - if debug_id and not query and not file_formats: + if (debug_id or code_id) and not query and not file_formats: maybe_renew_debug_files(difs) serialized: list[DebugFileSerializerResponse] = serialize(difs, request.user) @@ -588,6 +590,8 @@ class AssembleRequestFile(TypedDict): name: str chunks: list[str] debug_id: NotRequired[str] + code_id: NotRequired[str] + shader_group: NotRequired[str] AssembleRequestPayload = dict[str, AssembleRequestFile] @@ -610,6 +614,16 @@ class AssembleRequestFile(TypedDict): "pattern": "^[A-Fa-f0-9-]+$", "description": "The debug ID to store the file under, overriding the one read from the file.", }, + "code_id": { + "type": "string", + "pattern": "^[A-Fa-f0-9]+$", + "description": "The code ID to store the file under, for file types the server cannot read it from.", + }, + "shader_group": { + "type": "string", + "maxLength": SHADER_GROUP_MAX_LENGTH, + "description": "For `dxil` files: the shader group the binary belongs to, as recorded by the engine's shader symbol export.", + }, "chunks": { "type": "array", "items": {"type": "string", "pattern": _SHA1_PATTERN}, @@ -674,6 +688,19 @@ def get_file_info(file: AssembleRequestFile) -> tuple[str, str | None, list[str] return name, debug_id, chunks +def _cached_dif_lacks_shader_metadata(file: AssembleRequestFile, detail: object) -> bool: + """The request has a shader group or a shader code id that the cached file does not.""" + cached = detail if isinstance(detail, Mapping) else {} + data = cached.get("data") + lacks_group = bool(file.get("shader_group")) and not ( + isinstance(data, Mapping) and data.get("shader_group") + ) + lacks_code_id = normalize_shader_code_id(file.get("code_id")) is not None and not cached.get( + "codeId" + ) + return lacks_group or lacks_code_id + + def batch_assemble( project: Project, files: AssembleRequestPayload ) -> dict[str, DifAssembleFileResponse]: @@ -692,10 +719,16 @@ def batch_assemble( requested_debug_id = _get_requested_debug_id(file) cached_debug_id = detail.get("uuid") if isinstance(detail, Mapping) else None - if state == ChunkFileState.OK and not _is_proguard_reupload_clone_request( - file=file, - requested_debug_id=requested_debug_id, - selected_debug_id=cached_debug_id, + # A shader group or code id the cached file lacks goes through the database + # lookup below, so it can be stored on the already assembled file. + if ( + state == ChunkFileState.OK + and not _cached_dif_lacks_shader_metadata(file, detail) + and not _is_proguard_reupload_clone_request( + file=file, + requested_debug_id=requested_debug_id, + selected_debug_id=cached_debug_id, + ) ): file_response[checksum] = { "state": state, @@ -737,6 +770,20 @@ def batch_assemble( ) continue + # A file first uploaded without a manifest entry gets its shader group and + # code id when it is uploaded again with one; without the code id the + # shader binary lookup can never find it. + if debug_file.file_format == "dxil": + updates: dict[str, Any] = {} + shader_group = file.get("shader_group") + if shader_group and not (debug_file.data or {}).get("shader_group"): + updates["data"] = {**(debug_file.data or {}), "shader_group": shader_group} + code_id = normalize_shader_code_id(file.get("code_id")) + if code_id and not debug_file.code_id: + updates["code_id"] = code_id + if updates: + debug_file.update(**updates) + file_response[checksum] = { "state": ChunkFileState.OK, "detail": None, @@ -786,15 +833,17 @@ def batch_assemble( set_assemble_status(AssembleTask.DIF, project.id, checksum, ChunkFileState.CREATED) name, debug_id, chunks = get_file_info(file) - assemble_dif.apply_async( - kwargs={ - "project_id": project.id, - "name": name, - "debug_id": debug_id, - "checksum": checksum, - "chunks": chunks, - } - ) + kwargs: dict[str, Any] = { + "project_id": project.id, + "name": name, + "debug_id": debug_id, + "checksum": checksum, + "chunks": chunks, + } + for key in ("code_id", "shader_group"): + if key in file: + kwargs[key] = file[key] + assemble_dif.apply_async(kwargs=kwargs) file_response[checksum] = {"state": ChunkFileState.CREATED, "missingChunks": []} diff --git a/src/sentry/constants.py b/src/sentry/constants.py index 1562c2fa1b18..240708247254 100644 --- a/src/sentry/constants.py +++ b/src/sentry/constants.py @@ -375,6 +375,7 @@ def parse_log_level(level: str | None) -> LogLevel | None: "application/x-il2cpp-json": "il2cpp", "application/x-portable-pdb": "portablepdb", "application/x-dartsymbolmap+json": "dartsymbolmap", + "application/x-dxil": "dxil", } NATIVE_UNKNOWN_STRING = "" diff --git a/src/sentry/lang/native/gpu.py b/src/sentry/lang/native/gpu.py index 228b571b2eb1..3eba003c59fd 100644 --- a/src/sentry/lang/native/gpu.py +++ b/src/sentry/lang/native/gpu.py @@ -8,7 +8,9 @@ from typing import Any from xml.sax.saxutils import unescape as _xml_unescape +from sentry.constants import MAX_CULPRIT_LENGTH, MAX_TAG_VALUE_LENGTH from sentry.utils import metrics +from sentry.utils.strings import truncatechars logger = logging.getLogger(__name__) @@ -125,6 +127,13 @@ def _tag(key: str, value: Any) -> None: gpu_tags[key] = str(value) _tag("gpu.shader_hash", primary_shader.get("shader_hash")) + shader_group = _shader_group(response) + material, shader_class = _shader_group_parts(shader_group) + _tag("gpu.shader_group", truncatechars(shader_group, MAX_TAG_VALUE_LENGTH)) + _tag("gpu.material", truncatechars(material, MAX_TAG_VALUE_LENGTH)) + _tag("gpu.shader_class", truncatechars(shader_class, MAX_TAG_VALUE_LENGTH)) + if shader_group: + data["culprit"] = truncatechars(shader_group, MAX_CULPRIT_LENGTH) _tag("gpu.shader_type", primary_shader.get("shader_type")) _tag("gpu.device", gpu_state.get("device_name")) _tag("gpu.generation", _generation_name(gpu_state)) @@ -149,11 +158,57 @@ def _tag(key: str, value: Any) -> None: return data +def _faulted_frame(response: Mapping[str, Any]) -> dict[str, Any]: + """The frame of the shader that faulted, which teapot marks with `data.faulted`. + + The frames follow the order of the active shaders. Without a mark, the first + frame is the faulted one on a hang, but on a page fault it can be any shader + that was running, so there is none. + """ + frames = [frame for frame in response.get("frames") or [] if isinstance(frame, dict)] + for frame in frames: + if (frame.get("data") or {}).get("faulted") is True: + return frame + if not frames or response.get("fault_category") == "page_fault": + return {} + return frames[0] + + def _primary_shader(response: Mapping[str, Any]) -> dict[str, Any]: - active = (response.get("shader_context") or {}).get("active_shaders") or [] + """The active shader of the faulted frame. When that frame has no hash, the first + active shader, except on a page fault (see `_faulted_frame`).""" + active = [ + s + for s in (response.get("shader_context") or {}).get("active_shaders") or [] + if isinstance(s, dict) + ] + shader_hash = (_faulted_frame(response).get("data") or {}).get("shader_hash") + if shader_hash: + return next((s for s in active if s.get("shader_hash") == shader_hash), {}) + if response.get("fault_category") == "page_fault": + return {} return active[0] if active else {} +def _shader_group(response: Mapping[str, Any]) -> str | None: + """The UE shader group recorded with the faulted frame's dxil debug file, if any.""" + group = (_faulted_frame(response).get("data") or {}).get("shader_group") + return group if isinstance(group, str) and group else None + + +def _shader_group_parts(group: str | None) -> tuple[str | None, str | None]: + """``(material, shader_class)`` from ``Material/Quality/VertexFactory/ShaderClass/Permutation`` + or ``Global/ShaderClass/Permutation``.""" + if not group: + return None, None + parts = [part for part in group.split("/") if part] + if len(parts) < 2: + return None, None + if parts[0] == "Global": + return None, parts[1] + return parts[0], parts[-2] if len(parts) >= 3 else None + + def _raw_sections(response: Mapping[str, Any]) -> dict[str, Any]: raw = (response.get("shader_context") or {}).get("raw") out: dict[str, Any] = {} @@ -321,12 +376,13 @@ def _build_gpu_crash_context(response: Mapping[str, Any]) -> dict[str, Any]: fault = response.get("fault") or {} gpu_state = response.get("gpu_state") or {} primary_shader = _primary_shader(response) - frames = response.get("frames") or [] - primary_frame = frames[0] if frames and isinstance(frames[0], dict) else {} + primary_frame = _faulted_frame(response) resource = _faulting_resource(fault) sections = _raw_sections(response) faulted = _faulted_warps(sections) first_faulted = faulted[0] if faulted else {} + shader_group = _shader_group(response) + material, shader_class = _shader_group_parts(shader_group) flat: dict[str, Any] = { "type": "gpu_crash", @@ -351,6 +407,9 @@ def _build_gpu_crash_context(response: Mapping[str, Any]) -> dict[str, Any]: "resource_destroyed": resource.get("was_destroyed") or None, "shader_hash": primary_shader.get("shader_hash"), "shader_type": primary_shader.get("shader_type"), + "shader_group": shader_group, + "material": material, + "shader_class": shader_class, "shader_size": _shader_size(sections), "shader_debug_info_uid": primary_shader.get("shader_debug_info_uid"), "status": response.get("status"), @@ -453,6 +512,7 @@ def _normalize_gpu_frames(teapot_frames: list[Any]) -> list[dict[str, Any]]: frame: dict[str, Any] = {} for field in ( "function", + "raw_function", "module", "filename", "abs_path", @@ -477,6 +537,9 @@ def _normalize_gpu_frames(teapot_frames: list[Any]) -> list[dict[str, Any]]: ) if not frame.get("module") and frame.get("package"): frame["module"] = frame["package"] + group = raw_data.get("shader_group") + if isinstance(group, str) and group and not frame.get("filename"): + frame["filename"] = group frame.setdefault("data", {}) frame["data"].setdefault("symbolicator_status", "symbolicated") diff --git a/src/sentry/lang/native/teapot.py b/src/sentry/lang/native/teapot.py index b7b0c65d97cf..bd53ffd3faf0 100644 --- a/src/sentry/lang/native/teapot.py +++ b/src/sentry/lang/native/teapot.py @@ -13,7 +13,7 @@ import hmac import logging import re -from collections.abc import Sequence +from collections.abc import Mapping, Sequence from typing import Any, Protocol import orjson @@ -56,11 +56,19 @@ class TeapotAttachment(Protocol): # Fallbacks for the `teapot.timeout-seconds` / `teapot.max-attempts` options. -# The timeout is tight on purpose: decode is sub-second, so a slow teapot should -# fail fast (the circuit breaker then takes over) rather than hold a worker. -DEFAULT_TIMEOUT = 5 +# Most decodes take milliseconds, but a page fault dump can take a few seconds, +# and the request also covers teapot's shader binary lookups. A teapot that is +# down still fails well inside the task's deadline, and the breaker takes over. +DEFAULT_TIMEOUT = 15 DEFAULT_MAX_ATTEMPTS = 2 +# Teapot's budget for the decode and the shader binary lookups ends before the +# timeout: it stops the lookups early and answers with what it has, instead of +# being cut off. A decode that alone outlasts the budget gets a 504. +# `MIN_DECODE_BUDGET_MS` is teapot's lower limit. +DECODE_BUDGET_MARGIN_MS = 2000 +MIN_DECODE_BUDGET_MS = 1000 + RETRYABLE_STATUS = (502, 503, 504) # Bounded exponential backoff between transient retries. @@ -75,6 +83,10 @@ def _timeout() -> int: return DEFAULT_TIMEOUT +def _max_decode_ms() -> int: + return max(MIN_DECODE_BUDGET_MS, _timeout() * 1000 - DECODE_BUDGET_MARGIN_MS) + + def _max_attempts() -> int: try: return max(1, int(options.get("teapot.max-attempts"))) @@ -109,6 +121,13 @@ def _redact_urls(text: str) -> str: return _URL_QUERY_RE.sub(r"\1?", text) +def _redact(text: str, secrets: Sequence[str]) -> str: + """Presigned URL tokens and the given secrets (the sources' tokens) removed.""" + for secret in secrets: + text = text.replace(secret, "") + return _redact_urls(text) + + def _resolve_url() -> str | None: base = getattr(settings, "SENTRY_TEAPOT_URL", None) return base.rstrip("/") if base else None @@ -139,6 +158,7 @@ def symbolicate( self, dump: TeapotAttachment, shader_debug_info: Sequence[TeapotAttachment] | None = None, + sources: Sequence[Mapping[str, Any]] | None = None, ) -> dict[str, Any]: shader_debug_info = shader_debug_info or [] url = f"{self.base_url}/symbolicate" @@ -156,7 +176,15 @@ def symbolicate( } for att in shader_debug_info ], + "options": {"max_decode_ms": _max_decode_ms()}, } + secrets: list[str] = [] + if sources: + # Carries the system token; never log the body. + body["sources"] = list(sources) + secrets = [ + s["token"] for s in sources if isinstance(s.get("token"), str) and s["token"] + ] data = orjson.dumps(body) headers = { "X-Teapot-Version": "1", @@ -165,7 +193,7 @@ def symbolicate( "Idempotency-Key": self.event_id, **_auth_headers(data), } - return self._send(url, headers=headers, data=data) + return self._send(url, headers=headers, data=data, secrets=secrets) def _storage_url(self, session: Any, att: TeapotAttachment) -> str: """Short-lived self-authenticating (presigned) GET URL for the attachment — @@ -174,7 +202,9 @@ def _storage_url(self, session: Any, att: TeapotAttachment) -> str: raise TeapotRequestError(f"attachment {att.name!r} is not in objectstore") return get_internal_download_url(session, att.stored_id) - def _send(self, url: str, headers: dict[str, str], data: bytes) -> dict[str, Any]: + def _send( + self, url: str, headers: dict[str, str], data: bytes, secrets: Sequence[str] = () + ) -> dict[str, Any]: def attempt() -> dict[str, Any]: try: resp = requests.post(url, data=data, headers=headers, timeout=_timeout()) @@ -185,7 +215,8 @@ def attempt() -> dict[str, Any]: raise _RetryableTeapotError(f"teapot returned {resp.status_code}") if resp.status_code >= 400: - safe_body = _redact_urls(resp.text[:512]) + # Redact before truncating, so a cut can't leave part of a secret. + safe_body = _redact(resp.text, secrets)[:512] logger.warning( "teapot.request_failed", extra={ @@ -226,6 +257,7 @@ def submit_to_teapot( event_id: str, dump: TeapotAttachment, shader_debug_info: Sequence[TeapotAttachment] | None = None, + sources: Sequence[Mapping[str, Any]] | None = None, ) -> dict[str, Any] | None: """Best-effort teapot invocation. @@ -233,11 +265,13 @@ def submit_to_teapot( URL, missing attachment, 4xx) — teapot is healthy, so the caller should not trip the circuit breaker. Raises ``TeapotUnavailable`` on a genuine outage (network, timeout, 5xx) so the caller can. + + ``sources`` tells teapot where to fetch the shader binaries itself. """ try: client = TeapotClient(project=project, event_id=event_id) - return client.symbolicate(dump, shader_debug_info or []) + return client.symbolicate(dump, shader_debug_info or [], sources=sources) except TeapotRequestError as e: logger.info("teapot.request_error", extra={"event_id": event_id, "error": str(e)}) return None diff --git a/src/sentry/models/debugfile.py b/src/sentry/models/debugfile.py index ac6099d05832..47095ab031b3 100644 --- a/src/sentry/models/debugfile.py +++ b/src/sentry/models/debugfile.py @@ -69,6 +69,20 @@ _proguard_file_re = re.compile(r"/proguard/(?:mapping-)?(.*?)\.txt$") +_DXIL_HEADER = b"DXBC" +_SHADER_CODE_ID_RE = re.compile(r"^[0-9a-f]{16}$") + + +def normalize_shader_code_id(code_id: str | None) -> str | None: + """The `code_id` of a `dxil` file in lower case, or None unless it is 16 hex characters.""" + if code_id is None: + return None + code_id = code_id.lower() + return code_id if _SHADER_CODE_ID_RE.match(code_id) else None + + +SHADER_GROUP_MAX_LENGTH = 512 + def _dif_file_extension(file_format: str, file_type: str | None) -> str: if file_format == "breakpad": @@ -95,6 +109,8 @@ def _dif_file_extension(file_format: str, file_type: str | None) -> str: return ".json" if file_format == "dartsymbolmap": return ".json" + if file_format == "dxil": + return ".dxil" return "" @@ -452,6 +468,7 @@ def clean_redundant_difs(project: Project, debug_id: str) -> None: bcsymbolmap_seen = False uuidmap_seen = False il2cpp_seen = False + dxil_seen = False for i, dif in enumerate(difs): mime_type = dif.get_content_type() if mime_type == DIF_MIMETYPES["bcsymbolmap"]: @@ -469,6 +486,11 @@ def clean_redundant_difs(project: Project, debug_id: str) -> None: il2cpp_seen = True else: dif.delete() + elif mime_type == DIF_MIMETYPES["dxil"]: + if not dxil_seen: + dxil_seen = True + else: + dif.delete() else: # We always keep the latest file. If it has no features, likely the # previous files did not have features either and will be removed, or we @@ -483,9 +505,13 @@ def detect_single_dif_from_path( path: str, name: str | None = None, debug_id: str | None = None, + code_id: str | None = None, + shader_group: str | None = None, ) -> DifMeta: """Like :func:`detect_dif_from_path`, but requires exactly one architecture.""" - result = detect_dif_from_path(path, name=name, debug_id=debug_id) + result = detect_dif_from_path( + path, name=name, debug_id=debug_id, code_id=code_id, shader_group=shader_group + ) if len(result) != 1: raise BadDif("Object contains %s architectures (1 expected)" % len(result)) @@ -508,6 +534,7 @@ def _get_dif_object_name(meta: DifMeta) -> str: "uuidmap", "il2cpp", "dartsymbolmap", + "dxil", ): return meta.name if meta.file_format == "breakpad": @@ -872,6 +899,7 @@ class DifKind(enum.Enum): UuidMap = enum.auto() Il2Cpp = enum.auto() DartSymbolMap = enum.auto() + Dxil = enum.auto() # TODO(flub): should we include proguard? The tradeoff is that we'd be matching the # regex of it twice. That cost is probably not too great to worry about. @@ -891,15 +919,28 @@ def determine_dif_kind(path: str) -> DifKind: return DifKind.Il2Cpp elif data.startswith(b"["): return DifKind.DartSymbolMap + elif data.startswith(_DXIL_HEADER): + return DifKind.Dxil else: return DifKind.Object +def _shader_debug_id(debug_id: str | None) -> str: + if debug_id is None: + raise BadDif("Missing debug_id for dxil") + try: + return normalize_debug_id(debug_id) + except SymbolicError as e: + raise BadDif(f"Invalid debug_id for dxil: {e}") + + def detect_dif_from_path( path: str, name: str | None = None, debug_id: str | None = None, accept_unknown: bool = False, + code_id: str | None = None, + shader_group: str | None = None, ) -> list[DifMeta]: """Detects which kind of Debug Information File (DIF) the file at `path` is. @@ -1014,6 +1055,31 @@ def detect_dif_from_path( data=data, ) ] + elif dif_kind == DifKind.Dxil: + if code_id is not None: + code_id = normalize_shader_code_id(code_id) + if code_id is None: + raise BadDif("Invalid code_id for dxil (expected 16 hex characters)") + shader_group = (shader_group or "").strip() + if len(shader_group) > SHADER_GROUP_MAX_LENGTH: + raise BadDif( + f"Invalid shader_group for dxil (max {SHADER_GROUP_MAX_LENGTH} characters)" + ) + logger.debug("File loaded as dxil: %s", path) + dxil_data: dict[str, Any] = {"features": ["symtab"]} + if shader_group: + dxil_data["shader_group"] = shader_group + return [ + DifMeta( + file_format="dxil", + arch="any", + debug_id=_shader_debug_id(debug_id), + code_id=code_id, + name=name, + path=path, + data=dxil_data, + ) + ] else: # native debug information files (MachO, ELF or Breakpad) try: diff --git a/src/sentry/options/defaults.py b/src/sentry/options/defaults.py index eaa32402b347..65cb63767d72 100644 --- a/src/sentry/options/defaults.py +++ b/src/sentry/options/defaults.py @@ -537,10 +537,18 @@ default=False, flags=FLAG_ALLOW_EMPTY | FLAG_PRIORITIZE_DISK | FLAG_AUTOMATOR_MODIFIABLE, ) -# Tight timeout: decode is sub-second, so a slow teapot should fail fast. +# Covers the decode (seconds for a page fault dump) and the shader binary lookups; +# teapot gets this minus 2 s as its budget (`sentry.lang.native.teapot`). register( "teapot.timeout-seconds", - default=5, + default=15, + flags=FLAG_AUTOMATOR_MODIFIABLE, +) +# Sends teapot the project's debug-files source, so that it looks up the shader +# binaries of a GPU crash itself (with the system token). +register( + "teapot.shader-sources.enabled", + default=False, flags=FLAG_AUTOMATOR_MODIFIABLE, ) # Retries only on transient 5xx; low so a slow teapot can't pile up work. diff --git a/src/sentry/tasks/assemble.py b/src/sentry/tasks/assemble.py index 0234463f0eee..878d13db8567 100644 --- a/src/sentry/tasks/assemble.py +++ b/src/sentry/tasks/assemble.py @@ -323,6 +323,8 @@ def assemble_dif( checksum: str, chunks: Sequence[str], debug_id: str | None = None, + code_id: str | None = None, + shader_group: str | None = None, **kwargs: object, ) -> None: """ @@ -372,7 +374,13 @@ def assemble_dif( # We only permit split difs to hit this endpoint. # The client is required to split them up first or we error. try: - meta = detect_single_dif_from_path(temp_file.name, name=name, debug_id=debug_id) + meta = detect_single_dif_from_path( + temp_file.name, + name=name, + debug_id=debug_id, + code_id=code_id, + shader_group=shader_group, + ) if file is None: temp_file.seek(0) dif, created = create_dif_from_fileobj(project, meta, temp_file) diff --git a/src/sentry/tasks/gpu_crash.py b/src/sentry/tasks/gpu_crash.py index c5660b0246ab..998a561ef7dc 100644 --- a/src/sentry/tasks/gpu_crash.py +++ b/src/sentry/tasks/gpu_crash.py @@ -19,6 +19,7 @@ from sentry.ingest.event_payload import load_event_payload from sentry.killswitches import killswitch_matches_context from sentry.lang.native.gpu import apply_gpu_crash_symbolication +from sentry.lang.native.sources import get_internal_source from sentry.lang.native.teapot import TeapotAttachment, TeapotUnavailable, submit_to_teapot from sentry.lang.native.utils import ( find_all_shader_debug_attachments, @@ -142,6 +143,9 @@ def _try_symbolicate(data: MutableMapping[str, Any], project_id: int, event_id: metrics.incr("tasks.gpu_crash.skipped", tags={"reason": "project_missing"}) return False + sources = ( + [get_internal_source(project)] if options.get("teapot.shader-sources.enabled") else None + ) try: with metrics.timer("tasks.gpu_crash.teapot"): response = submit_to_teapot( @@ -149,6 +153,7 @@ def _try_symbolicate(data: MutableMapping[str, Any], project_id: int, event_id: event_id, cast(TeapotAttachment, dump), cast(Sequence[TeapotAttachment], shaders), + sources=sources, ) except TeapotUnavailable: # A genuine outage — feed the breaker so repeated failures trip it. diff --git a/tests/sentry/api/endpoints/test_debug_files.py b/tests/sentry/api/endpoints/test_debug_files.py index 17b73148449a..f28615bc06fc 100644 --- a/tests/sentry/api/endpoints/test_debug_files.py +++ b/tests/sentry/api/endpoints/test_debug_files.py @@ -1,5 +1,6 @@ import os import zipfile +from datetime import timedelta from io import BytesIO from unittest.mock import patch from uuid import uuid4 @@ -8,6 +9,7 @@ from django.core.files.uploadedfile import SimpleUploadedFile from django.http import HttpResponse from django.urls import reverse +from django.utils import timezone from rest_framework.response import Response from symbolic.debuginfo import Archive @@ -168,6 +170,42 @@ def test_dsyms_search(self) -> None: dsyms = response.data assert len(dsyms) == 20 + def test_dsyms_lookup_by_id_renews_files(self) -> None: + old = timezone.now() - timedelta(days=60) + by_debug_id = self.create_dif_file( + debug_id="dfb8e43a-f242-3d73-a453-aeb6a777ef75", code_id="ae0459704fc7256" + ) + by_code_id = self.create_dif_file( + debug_id="a223507a-29b5-0f71-32d4-4901c2228311", code_id="658794f3a75293d1" + ) + ProjectDebugFile.objects.filter(id__in=[by_debug_id.id, by_code_id.id]).update( + date_accessed=old + ) + + for query in ("debug_id=dfb8e43a-f242-3d73-a453-aeb6a777ef75", "code_id=658794f3a75293d1"): + response = self.client.get(f"{self.url}?{query}") + assert response.status_code == 200, response.content + assert len(response.data) == 1 + + by_debug_id.refresh_from_db() + by_code_id.refresh_from_db() + assert by_debug_id.date_accessed > old + assert by_code_id.date_accessed > old + + def test_dsyms_search_and_format_filter_do_not_renew(self) -> None: + old = timezone.now() - timedelta(days=60) + dif = self.create_dif_file( + debug_id="a223507a-29b5-0f71-32d4-4901c2228311", code_id="658794f3a75293d1" + ) + ProjectDebugFile.objects.filter(id=dif.id).update(date_accessed=old) + + for query in ("query=658794f3a75293d1", "code_id=658794f3a75293d1&file_formats=pe"): + response = self.client.get(f"{self.url}?{query}") + assert response.status_code == 200, response.content + + dif.refresh_from_db() + assert dif.date_accessed == old + def test_dsyms_debugid_codeid_full_match(self) -> None: self._do_test_dsyms_by_debugid_and_codeid( ("dfb8e43a-f242-3d73-a453-aeb6a777ef75", "ae0459704fc7256"), diff --git a/tests/sentry/lang/native/test_gpu_shader_group.py b/tests/sentry/lang/native/test_gpu_shader_group.py new file mode 100644 index 000000000000..4c653e1411b5 --- /dev/null +++ b/tests/sentry/lang/native/test_gpu_shader_group.py @@ -0,0 +1,162 @@ +from __future__ import annotations + +from collections.abc import MutableMapping +from typing import Any + +from sentry.constants import MAX_CULPRIT_LENGTH, MAX_TAG_VALUE_LENGTH +from sentry.lang.native.gpu import apply_gpu_crash_symbolication + +MATERIAL_GROUP = "M_GpuHang_992b2d801a2e9ddf/Default/FLocalVertexFactory/FLumenCardPS/0" +GLOBAL_GROUP = "Global/FHeavyComputeLoop/0" + + +def _response(frame: dict[str, Any]) -> dict[str, Any]: + return { + "status": "completed", + "handler": "aftermath", + "fault_category": "shader_hang", + "title": "GPU hang in Main", + "fingerprint": ["gpu", "shader_hang", "49478268ce7710e2"], + "markers": [], + "fault": {"type": "Timeout"}, + "gpu_state": {"device_name": "GA102-A"}, + "shader_context": { + "active_shaders": [{"shader_hash": "49478268ce7710e2", "shader_type": "Pixel"}] + }, + "frames": [frame], + "missing_difs": [], + } + + +def _apply(frame: dict[str, Any]) -> MutableMapping[str, Any]: + data: dict[str, Any] = {"event_id": "g" * 32, "platform": "native"} + result = apply_gpu_crash_symbolication(data, _response(frame)) + assert result is not None + return result + + +def _frame(**overrides: Any) -> dict[str, Any]: + frame = { + "function": "Main", + "module": MATERIAL_GROUP, + "data": {"shader_hash": "49478268ce7710e2", "shader_group": MATERIAL_GROUP}, + } + frame.update(overrides) + return frame + + +def test_material_group_fills_tags_culprit_and_context() -> None: + data = _apply(_frame()) + + tags = dict(data["tags"]) + assert tags["gpu.shader_group"] == MATERIAL_GROUP + assert tags["gpu.material"] == "M_GpuHang_992b2d801a2e9ddf" + assert tags["gpu.shader_class"] == "FLumenCardPS" + assert data["culprit"] == MATERIAL_GROUP + + gpu_crash = data["contexts"]["gpu_crash"] + assert gpu_crash["shader_group"] == MATERIAL_GROUP + assert gpu_crash["material"] == "M_GpuHang_992b2d801a2e9ddf" + assert gpu_crash["shader_class"] == "FLumenCardPS" + + +def test_long_group_is_truncated_for_culprit_and_tags() -> None: + group = "M_" + "x" * 600 + "/Default/FLocalVertexFactory/FLumenCardPS/0" + data = _apply( + _frame(module=group, data={"shader_hash": "49478268ce7710e2", "shader_group": group}) + ) + + assert len(data["culprit"]) <= MAX_CULPRIT_LENGTH + assert data["culprit"].startswith("M_xxx") + tags = dict(data["tags"]) + assert len(tags["gpu.shader_group"]) <= MAX_TAG_VALUE_LENGTH + assert len(tags["gpu.material"]) <= MAX_TAG_VALUE_LENGTH + assert tags["gpu.material"].startswith("M_xxx") + assert tags["gpu.shader_class"] == "FLumenCardPS" + assert data["contexts"]["gpu_crash"]["shader_group"] == group + + +def test_global_group_has_no_material() -> None: + data = _apply(_frame(module=GLOBAL_GROUP, data={"shader_group": GLOBAL_GROUP})) + + tags = dict(data["tags"]) + assert tags["gpu.shader_group"] == GLOBAL_GROUP + assert tags["gpu.shader_class"] == "FHeavyComputeLoop" + assert "gpu.material" not in tags + assert "material" not in data["contexts"]["gpu_crash"] + + +def test_group_fills_the_empty_filename_slot() -> None: + data = _apply(_frame()) + + (applied,) = data["exception"]["values"][0]["stacktrace"]["frames"] + assert applied["filename"] == MATERIAL_GROUP + + +def test_real_filename_is_kept() -> None: + data = _apply(_frame(filename="/Game/Shaders/Foo.usf", lineno=12)) + + (applied,) = data["exception"]["values"][0]["stacktrace"]["frames"] + assert applied["filename"] == "/Game/Shaders/Foo.usf" + + +def test_no_group_changes_nothing() -> None: + data = _apply({"function": "compute_01", "module": "shader_658794f3a75293d1", "data": {}}) + + tags = dict(data["tags"]) + assert "gpu.shader_group" not in tags and "gpu.material" not in tags + assert "culprit" not in data + assert "shader_group" not in data["contexts"]["gpu_crash"] + (applied,) = data["exception"]["values"][0]["stacktrace"]["frames"] + assert "filename" not in applied + + +def _apply_page_fault(frames: list[dict[str, Any]]) -> MutableMapping[str, Any]: + response = _response(frames[0]) + response.update( + fault_category="page_fault", + frames=frames, + shader_context={ + "active_shaders": [ + {"shader_hash": frame["data"]["shader_hash"], "shader_type": "Compute"} + for frame in frames + ] + }, + ) + data: dict[str, Any] = {"event_id": "g" * 32, "platform": "native"} + result = apply_gpu_crash_symbolication(data, response) + assert result is not None + return result + + +def test_page_fault_takes_the_group_of_the_faulted_frame() -> None: + running = _frame(data={"shader_hash": "1111111111111111", "shader_group": GLOBAL_GROUP}) + faulted = _frame( + instruction_addr="0x50", + data={"shader_hash": "49478268ce7710e2", "shader_group": MATERIAL_GROUP, "faulted": True}, + ) + data = _apply_page_fault([running, faulted]) + + tags = dict(data["tags"]) + assert tags["gpu.shader_group"] == MATERIAL_GROUP + assert tags["gpu.shader_hash"] == "49478268ce7710e2" + assert data["culprit"] == MATERIAL_GROUP + assert data["contexts"]["gpu_crash"]["faulting_pc"] == "0x50" + + +def test_page_fault_without_a_faulted_frame_names_no_shader() -> None: + running = _frame(data={"shader_hash": "49478268ce7710e2", "shader_group": MATERIAL_GROUP}) + data = _apply_page_fault([running]) + + tags = dict(data["tags"]) + assert "gpu.shader_group" not in tags and "gpu.shader_hash" not in tags + assert "culprit" not in data + assert "shader_group" not in data["contexts"]["gpu_crash"] + + +def test_decoder_name_is_kept_as_raw_function() -> None: + data = _apply(_frame(raw_function="fragment_01")) + + (applied,) = data["exception"]["values"][0]["stacktrace"]["frames"] + assert applied["function"] == "Main" + assert applied["raw_function"] == "fragment_01" diff --git a/tests/sentry/lang/native/test_teapot.py b/tests/sentry/lang/native/test_teapot.py index bfa0327fec5d..b2d713c67c28 100644 --- a/tests/sentry/lang/native/test_teapot.py +++ b/tests/sentry/lang/native/test_teapot.py @@ -7,6 +7,7 @@ import contextlib import hashlib import hmac +import logging from collections.abc import Iterator from typing import Any from unittest import mock @@ -540,6 +541,63 @@ def test_client_sends_presigned_urls(mock_objectstore: mock.Mock) -> None: } +def test_client_sends_sources_only_when_given(mock_objectstore: mock.Mock) -> None: + source = {"type": "sentry", "id": "sentry:project", "url": "http://s/dsyms/", "token": "t"} + with ( + _configured_teapot(), + mock.patch("sentry.lang.native.teapot.requests.post") as mock_post, + ): + mock_post.return_value = _FakeResponse(200, _completed_response()) + TeapotClient(_FakeProject(), "abc").symbolicate(_FakeAttachment(stored_id="dump")) + TeapotClient(_FakeProject(), "abc").symbolicate( + _FakeAttachment(stored_id="dump"), sources=[source] + ) + + without, with_sources = (orjson.loads(c.kwargs["data"]) for c in mock_post.call_args_list) + assert "sources" not in without + assert with_sources["sources"] == [source] + + +@pytest.mark.parametrize(("timeout", "max_decode_ms"), [(15, 13000), (2, 1000)]) +def test_client_gives_teapot_a_budget_below_its_timeout( + mock_objectstore: mock.Mock, timeout: int, max_decode_ms: int +) -> None: + with ( + _configured_teapot(), + mock.patch("sentry.lang.native.teapot._timeout", return_value=timeout), + mock.patch("sentry.lang.native.teapot.requests.post") as mock_post, + ): + mock_post.return_value = _FakeResponse(200, _completed_response()) + TeapotClient(_FakeProject(), "abc").symbolicate(_FakeAttachment(stored_id="dump")) + + kwargs = mock_post.call_args.kwargs + assert kwargs["timeout"] == timeout + assert orjson.loads(kwargs["data"])["options"] == {"max_decode_ms": max_decode_ms} + + +def test_client_redacts_source_tokens_from_errors( + mock_objectstore: mock.Mock, caplog: pytest.LogCaptureFixture +) -> None: + token = "system-token-0123456789abcdef0123456789abcdef" + source = {"type": "sentry", "id": "sentry:project", "url": "http://s/dsyms/", "token": token} + echoed = f'{{"status": "failed", "error": {{"message": "bad JSON body; last read: {token}"}}}}' + with ( + _configured_teapot(), + mock.patch("sentry.lang.native.teapot.requests.post") as mock_post, + caplog.at_level(logging.WARNING, logger="sentry.lang.native.teapot"), + ): + mock_post.return_value = _FakeResponse(400, echoed) + with pytest.raises(TeapotRequestError) as exc: + TeapotClient(_FakeProject(), "abc").symbolicate( + _FakeAttachment(stored_id="dump"), sources=[source] + ) + + assert token not in str(exc.value) + assert "" in str(exc.value) + (record,) = [r for r in caplog.records if r.getMessage() == "teapot.request_failed"] + assert token not in str(record.__dict__) + + def test_client_signs_request_when_secret_set(mock_objectstore: mock.Mock) -> None: from django.conf import settings diff --git a/tests/sentry/models/test_debugfile_shader.py b/tests/sentry/models/test_debugfile_shader.py new file mode 100644 index 000000000000..ca5ddde348df --- /dev/null +++ b/tests/sentry/models/test_debugfile_shader.py @@ -0,0 +1,277 @@ +from __future__ import annotations + +import io +from pathlib import Path +from unittest import mock + +import pytest + +from sentry.api.endpoints import debug_files +from sentry.api.endpoints.debug_files import batch_assemble +from sentry.api.serializers import serialize +from sentry.constants import KNOWN_DIF_FORMATS +from sentry.models.debugfile import ( + DIF_MIMETYPES, + SHADER_GROUP_MAX_LENGTH, + BadDif, + DifKind, + ProjectDebugFile, + _dif_file_extension, + _get_dif_object_name, + create_dif_from_fileobj, + detect_dif_from_path, + detect_single_dif_from_path, + determine_dif_kind, +) +from sentry.models.project import Project +from sentry.tasks.assemble import AssembleTask, ChunkFileState, set_assemble_status +from sentry.testutils.pytest.fixtures import django_db_all + +DXC_HASH = "731322bf682bdc554eefdd89098b7383" +DEBUG_ID = "731322bf-682b-dc55-4eef-dd89098b7383" +AFTERMATH_HASH = "0123456789abcdef" + +DXIL_BYTES = b"DXBC" + b"\x00" * 60 + + +def _write(tmp_path: Path, name: str, data: bytes) -> str: + path = tmp_path / name + path.write_bytes(data) + return str(path) + + +def test_dxil_format_registered() -> None: + assert KNOWN_DIF_FORMATS["application/x-dxil"] == "dxil" + assert DIF_MIMETYPES["dxil"] == "application/x-dxil" + assert _dif_file_extension("dxil", "dxil") == ".dxil" + + +def test_determine_dif_kind_dxil(tmp_path: Path) -> None: + assert determine_dif_kind(_write(tmp_path, "a.dxil", DXIL_BYTES)) == DifKind.Dxil + + +def test_detect_dxil(tmp_path: Path) -> None: + path = _write(tmp_path, "shader.dxil", DXIL_BYTES) + + (meta,) = detect_dif_from_path( + path, name="shader.dxil", debug_id=DXC_HASH, code_id=AFTERMATH_HASH.upper() + ) + + assert meta.file_format == "dxil" + assert meta.arch == "any" + assert meta.debug_id == DEBUG_ID + assert meta.code_id == AFTERMATH_HASH + assert meta.name == "shader.dxil" + assert meta.data == {"features": ["symtab"]} + assert _get_dif_object_name(meta) == "shader.dxil" + + +def test_detect_dxil_without_code_id(tmp_path: Path) -> None: + path = _write(tmp_path, "shader.dxil", DXIL_BYTES) + + meta = detect_single_dif_from_path(path, name="shader.dxil", debug_id=DEBUG_ID) + + assert meta.file_format == "dxil" + assert meta.code_id is None + + +def test_detect_dxil_requires_debug_id(tmp_path: Path) -> None: + path = _write(tmp_path, "shader.dxil", DXIL_BYTES) + + with pytest.raises(BadDif, match="Missing debug_id"): + detect_dif_from_path(path, name="shader.dxil") + + +def test_detect_dxil_rejects_invalid_debug_id(tmp_path: Path) -> None: + path = _write(tmp_path, "shader.dxil", DXIL_BYTES) + + with pytest.raises(BadDif, match="Invalid debug_id"): + detect_dif_from_path(path, name="shader.dxil", debug_id="not-a-debug-id") + + +def test_detect_dxil_rejects_invalid_code_id(tmp_path: Path) -> None: + path = _write(tmp_path, "shader.dxil", DXIL_BYTES) + + with pytest.raises(BadDif, match="Invalid code_id"): + detect_dif_from_path(path, name="shader.dxil", debug_id=DXC_HASH, code_id="abc") + + +@pytest.mark.parametrize( + "shader_group,stored", + ((" GlobalShaders ", "GlobalShaders"), (None, None), ("", None), (" ", None)), +) +def test_detect_dxil_stores_shader_group( + tmp_path: Path, shader_group: str | None, stored: str | None +) -> None: + path = _write(tmp_path, "shader.dxil", DXIL_BYTES) + + meta = detect_single_dif_from_path( + path, name="shader.dxil", debug_id=DXC_HASH, shader_group=shader_group + ) + + expected: dict[str, object] = {"features": ["symtab"]} + if stored is not None: + expected["shader_group"] = stored + assert meta.data == expected + + +def test_detect_dxil_rejects_overlong_shader_group(tmp_path: Path) -> None: + path = _write(tmp_path, "shader.dxil", DXIL_BYTES) + + with pytest.raises(BadDif, match="Invalid shader_group"): + detect_dif_from_path( + path, + name="shader.dxil", + debug_id=DXC_HASH, + shader_group="g" * (SHADER_GROUP_MAX_LENGTH + 1), + ) + + +def _create_shader_dif( + project: Project, + tmp_path: Path, + name: str, + data: bytes, + shader_group: str | None, + code_id: str | None = AFTERMATH_HASH, +) -> tuple[ProjectDebugFile, bool]: + path = _write(tmp_path, name, data) + meta = detect_single_dif_from_path( + path, name=name, debug_id=DXC_HASH, code_id=code_id, shader_group=shader_group + ) + session = mock.Mock() + session.put.return_value = f"difs/{name}" + with mock.patch("sentry.models.debugfile.get_session", return_value=session): + return create_dif_from_fileobj(project, meta, io.BytesIO(data)) + + +@django_db_all +def test_create_dxil_stores_shader_group(default_project: Project, tmp_path: Path) -> None: + dif, created = _create_shader_dif(default_project, tmp_path, "a.dxil", DXIL_BYTES, "Group A") + + assert created is True + dif.refresh_from_db() + assert dif.file_format == "dxil" + assert dif.data["shader_group"] == "Group A" + + +@django_db_all +def test_assemble_backfills_shader_group_for_known_checksum( + default_project: Project, tmp_path: Path +) -> None: + first, _ = _create_shader_dif(default_project, tmp_path, "a.dxil", DXIL_BYTES, None) + assert first.checksum is not None + + response = batch_assemble( + default_project, + { + first.checksum: { + "name": "a.dxil", + "debug_id": DEBUG_ID, + "chunks": [first.checksum], + "code_id": AFTERMATH_HASH, + "shader_group": "Group A", + } + }, + ) + + assert response[first.checksum]["state"] == ChunkFileState.OK + first.refresh_from_db() + assert first.data["shader_group"] == "Group A" + + +@django_db_all +def test_assemble_backfills_shader_group_despite_cached_ok_status( + default_project: Project, tmp_path: Path +) -> None: + first, _ = _create_shader_dif(default_project, tmp_path, "a.dxil", DXIL_BYTES, None) + assert first.checksum is not None + set_assemble_status( + AssembleTask.DIF, default_project.id, first.checksum, ChunkFileState.OK, detail=None + ) + + response = batch_assemble( + default_project, + { + first.checksum: { + "name": "a.dxil", + "debug_id": DEBUG_ID, + "chunks": [first.checksum], + "code_id": AFTERMATH_HASH, + "shader_group": "Group A", + } + }, + ) + + assert response[first.checksum]["state"] == ChunkFileState.OK + first.refresh_from_db() + assert first.data["shader_group"] == "Group A" + + +@pytest.mark.parametrize("cached", [False, True]) +@django_db_all +def test_assemble_backfills_code_id_for_known_checksum( + default_project: Project, tmp_path: Path, cached: bool +) -> None: + first, _ = _create_shader_dif( + default_project, tmp_path, "a.dxil", DXIL_BYTES, "Group A", code_id=None + ) + assert first.checksum is not None + if cached: + set_assemble_status( + AssembleTask.DIF, + default_project.id, + first.checksum, + ChunkFileState.OK, + serialize(first), + ) + + response = batch_assemble( + default_project, + { + first.checksum: { + "name": "a.dxil", + "debug_id": DEBUG_ID, + "chunks": [first.checksum], + "code_id": AFTERMATH_HASH.upper(), + "shader_group": "Group A", + } + }, + ) + + assert response[first.checksum]["state"] == ChunkFileState.OK + first.refresh_from_db() + assert first.code_id == AFTERMATH_HASH + assert list(ProjectDebugFile.objects.filter(code_id=AFTERMATH_HASH)) == [first] + + +@django_db_all +def test_assemble_reuses_cached_ok_status_when_group_is_stored( + default_project: Project, tmp_path: Path +) -> None: + first, _ = _create_shader_dif(default_project, tmp_path, "a.dxil", DXIL_BYTES, "Group A") + assert first.checksum is not None + set_assemble_status( + AssembleTask.DIF, default_project.id, first.checksum, ChunkFileState.OK, serialize(first) + ) + + with mock.patch.object( + debug_files, + "_find_existing_debug_files", + wraps=debug_files._find_existing_debug_files, + ) as find_existing: + response = batch_assemble( + default_project, + { + first.checksum: { + "name": "a.dxil", + "debug_id": DEBUG_ID, + "chunks": [first.checksum], + "code_id": AFTERMATH_HASH, + "shader_group": "Group A", + } + }, + ) + + assert response[first.checksum]["state"] == ChunkFileState.OK + assert list(find_existing.call_args.kwargs["checksums"]) == [] diff --git a/tests/sentry/tasks/test_gpu_crash.py b/tests/sentry/tasks/test_gpu_crash.py index d460179b79ce..979f7931b16a 100644 --- a/tests/sentry/tasks/test_gpu_crash.py +++ b/tests/sentry/tasks/test_gpu_crash.py @@ -48,7 +48,7 @@ def circuit_breaker() -> Iterator[mock.Mock]: def test_try_symbolicate_happy_path(default_project: Project, circuit_breaker: mock.Mock) -> None: data: dict[str, Any] = {"event_id": "e", "project": default_project.id} with ( - override_options({"teapot.enabled": True}), + override_options({"teapot.enabled": True, "teapot.shader-sources.enabled": True}), mock.patch(FIND_DUMP, return_value=_FakeAttachment()), mock.patch(FIND_SHADERS, return_value=[]), mock.patch(SUBMIT_TEAPOT, return_value=_completed()) as teapot, @@ -59,12 +59,36 @@ def test_try_symbolicate_happy_path(default_project: Project, circuit_breaker: m assert changed is True assert teapot.call_count == 1 assert apply.call_count == 1 + (source,) = teapot.call_args.kwargs["sources"] + assert source["type"] == "sentry" + assert ( + f"/projects/{default_project.organization.slug}/{default_project.slug}/files/dsyms/" + in (source["url"]) + ) + assert source["token"] # A successful decode closes the loop on the breaker. circuit_breaker.record_success.assert_called_once() circuit_breaker.record_error.assert_not_called() +@django_db_all +def test_try_symbolicate_sends_no_sources_by_default( + default_project: Project, circuit_breaker: mock.Mock +) -> None: + data: dict[str, Any] = {"event_id": "e", "project": default_project.id} + with ( + override_options({"teapot.enabled": True}), + mock.patch(FIND_DUMP, return_value=_FakeAttachment()), + mock.patch(FIND_SHADERS, return_value=[]), + mock.patch(SUBMIT_TEAPOT, return_value=_completed()) as teapot, + mock.patch(APPLY, return_value=data), + ): + _try_symbolicate(data, default_project.id, "e") + + assert teapot.call_args.kwargs["sources"] is None + + @django_db_all def test_try_symbolicate_skips_when_circuit_open( default_project: Project, circuit_breaker: mock.Mock