Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 51 additions & 1 deletion tools/run_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -443,16 +443,59 @@ def parse_perf_data(op, result_file):
except (json.JSONDecodeError, ValueError):
return {"status": "Error", "reason": f"Invalid JSON in {result_file}"}

# The benchmark output may contain multiple top-level keys (e.g. a new op
# name AND a legacy identifier for the same operator). We first match by
# the exact op name; if that record lacks performance details we scan all
# remaining keys for the first one that *does* carry details, so that the
# parsed data block is always populated regardless of which key pytest's
# JSON plugin attributed the details to.
data = raw_data.get(op, {})
if not data:
return {"status": "NotFound"}

# If the primary key has no details, try to find them on a sibling key.
# This also covers cases where the primary key has result="failed" but a
# legacy key still recorded partial benchmark data.
if not data.get("details"):
for alt_key, alt_value in raw_data.items():
if alt_key == op:
continue
alt_details = alt_value.get("details")
if alt_details:
data = dict(data) # shallow copy to leave original intact
data["details"] = alt_details
break

result = data.get("result", "NotFound")
if result in ("failed", "skipped"):
# Still try to populate bench data so summary.json is not empty.
bench_res = {}
records = data.get("details", [])
for item in records:
dtype = DTYPE_MAP.get(item.get("dtype", ""), item.get("dtype", ""))
details = {}
total = 0.0
count = 0
for res in item.get("result", []):
shape = str(res.get("shape_detail", "Unknown")).replace(" ", "")
details.setdefault(shape, {})
details[shape]["base"] = res.get("latency_base", 0.0)
details[shape]["gems"] = res.get("latency", 0.0)
speedup = res.get("speedup", 0.0)
details[shape]["speedup"] = speedup
count += 1
total += speedup
if details:
bench_res[dtype] = {
"result": "OK",
"details": details,
"speedup": total / count,
}
return {
"status": result.title(),
"reason": data.get("reason", "Unknown"),
"test_case": data.get("test_case", "Unknown"),
"data": bench_res,
}

bench_res = {}
Expand Down Expand Up @@ -484,8 +527,15 @@ def parse_perf_data(op, result_file):
"result": "Unknown", "details": {}, "speedup": 0,
}

# Use the primary key's result status ("passed"/"failed"), but fall back
# to "Passed" when bench_res has valid data even if the primary key
# doesn't carry an explicit result field.
status = (
result.title() if result not in ("NotFound",)
else "Passed" if bench_res else "NotFound"
)
return {
"status": result.title(),
"status": status,
"data": bench_res,
"test_case": data.get("test_case", "Unknown"),
}
Expand Down
Loading