diff --git a/CHANGELOG.md b/CHANGELOG.md index 4da51d4..4aaa845 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,10 @@ # Changelog +## 1.3.5 (2026-07-14) +* Expose span status in the trace view: populate `kind`, `statusCode`, and `statusMessage` data frame fields, derived from Cloud Trace span labels (`/error/message`, `/error/name`, and HTTP status code labels per OpenTelemetry HTTP semantics) +* Mark failed spans with an `error: true` tag so they are highlighted with the error icon in the Grafana trace view +* Show the span's `/stacktrace` label in the "Stack Traces" section via the `stackTraces` field +* Document that the Cloud Trace v1 read API does not return span events (`TimeEvents`), so error details are only available from span labels + ## 1.3.4 (2026-05-08) * Upgrade @grafana/google-sdk from 0.0.3 to 0.3.5 to fix ConnectionConfig mutating React props ([#44](https://github.com/GoogleCloudPlatform/cloud-trace-data-source-plugin/issues/44)) * Upgrade @grafana/data, @grafana/runtime, @grafana/ui to ^11.0.0 diff --git a/README.md b/README.md index 257151b..4cece75 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,22 @@ You can combine multiple patterns (one per line) to match the union of all patte After making a `Filter` query, a table will be displayed with all of the matching traces (Example: `http.scheme:http http.server_name:testserver MinLatency:500ms`) +### Span status and errors + +The trace view marks spans as failed (error icon and `statusCode`/`statusMessage` fields) based on +the span's Cloud Trace labels: + +- `/error/message` or `/error/name`: the span is marked as an error, using the label value as the + status message +- `/http/status_code`, `http.status_code`, or `http.response.status_code`: the span is marked as an + error for status codes >= 500 (>= 400 for client spans, per OpenTelemetry HTTP semantics) + +If the span has a `/stacktrace` label, its value is shown in the span's "Stack Traces" section. + +Note: the Cloud Trace v1 read API used by this plugin does not return span events (`TimeEvents`), +so events such as OpenTelemetry exceptions attached to a span cannot be displayed. Error details +are only available when they are present as span labels. + ### Annotations The plugin supports Grafana annotations. You can use trace queries as annotation sources to overlay trace data on your dashboards. diff --git a/package.json b/package.json index f41f731..84211e1 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "googlecloud-trace-datasource", - "version": "1.3.4", + "version": "1.3.5", "description": "Backend Grafana plugin that enables visualization of GCP Cloud Trace traces and spans in Grafana.", "scripts": { "postinstall": "rm -rf node_modules/flatted/golang", diff --git a/pkg/plugin/cloudtrace/cloudtrace.go b/pkg/plugin/cloudtrace/cloudtrace.go index 45bfc79..e7308ab 100644 --- a/pkg/plugin/cloudtrace/cloudtrace.go +++ b/pkg/plugin/cloudtrace/cloudtrace.go @@ -18,6 +18,7 @@ import ( "encoding/json" "fmt" "regexp" + "strconv" "strings" "time" @@ -32,6 +33,21 @@ const ( gaeServiceVersionKey = "g.co/gae/app/version" otelMethodKey = "http.method" cloudTraceMethodKey = "/http/method" + + errorMessageKey = "/error/message" + errorNameKey = "/error/name" + cloudTraceStatusKey = "/http/status_code" + otelStatusKey = "http.status_code" + otelHTTPResponseStatus = "http.response.status_code" + stackTraceKey = "/stacktrace" + errorTagKey = "error" +) + +// Span status codes as expected by the Grafana trace schema, +// following the OpenTelemetry convention +const ( + StatusCodeUnset int64 = 0 + StatusCodeError int64 = 2 ) // Regex for individual filters within query text @@ -90,13 +106,21 @@ func GetSpanOperationName(span *tracepb.TraceSpan) string { // GetTags converts Google Trace labels to Grafana service and span tags func GetTags(span *tracepb.TraceSpan) (serviceTags json.RawMessage, spanTags json.RawMessage, err error) { spanLabels := span.GetLabels() - serviceTagsMapArray := []map[string]string{} - spanTagsMapArray := []map[string]string{} + serviceTagsMapArray := []map[string]any{} + spanTagsMapArray := []map[string]any{} for key, value := range spanLabels { if strings.HasPrefix(key, servicePrefix) || strings.HasPrefix(key, gaeServicePrefix) { - serviceTagsMapArray = append(serviceTagsMapArray, map[string]string{"key": key, "value": value}) + serviceTagsMapArray = append(serviceTagsMapArray, map[string]any{"key": key, "value": value}) } else { - spanTagsMapArray = append(spanTagsMapArray, map[string]string{"key": key, "value": value}) + spanTagsMapArray = append(spanTagsMapArray, map[string]any{"key": key, "value": value}) + } + } + + // Grafana's trace view highlights spans that carry an `error: true` tag, + // so mark failed spans for versions that don't key off the statusCode field + if code, _ := GetStatus(span); code == StatusCodeError { + if _, ok := spanLabels[errorTagKey]; !ok { + spanTagsMapArray = append(spanTagsMapArray, map[string]any{"key": errorTagKey, "value": true}) } } @@ -113,6 +137,75 @@ func GetTags(span *tracepb.TraceSpan) (serviceTags json.RawMessage, spanTags jso return serviceTags, spanTags, nil } +// GetStatus derives a Grafana span status code and message from the span's +// labels. The Cloud Trace v1 read API has no structured status field, so +// error information is only available through well-known labels: the +// canonical `/error/*` labels, or an HTTP status code label indicating a +// failed request. +func GetStatus(span *tracepb.TraceSpan) (int64, string) { + labels := span.GetLabels() + + errMsg := labels[errorMessageKey] + errName := labels[errorNameKey] + if errMsg != "" || errName != "" { + if errMsg == "" { + errMsg = errName + } + return StatusCodeError, errMsg + } + + httpStatus := labels[cloudTraceStatusKey] + if httpStatus == "" { + httpStatus = labels[otelStatusKey] + } + if httpStatus == "" { + httpStatus = labels[otelHTTPResponseStatus] + } + if code, err := strconv.Atoi(httpStatus); err == nil { + // Per OpenTelemetry HTTP semantics, 4xx responses are errors for + // client spans but not for server spans + threshold := 500 + if span.GetKind() == tracepb.TraceSpan_RPC_CLIENT { + threshold = 400 + } + if code >= threshold { + return StatusCodeError, fmt.Sprintf("HTTP %d", code) + } + } + + return StatusCodeUnset, "" +} + +// GetSpanKind maps the Cloud Trace span kind to the values expected by the +// Grafana trace schema +func GetSpanKind(span *tracepb.TraceSpan) string { + switch span.GetKind() { + case tracepb.TraceSpan_RPC_SERVER: + return "server" + case tracepb.TraceSpan_RPC_CLIENT: + return "client" + default: + return "" + } +} + +// GetStackTraces returns the span's `/stacktrace` label as the JSON string +// array expected by the Grafana trace schema, or nil if the span has none +func GetStackTraces(span *tracepb.TraceSpan) *json.RawMessage { + st := span.GetLabels()[stackTraceKey] + if st == "" { + return nil + } + + b, err := json.Marshal([]string{st}) + if err != nil { + return nil + } + + raw := json.RawMessage(b) + return &raw +} + // GetListTracesFilter takes the raw query text from a user and converts it // to a filter string as expected by the Cloud Trace API func GetListTracesFilter(queryText string) (string, error) { diff --git a/pkg/plugin/cloudtrace/cloudtrace_test.go b/pkg/plugin/cloudtrace/cloudtrace_test.go index daf4b2c..ac696e8 100644 --- a/pkg/plugin/cloudtrace/cloudtrace_test.go +++ b/pkg/plugin/cloudtrace/cloudtrace_test.go @@ -451,3 +451,199 @@ func TestGetListTracesFilter(t *testing.T) { }) } } + +func TestGetStatus(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + span *tracepb.TraceSpan + expectedCode int64 + expectedMessage string + }{ + { + name: "Span with no labels", + span: &tracepb.TraceSpan{}, + expectedCode: cloudtrace.StatusCodeUnset, + expectedMessage: "", + }, + { + name: "Span with error message label", + span: &tracepb.TraceSpan{ + Labels: map[string]string{ + "/error/message": "something went wrong", + "/error/name": "RuntimeError", + }, + }, + expectedCode: cloudtrace.StatusCodeError, + expectedMessage: "something went wrong", + }, + { + name: "Span with only error name label", + span: &tracepb.TraceSpan{ + Labels: map[string]string{ + "/error/name": "RuntimeError", + }, + }, + expectedCode: cloudtrace.StatusCodeError, + expectedMessage: "RuntimeError", + }, + { + name: "Server span with HTTP 500", + span: &tracepb.TraceSpan{ + Kind: tracepb.TraceSpan_RPC_SERVER, + Labels: map[string]string{ + "/http/status_code": "500", + }, + }, + expectedCode: cloudtrace.StatusCodeError, + expectedMessage: "HTTP 500", + }, + { + name: "Server span with HTTP 404 is not an error", + span: &tracepb.TraceSpan{ + Kind: tracepb.TraceSpan_RPC_SERVER, + Labels: map[string]string{ + "/http/status_code": "404", + }, + }, + expectedCode: cloudtrace.StatusCodeUnset, + expectedMessage: "", + }, + { + name: "Client span with HTTP 404 is an error", + span: &tracepb.TraceSpan{ + Kind: tracepb.TraceSpan_RPC_CLIENT, + Labels: map[string]string{ + "/http/status_code": "404", + }, + }, + expectedCode: cloudtrace.StatusCodeError, + expectedMessage: "HTTP 404", + }, + { + name: "Span with OTel status code label", + span: &tracepb.TraceSpan{ + Labels: map[string]string{ + "http.status_code": "503", + }, + }, + expectedCode: cloudtrace.StatusCodeError, + expectedMessage: "HTTP 503", + }, + { + name: "Span with new OTel semconv status code label", + span: &tracepb.TraceSpan{ + Labels: map[string]string{ + "http.response.status_code": "502", + }, + }, + expectedCode: cloudtrace.StatusCodeError, + expectedMessage: "HTTP 502", + }, + { + name: "Span with successful HTTP status", + span: &tracepb.TraceSpan{ + Labels: map[string]string{ + "/http/status_code": "200", + }, + }, + expectedCode: cloudtrace.StatusCodeUnset, + expectedMessage: "", + }, + { + name: "Span with unparsable HTTP status", + span: &tracepb.TraceSpan{ + Labels: map[string]string{ + "/http/status_code": "abc", + }, + }, + expectedCode: cloudtrace.StatusCodeUnset, + expectedMessage: "", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + code, message := cloudtrace.GetStatus(tc.span) + + require.Equal(t, tc.expectedCode, code) + require.Equal(t, tc.expectedMessage, message) + }) + } +} + +func TestGetSpanKind(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + span *tracepb.TraceSpan + expectedKind string + }{ + { + name: "Unspecified span kind", + span: &tracepb.TraceSpan{}, + expectedKind: "", + }, + { + name: "Server span", + span: &tracepb.TraceSpan{Kind: tracepb.TraceSpan_RPC_SERVER}, + expectedKind: "server", + }, + { + name: "Client span", + span: &tracepb.TraceSpan{Kind: tracepb.TraceSpan_RPC_CLIENT}, + expectedKind: "client", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.expectedKind, cloudtrace.GetSpanKind(tc.span)) + }) + } +} + +func TestGetStackTraces(t *testing.T) { + t.Parallel() + + t.Run("Span without stack trace", func(t *testing.T) { + require.Nil(t, cloudtrace.GetStackTraces(&tracepb.TraceSpan{})) + }) + + t.Run("Span with stack trace", func(t *testing.T) { + span := &tracepb.TraceSpan{ + Labels: map[string]string{ + "/stacktrace": `{"stack_frame":[{"method_name":"main"}]}`, + }, + } + + stackTraces := cloudtrace.GetStackTraces(span) + require.NotNil(t, stackTraces) + + var traces []string + require.NoError(t, json.Unmarshal(*stackTraces, &traces)) + require.Equal(t, []string{`{"stack_frame":[{"method_name":"main"}]}`}, traces) + }) +} + +func TestGetTagsErrorSpan(t *testing.T) { + t.Parallel() + + span := &tracepb.TraceSpan{ + Labels: map[string]string{ + "/error/message": "something went wrong", + }, + } + + _, spanTags, err := cloudtrace.GetTags(span) + require.NoError(t, err) + + var spanTagsMap []map[string]any + require.NoError(t, json.Unmarshal(spanTags, &spanTagsMap)) + require.ElementsMatch(t, []map[string]any{ + {"key": "/error/message", "value": "something went wrong"}, + {"key": "error", "value": true}, + }, spanTagsMap) +} diff --git a/pkg/plugin/plugin.go b/pkg/plugin/plugin.go index f9c33b4..30dbff9 100644 --- a/pkg/plugin/plugin.go +++ b/pkg/plugin/plugin.go @@ -381,6 +381,10 @@ func createTraceSpanFrame(trace *tracepb.Trace) *data.Frame { startTimeField := data.NewField("startTime", nil, []time.Time{}) durationField := data.NewField("duration", nil, []float64{}) tagsField := data.NewField("tags", nil, []json.RawMessage{}) + kindField := data.NewField("kind", nil, []string{}) + statusCodeField := data.NewField("statusCode", nil, []int64{}) + statusMessageField := data.NewField("statusMessage", nil, []string{}) + stackTracesField := data.NewField("stackTraces", nil, []*json.RawMessage{}) // Add values to each field for each span for _, s := range trace.Spans { @@ -400,6 +404,12 @@ func createTraceSpanFrame(trace *tracepb.Trace) *data.Frame { startTimeField.Append(s.GetStartTime().AsTime()) duration := float64(s.GetEndTime().AsTime().UnixMicro()-s.GetStartTime().AsTime().UnixMicro()) / 1000 durationField.Append(duration) + + statusCode, statusMessage := cloudtrace.GetStatus(s) + statusCodeField.Append(statusCode) + statusMessageField.Append(statusMessage) + kindField.Append(cloudtrace.GetSpanKind(s)) + stackTracesField.Append(cloudtrace.GetStackTraces(s)) } f.Fields = append(f.Fields, @@ -412,6 +422,10 @@ func createTraceSpanFrame(trace *tracepb.Trace) *data.Frame { tagsField, startTimeField, durationField, + kindField, + statusCodeField, + statusMessageField, + stackTracesField, ) return f diff --git a/pkg/plugin/plugin_test.go b/pkg/plugin/plugin_test.go index 998bfe4..7546a5f 100644 --- a/pkg/plugin/plugin_test.go +++ b/pkg/plugin/plugin_test.go @@ -230,10 +230,10 @@ func TestQueryData_SingleTraceSpans(t *testing.T) { traceFrame := resp.Responses[refID].Frames[0] require.Equal(t, traceID, traceFrame.Name) - require.Len(t, traceFrame.Fields, 9) + require.Len(t, traceFrame.Fields, 13) require.Equal(t, data.VisTypeTrace, string(traceFrame.Meta.PreferredVisualization)) - expectedFrame := []byte(`{"schema":{"name":"123","meta":{"typeVersion":[0,0],"preferredVisualisationType":"trace"},"fields":[{"name":"traceID","type":"string","typeInfo":{"frame":"string"}},{"name":"parentSpanID","type":"string","typeInfo":{"frame":"string"}},{"name":"spanID","type":"string","typeInfo":{"frame":"string"}},{"name":"serviceName","type":"string","typeInfo":{"frame":"string"}},{"name":"operationName","type":"string","typeInfo":{"frame":"string"}},{"name":"serviceTags","type":"other","typeInfo":{"frame":"json.RawMessage"}},{"name":"tags","type":"other","typeInfo":{"frame":"json.RawMessage"}},{"name":"startTime","type":"time","typeInfo":{"frame":"time.Time"}},{"name":"duration","type":"number","typeInfo":{"frame":"float64"}}]},"data":{"values":[["123"],["0"],["1"],[""],["spanName"],[[]],[[{"key":"key1","value":"value1"}]],[1660920349373],[1]]}}`) + expectedFrame := []byte(`{"schema":{"name":"123","meta":{"typeVersion":[0,0],"preferredVisualisationType":"trace"},"fields":[{"name":"traceID","type":"string","typeInfo":{"frame":"string"}},{"name":"parentSpanID","type":"string","typeInfo":{"frame":"string"}},{"name":"spanID","type":"string","typeInfo":{"frame":"string"}},{"name":"serviceName","type":"string","typeInfo":{"frame":"string"}},{"name":"operationName","type":"string","typeInfo":{"frame":"string"}},{"name":"serviceTags","type":"other","typeInfo":{"frame":"json.RawMessage"}},{"name":"tags","type":"other","typeInfo":{"frame":"json.RawMessage"}},{"name":"startTime","type":"time","typeInfo":{"frame":"time.Time"}},{"name":"duration","type":"number","typeInfo":{"frame":"float64"}},{"name":"kind","type":"string","typeInfo":{"frame":"string"}},{"name":"statusCode","type":"number","typeInfo":{"frame":"int64"}},{"name":"statusMessage","type":"string","typeInfo":{"frame":"string"}},{"name":"stackTraces","type":"other","typeInfo":{"frame":"json.RawMessage","nullable":true}}]},"data":{"values":[["123"],["0"],["1"],[""],["spanName"],[[]],[[{"key":"key1","value":"value1"}]],[1660920349373],[1],["server"],[0],[""],[null]]}}`) serializedFrame, err := traceFrame.MarshalJSON() require.NoError(t, err) @@ -242,6 +242,84 @@ func TestQueryData_SingleTraceSpans(t *testing.T) { client.AssertExpectations(t) } +func TestQueryData_TraceSpansWithError(t *testing.T) { + to := time.Now() + from := to.Add(-1 * time.Hour) + traceID := "456" + startTime := timestamppb.New(time.UnixMilli(1660920349373)) + endTime := timestamppb.New(time.UnixMilli(1660920349374)) + stackTrace := `{"stack_frame":[{"method_name":"main"}]}` + + spans := []*tracepb.TraceSpan{ + { + SpanId: 1, + Kind: tracepb.TraceSpan_RPC_CLIENT, + Name: "failedSpan", + StartTime: startTime, + EndTime: endTime, + Labels: map[string]string{ + "/error/message": "something went wrong", + "/stacktrace": stackTrace, + }, + }, + } + trace := tracepb.Trace{ + ProjectId: "testProject", + TraceId: traceID, + Spans: spans, + } + + client := mocks.NewAPI(t) + client.On("GetTrace", mock.Anything, &cloudtrace.TraceQuery{ + ProjectID: "testing", + TraceID: traceID, + }).Return(&trace, nil) + client.On("Close").Return(nil) + + ds := CloudTraceDatasource{ + client: client, + } + refID := "test" + resp, err := ds.QueryData(context.Background(), &backend.QueryDataRequest{ + Queries: []backend.DataQuery{ + { + JSON: []byte(`{"projectId": "testing", "queryType": "traceID", "traceId": "456"}`), + RefID: refID, + TimeRange: backend.TimeRange{ + From: from, + To: to, + }, + MaxDataPoints: 20, + }, + }, + }) + ds.Dispose() + require.NoError(t, err) + require.Len(t, resp.Responses[refID].Frames, 1) + + traceFrame := resp.Responses[refID].Frames[0] + fieldsByName := map[string]*data.Field{} + for _, field := range traceFrame.Fields { + fieldsByName[field.Name] = field + } + + require.Equal(t, int64(2), fieldsByName["statusCode"].At(0)) + require.Equal(t, "something went wrong", fieldsByName["statusMessage"].At(0)) + require.Equal(t, "client", fieldsByName["kind"].At(0)) + + stackTraces := fieldsByName["stackTraces"].At(0).(*json.RawMessage) + require.NotNil(t, stackTraces) + var traces []string + require.NoError(t, json.Unmarshal(*stackTraces, &traces)) + require.Equal(t, []string{stackTrace}, traces) + + var spanTags []map[string]any + require.NoError(t, json.Unmarshal(fieldsByName["tags"].At(0).(json.RawMessage), &spanTags)) + require.Contains(t, spanTags, map[string]any{"key": "error", "value": true}) + + client.AssertExpectations(t) +} + func TestQueryData_SingleTraceTable(t *testing.T) { to := time.Now() from := to.Add(-1 * time.Hour)