From a421a04aae0bd30c8658ef1326b20ded8da53b9e Mon Sep 17 00:00:00 2001 From: bdchatham Date: Sat, 20 Jun 2026 11:24:23 -0700 Subject: [PATCH 1/2] feat(sidecar): reason-labeled divergence metrics + report digest analyzer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The lightweight operational surface: proactively alert on what structurally diverged, and a simple tool to pull the concrete S3 record for investigation. - shadow: add a `reason` label to seictl_shadow_divergences_total via a new ReasonFor(*CompareResult) — deepest-layer-first (layer2- / layer1-results / layer1-receipt / layer0-apphash), with layer{1,2}-indeterminate so a fail-closed indeterminate never reads as a header mismatch. - digest: add seictl_evm_logical_digest_runs_total{normalization,result} (the oracle-liveness signal) and _mismatches_total{normalization,reason=bucket|final}, closing the digest task's alertability gap. - report: extend the renderer to endpoint-digest records — `report list` lists them and a new `report digest` subcommand fetches + renders an EndpointDigestRecord (overall + per-bucket match, axes_proved). - fix a pre-existing doc-drift: StateDivergence.Kind comment omitted `balance`. Cross-reviewed (systems-engineer: ship, no correctness/cardinality defects; idiomatic: reads native). Reason values + the mismatch fallback are tested; the report renderer is fail-closed on malformed input. Co-Authored-By: Claude Opus 4.8 --- report.go | 184 ++++++++++++++++++++++++++++ report_list.go | 31 ++++- report_list_test.go | 57 +++++++++ report_test.go | 48 ++++++++ sidecar/shadow/comparator.go | 39 ++++++ sidecar/shadow/metrics.go | 6 +- sidecar/shadow/reason_test.go | 85 +++++++++++++ sidecar/shadow/types.go | 2 +- sidecar/tasks/evm_logical_digest.go | 1 + sidecar/tasks/metrics.go | 61 +++++++++ sidecar/tasks/metrics_test.go | 84 +++++++++++++ sidecar/tasks/result_compare.go | 4 +- sidecar/tasks/result_export_test.go | 6 +- 13 files changed, 598 insertions(+), 10 deletions(-) create mode 100644 sidecar/shadow/reason_test.go create mode 100644 sidecar/tasks/metrics.go create mode 100644 sidecar/tasks/metrics_test.go diff --git a/report.go b/report.go index f1fe17d..7d7c3b1 100644 --- a/report.go +++ b/report.go @@ -1,15 +1,21 @@ package main import ( + "compress/gzip" "context" "encoding/json" "fmt" + "io" "os" + "sort" + "strings" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/urfave/cli/v3" seis3 "github.com/sei-protocol/seictl/sidecar/s3" "github.com/sei-protocol/seictl/sidecar/shadow" + "github.com/sei-protocol/seictl/sidecar/tasks" ) var reportCmd = cli.Command{ @@ -17,6 +23,7 @@ var reportCmd = cli.Command{ Usage: "Analyze shadow chain comparison data", Commands: []*cli.Command{ &reportDivergenceCmd, + &reportDigestCmd, &reportListCmd, }, } @@ -114,6 +121,183 @@ func runReportDivergence(ctx context.Context, cmd *cli.Command) error { return nil } +var reportDigestCmd = cli.Command{ + Name: "digest", + Usage: "Fetch and render an evm-logical-digest record from S3", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "env", + Usage: "Environment shorthand (expands to '{env}-sei-shadow-results')", + }, + &cli.StringFlag{ + Name: "bucket", + Sources: cli.EnvVars("SEI_RESULT_EXPORT_BUCKET"), + Usage: "S3 bucket containing the record", + }, + &cli.StringFlag{ + Name: "key", + Usage: "S3 object key (e.g. shadow-results/endpoint-digest-198740042-semantic.json.gz)", + }, + &cli.IntFlag{ + Name: "height", + Usage: "Block height of the digest record (used with --normalization, alternative to --key)", + }, + &cli.StringFlag{ + Name: "normalization", + Usage: "Memiavl normalization of the digest record (used with --height)", + }, + &cli.StringFlag{ + Name: "prefix", + Sources: cli.EnvVars("SEI_RESULT_EXPORT_PREFIX"), + Usage: "S3 key prefix (used with --height to compute key)", + Value: "shadow-results/", + }, + &cli.StringFlag{ + Name: "region", + Sources: cli.EnvVars("SEI_RESULT_EXPORT_REGION"), + Usage: "AWS region", + Value: "eu-central-1", + }, + &cli.BoolFlag{ + Name: "json", + Usage: "Output raw JSON instead of markdown", + }, + }, + Action: runReportDigest, +} + +func runReportDigest(ctx context.Context, cmd *cli.Command) error { + if cmd.IsSet("key") && cmd.IsSet("height") { + return fmt.Errorf("--key and --height are mutually exclusive") + } + + bucket := cmd.String("bucket") + key := cmd.String("key") + region := cmd.String("region") + prefix := cmd.String("prefix") + + if cmd.IsSet("env") || cmd.IsSet("height") { + resolved, resolvedPrefix, resolvedRegion, err := resolveS3Ref( + cmd.String("env"), bucket, prefix, region, + ) + if err != nil { + return err + } + bucket = resolved + region = resolvedRegion + if cmd.IsSet("height") { + if !cmd.IsSet("normalization") { + return fmt.Errorf("--normalization is required with --height") + } + key = fmt.Sprintf("%sendpoint-digest-%d-%s.json.gz", + resolvedPrefix, cmd.Int("height"), cmd.String("normalization")) + } + } + + if bucket == "" { + return fmt.Errorf("one of --env or --bucket is required") + } + if key == "" { + return fmt.Errorf("one of --key or --height is required") + } + + downloader, err := seis3.DefaultDownloaderFactory(ctx, region) + if err != nil { + return err + } + + record, err := fetchDigestRecord(ctx, downloader, bucket, key) + if err != nil { + return err + } + + if cmd.Bool("json") { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + return enc.Encode(record) + } + + fmt.Print(renderDigestMarkdown(record)) + return nil +} + +// fetchDigestRecord downloads and decodes an EndpointDigestRecord from S3, +// gunzipping when the key is .gz — mirroring shadow.FetchReport for the digest +// artifact the evm-logical-digest task publishes. +func fetchDigestRecord(ctx context.Context, downloader seis3.Downloader, bucket, key string) (*tasks.EndpointDigestRecord, error) { + resp, err := downloader.GetObject(ctx, &s3.GetObjectInput{ + Bucket: &bucket, + Key: &key, + }) + if err != nil { + return nil, fmt.Errorf("downloading s3://%s/%s: %w", bucket, key, err) + } + defer resp.Body.Close() + + var reader io.Reader = resp.Body + if strings.HasSuffix(key, ".gz") { + gz, err := gzip.NewReader(resp.Body) + if err != nil { + return nil, fmt.Errorf("decompressing digest record: %w", err) + } + defer gz.Close() + reader = gz + } + + var record tasks.EndpointDigestRecord + if err := json.NewDecoder(reader).Decode(&record); err != nil { + return nil, fmt.Errorf("decoding digest record: %w", err) + } + return &record, nil +} + +// renderDigestMarkdown produces a human-readable view of an EndpointDigestRecord +// in the same shape as the divergence report renderer: header, overall verdict, +// per-bucket flatkv/memiavl digests, and the axes the digest proves. +func renderDigestMarkdown(r *tasks.EndpointDigestRecord) string { + var b strings.Builder + + fmt.Fprintf(&b, "# EVM Logical Digest — Height %d (%s)\n\n", r.Height, r.Normalization) + fmt.Fprintf(&b, "**Generated at:** %s\n\n", r.GeneratedAt) + fmt.Fprintf(&b, "**Overall match:** %s\n\n", matchIcon(r.Match)) + fmt.Fprintf(&b, "| Backend | Final Digest |\n") + fmt.Fprintf(&b, "|---------|--------------|\n") + fmt.Fprintf(&b, "| flatkv | %s |\n", truncateDigest(r.FlatKVDigest)) + fmt.Fprintf(&b, "| memiavl | %s |\n\n", truncateDigest(r.MemIAVLDigest)) + + fmt.Fprintf(&b, "## Per-Bucket Digests\n\n") + fmt.Fprintf(&b, "| Bucket | FlatKV | MemIAVL | Match |\n") + fmt.Fprintf(&b, "|--------|--------|---------|-------|\n") + for _, name := range []string{"account", "code", "storage", "legacy"} { + bkt, ok := r.PerBucket[name] + if !ok { + continue + } + fmt.Fprintf(&b, "| %s | %s | %s | %s |\n", + name, truncateDigest(bkt.FlatKV), truncateDigest(bkt.MemIAVL), matchIcon(bkt.Match)) + } + fmt.Fprintf(&b, "\n") + + axes := append([]string(nil), r.AxesProved...) + sort.Strings(axes) + fmt.Fprintf(&b, "**Axes proved:** %s\n", strings.Join(axes, ", ")) + return b.String() +} + +func matchIcon(match bool) string { + if match { + return "✅" + } + return "❌" +} + +func truncateDigest(h string) string { + if len(h) <= 16 { + return h + } + return h[:8] + "..." + h[len(h)-4:] +} + // resolveS3Ref converts --env/--bucket/--prefix/--region flags to concrete values. func resolveS3Ref(env, bucket, prefix, region string) (string, string, string, error) { switch { diff --git a/report_list.go b/report_list.go index 07840f2..f6b511f 100644 --- a/report_list.go +++ b/report_list.go @@ -20,6 +20,7 @@ import ( var ( comparePageRe = regexp.MustCompile(`(\d+)-(\d+)\.compare\.ndjson\.gz$`) divergenceReportRe = regexp.MustCompile(`divergence-(\d+)\.report\.json\.gz$`) + endpointDigestRe = regexp.MustCompile(`endpoint-digest-(\d+)-(\w+)\.json\.gz$`) ) var reportListCmd = cli.Command{ @@ -58,6 +59,7 @@ var reportListCmd = cli.Command{ type listOutput struct { Pages []pageEntry `json:"pages"` DivergenceReports []divergenceEntry `json:"divergenceReports"` + DigestRecords []digestEntry `json:"digestRecords"` TotalBlocks int64 `json:"totalBlocks"` } @@ -73,6 +75,12 @@ type divergenceEntry struct { Height int64 `json:"height"` } +type digestEntry struct { + Key string `json:"key"` + Height int64 `json:"height"` + Normalization string `json:"normalization"` +} + func runReportList(ctx context.Context, cmd *cli.Command) error { bucket, prefix, region, err := resolveS3Ref( cmd.String("env"), cmd.String("bucket"), cmd.String("prefix"), cmd.String("region"), @@ -94,6 +102,7 @@ func runReportList(ctx context.Context, cmd *cli.Command) error { var pages []pageEntry var reports []divergenceEntry + var digests []digestEntry for { resp, err := lister.ListObjectsV2(ctx, input) @@ -112,6 +121,9 @@ func runReportList(ctx context.Context, cmd *cli.Command) error { } else if m := divergenceReportRe.FindStringSubmatch(key); len(m) >= 2 { height, _ := strconv.ParseInt(m[1], 10, 64) reports = append(reports, divergenceEntry{Key: key, Height: height}) + } else if m := endpointDigestRe.FindStringSubmatch(key); len(m) >= 3 { + height, _ := strconv.ParseInt(m[1], 10, 64) + digests = append(digests, digestEntry{Key: key, Height: height, Normalization: m[2]}) } } if !aws.ToBool(resp.IsTruncated) { @@ -122,6 +134,12 @@ func runReportList(ctx context.Context, cmd *cli.Command) error { sort.Slice(pages, func(i, j int) bool { return pages[i].StartHeight < pages[j].StartHeight }) sort.Slice(reports, func(i, j int) bool { return reports[i].Height < reports[j].Height }) + sort.Slice(digests, func(i, j int) bool { + if digests[i].Height != digests[j].Height { + return digests[i].Height < digests[j].Height + } + return digests[i].Normalization < digests[j].Normalization + }) var totalBlocks int64 for _, p := range pages { @@ -131,6 +149,7 @@ func runReportList(ctx context.Context, cmd *cli.Command) error { out := listOutput{ Pages: pages, DivergenceReports: reports, + DigestRecords: digests, TotalBlocks: totalBlocks, } if out.Pages == nil { @@ -139,6 +158,9 @@ func runReportList(ctx context.Context, cmd *cli.Command) error { if out.DivergenceReports == nil { out.DivergenceReports = []divergenceEntry{} } + if out.DigestRecords == nil { + out.DigestRecords = []digestEntry{} + } if cmd.Bool("json") { enc := json.NewEncoder(os.Stdout) @@ -147,10 +169,10 @@ func runReportList(ctx context.Context, cmd *cli.Command) error { } // Human-readable output. - fmt.Fprintf(os.Stderr, "%d comparison page(s), %d divergence report(s), %d blocks covered\n\n", - len(pages), len(reports), totalBlocks) + fmt.Fprintf(os.Stderr, "%d comparison page(s), %d divergence report(s), %d digest record(s), %d blocks covered\n\n", + len(pages), len(reports), len(digests), totalBlocks) - if len(pages) == 0 && len(reports) == 0 { + if len(pages) == 0 && len(reports) == 0 && len(digests) == 0 { fmt.Fprintln(os.Stderr, "no data found") return nil } @@ -163,6 +185,9 @@ func runReportList(ctx context.Context, cmd *cli.Command) error { for _, r := range reports { fmt.Fprintf(w, "divergence\t%d\t-\n", r.Height) } + for _, d := range digests { + fmt.Fprintf(w, "digest (%s)\t%d\t-\n", d.Normalization, d.Height) + } w.Flush() return nil } diff --git a/report_list_test.go b/report_list_test.go index 236ddf7..4f753e9 100644 --- a/report_list_test.go +++ b/report_list_test.go @@ -71,6 +71,63 @@ func TestComparePageRe(t *testing.T) { } } +func TestEndpointDigestRe(t *testing.T) { + tests := []struct { + key string + wantH string + wantNorm string + want bool + }{ + { + key: "shadow-results/endpoint-digest-198032451-semantic.json.gz", + wantH: "198032451", + wantNorm: "semantic", + want: true, + }, + { + key: "prefix/endpoint-digest-1-translator.json.gz", + wantH: "1", + wantNorm: "translator", + want: true, + }, + { + // Divergence report — must NOT match. + key: "shadow-results/divergence-198032451.report.json.gz", + want: false, + }, + { + // Comparison page — must NOT match. + key: "shadow-results/198000000-198000099.compare.ndjson.gz", + want: false, + }, + { + key: "", + want: false, + }, + } + + for _, tt := range tests { + t.Run(tt.key, func(t *testing.T) { + m := endpointDigestRe.FindStringSubmatch(tt.key) + if !tt.want { + if len(m) >= 3 { + t.Fatalf("expected no match for %q, got %v", tt.key, m) + } + return + } + if len(m) < 3 { + t.Fatalf("expected match for %q, got nil", tt.key) + } + if m[1] != tt.wantH { + t.Errorf("height = %q, want %q", m[1], tt.wantH) + } + if m[2] != tt.wantNorm { + t.Errorf("normalization = %q, want %q", m[2], tt.wantNorm) + } + }) + } +} + func TestDivergenceReportRe(t *testing.T) { tests := []struct { key string diff --git a/report_test.go b/report_test.go index e35e638..1fd6acd 100644 --- a/report_test.go +++ b/report_test.go @@ -1,9 +1,57 @@ package main import ( + "encoding/json" + "strings" "testing" + + "github.com/sei-protocol/seictl/sidecar/tasks" ) +func TestRenderDigestMarkdown(t *testing.T) { + // Decode from JSON so the test does not depend on the unexported per-bucket + // value type, exercising the same shape fetchDigestRecord produces. + const recordJSON = `{ + "height": 198740042, + "normalization": "semantic", + "flatkv_digest": "aaaaaaaabbbbbbbbccccccccdddddddd", + "memiavl_digest": "aaaaaaaabbbbbbbbccccccccdddddddd", + "per_bucket": { + "account": {"flatkv": "11111111aaaa", "memiavl": "11111111aaaa", "match": true}, + "code": {"flatkv": "22222222bbbb", "memiavl": "33333333cccc", "match": false}, + "storage": {"flatkv": "44444444dddd", "memiavl": "44444444dddd", "match": true}, + "legacy": {"flatkv": "55555555eeee", "memiavl": "55555555eeee", "match": true} + }, + "match": false, + "axes_proved": ["nonce", "code", "code_hash", "storage", "legacy"], + "generated_at": "2026-06-17T00:00:00Z" + }` + + var record tasks.EndpointDigestRecord + if err := json.Unmarshal([]byte(recordJSON), &record); err != nil { + t.Fatalf("unmarshalling record: %v", err) + } + + out := renderDigestMarkdown(&record) + + wantContains := []string{ + "# EVM Logical Digest — Height 198740042 (semantic)", + "**Generated at:** 2026-06-17T00:00:00Z", + "**Overall match:** ❌", + "## Per-Bucket Digests", + "| account |", + "| code |", + "| storage |", + "| legacy |", + "**Axes proved:** code, code_hash, legacy, nonce, storage", + } + for _, w := range wantContains { + if !strings.Contains(out, w) { + t.Errorf("rendered output missing %q\n--- output ---\n%s", w, out) + } + } +} + func TestResolveS3Ref(t *testing.T) { tests := []struct { name string diff --git a/sidecar/shadow/comparator.go b/sidecar/shadow/comparator.go index d7b3aa2..c259253 100644 --- a/sidecar/shadow/comparator.go +++ b/sidecar/shadow/comparator.go @@ -159,6 +159,45 @@ func (c *Comparator) CompareBlock(ctx context.Context, height int64) (*CompareRe return result, nil } +// ReasonFor isolates the discrepancy type the verdict already determined, +// deepest-layer-first to mirror the DivergenceLayer attribution in CompareBlock: +// a Layer 2 axis divergence is more specific than the Layer 1 kind, which is +// more specific than the Layer 0 app-hash mismatch that triggered the descent. +// +// - layer2-: storage|balance|code|nonce (the diverging StateDivergence kind) +// - layer1-results: transaction-count mismatch (the result sets differ) +// - layer1-receipt: a per-transaction receipt field diverged +// - layer0-apphash: header-level divergence only +// +// An indeterminate (load-bearing check could not run) reports the layer it +// could not evaluate, since that is the actionable signal. Returns "" for a +// clean result; callers only label on a divergence. +func ReasonFor(result *CompareResult) string { + if result.Match { + return "" + } + if l2 := result.Layer2; l2 != nil { + if len(l2.Divergences) > 0 { + return "layer2-" + l2.Divergences[0].Kind + } + if l2.Indeterminate { + return "layer2-indeterminate" + } + } + if l1 := result.Layer1; l1 != nil { + if !l1.TxCountMatch { + return "layer1-results" + } + if len(l1.Divergences) > 0 { + return "layer1-receipt" + } + if l1.Indeterminate { + return "layer1-indeterminate" + } + } + return "layer0-apphash" +} + func (c *Comparator) layer2Enabled() bool { return c.keySource != nil && c.shadowState != nil && c.canonicalState != nil } diff --git a/sidecar/shadow/metrics.go b/sidecar/shadow/metrics.go index d859238..232a809 100644 --- a/sidecar/shadow/metrics.go +++ b/sidecar/shadow/metrics.go @@ -19,13 +19,15 @@ var ( // Divergences counts app-hash divergences detected. Increments at most // once per process lifetime — the comparison loop exits on first divergence. // divergence_layer is "0" for header-hash mismatch, "1" when Layer 1 - // isolated specific tx-receipt mismatches. + // isolated specific tx-receipt mismatches. reason conveys the discrepancy + // type the verdict isolated (deepest-layer-first): layer2-, layer1-, + // or layer0-apphash. See ReasonFor. Divergences = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "seictl_shadow_divergences_total", Help: "App-hash divergences detected by the shadow comparator. Increments once per process lifetime since the loop exits on first divergence.", }, - []string{"chain_id", "pod_name", "divergence_layer"}, + []string{"chain_id", "pod_name", "divergence_layer", "reason"}, ) ) diff --git a/sidecar/shadow/reason_test.go b/sidecar/shadow/reason_test.go new file mode 100644 index 0000000..183f416 --- /dev/null +++ b/sidecar/shadow/reason_test.go @@ -0,0 +1,85 @@ +package shadow + +import "testing" + +func TestReasonFor(t *testing.T) { + tests := []struct { + name string + result CompareResult + want string + }{ + { + name: "clean result has no reason", + result: CompareResult{Match: true}, + want: "", + }, + { + name: "layer0 apphash only", + result: CompareResult{ + Match: false, + Layer0: Layer0Result{AppHashMatch: false, LastResultsHashMatch: true, GasUsedMatch: true}, + }, + want: "layer0-apphash", + }, + { + name: "layer1 receipt field divergence", + result: CompareResult{ + Match: false, + Layer1: &Layer1Result{ + TxCountMatch: true, + Divergences: []TxDivergence{{TxIndex: 0, Fields: []FieldDivergence{{Field: "gasUsed"}}}}, + }, + }, + want: "layer1-receipt", + }, + { + name: "layer1 tx count mismatch is a results divergence", + result: CompareResult{ + Match: false, + Layer1: &Layer1Result{TxCountMatch: false}, + }, + want: "layer1-results", + }, + { + name: "layer1 indeterminate", + result: CompareResult{ + Match: false, + Layer1: &Layer1Result{TxCountMatch: true, Indeterminate: true}, + }, + want: "layer1-indeterminate", + }, + { + name: "layer2 storage divergence wins over layer1", + result: CompareResult{ + Match: false, + Layer1: &Layer1Result{TxCountMatch: false}, + Layer2: &Layer2Result{Divergences: []StateDivergence{{Kind: "storage"}}}, + }, + want: "layer2-storage", + }, + { + name: "layer2 balance divergence", + result: CompareResult{ + Match: false, + Layer2: &Layer2Result{Divergences: []StateDivergence{{Kind: "balance"}}}, + }, + want: "layer2-balance", + }, + { + name: "layer2 indeterminate forces a reason even without a divergence kind", + result: CompareResult{ + Match: false, + Layer2: &Layer2Result{Indeterminate: true}, + }, + want: "layer2-indeterminate", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := ReasonFor(&tt.result); got != tt.want { + t.Errorf("ReasonFor() = %q, want %q", got, tt.want) + } + }) + } +} diff --git a/sidecar/shadow/types.go b/sidecar/shadow/types.go index ef4ad6f..c6af421 100644 --- a/sidecar/shadow/types.go +++ b/sidecar/shadow/types.go @@ -136,7 +136,7 @@ type Layer2Result struct { // StateDivergence records a single logical-state mismatch between the shadow // and canonical chains. Values are hex for legibility in reports. type StateDivergence struct { - Kind string `json:"kind"` // storage | code | nonce + Kind string `json:"kind"` // storage | balance | code | nonce Addr string `json:"addr"` Slot string `json:"slot,omitempty"` // set only for storage Shadow string `json:"shadow"` diff --git a/sidecar/tasks/evm_logical_digest.go b/sidecar/tasks/evm_logical_digest.go index d365745..0fa5a6d 100644 --- a/sidecar/tasks/evm_logical_digest.go +++ b/sidecar/tasks/evm_logical_digest.go @@ -135,6 +135,7 @@ func (d *EvmLogicalDigester) run(ctx context.Context, req EvmLogicalDigestReques } record := buildEndpointDigest(req.Height, norm, flatkv, memiavl) + recordDigestMetrics(record) key := fmt.Sprintf("%sendpoint-digest-%d-%s.json.gz", prefix, req.Height, norm) emit, err := seis3.StreamGzipJSON(ctx, uploader, req.Bucket, key, record) diff --git a/sidecar/tasks/metrics.go b/sidecar/tasks/metrics.go new file mode 100644 index 0000000..d574fb8 --- /dev/null +++ b/sidecar/tasks/metrics.go @@ -0,0 +1,61 @@ +package tasks + +import "github.com/prometheus/client_golang/prometheus" + +var ( + // evmLogicalDigestRuns counts evm-logical-digest comparisons published, one + // per (height, normalization) record. result is "match" or "mismatch". This + // is the liveness signal: rate(...)==0 means the oracle stopped running, which + // divergence counters alone cannot distinguish from "ran and found nothing". + // pod/chain come from Prometheus scrape/target labels, not metric labels. + evmLogicalDigestRuns = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "seictl_evm_logical_digest_runs_total", + Help: "Total evm-logical-digest comparisons published, labelled by normalization and match outcome.", + }, + []string{"normalization", "result"}, + ) + + // evmLogicalDigestMismatches counts diverging buckets in mismatched records. + // reason is the diverging bucket (account|code|storage|legacy) or "final" when + // only the combined digest differs. Incremented once per diverging bucket, so + // a record with two bad buckets adds two — the granular alertability signal. + evmLogicalDigestMismatches = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: "seictl_evm_logical_digest_mismatches_total", + Help: "Total diverging buckets across evm-logical-digest mismatches, labelled by normalization and diverging bucket.", + }, + []string{"normalization", "reason"}, + ) +) + +func init() { + prometheus.MustRegister(evmLogicalDigestRuns) + prometheus.MustRegister(evmLogicalDigestMismatches) +} + +// recordDigestMetrics emits the run outcome and, on mismatch, one increment per +// diverging bucket. When the record mismatches but no per-bucket digest differs +// (only the combined FINAL digest), it attributes the mismatch to "final". +func recordDigestMetrics(record EndpointDigestRecord) { + result := "match" + if !record.Match { + result = "mismatch" + } + evmLogicalDigestRuns.WithLabelValues(record.Normalization, result).Inc() + + if record.Match { + return + } + + var anyBucket bool + for _, name := range []string{"account", "code", "storage", "legacy"} { + if b, ok := record.PerBucket[name]; ok && !b.Match { + evmLogicalDigestMismatches.WithLabelValues(record.Normalization, name).Inc() + anyBucket = true + } + } + if !anyBucket { + evmLogicalDigestMismatches.WithLabelValues(record.Normalization, "final").Inc() + } +} diff --git a/sidecar/tasks/metrics_test.go b/sidecar/tasks/metrics_test.go new file mode 100644 index 0000000..139643c --- /dev/null +++ b/sidecar/tasks/metrics_test.go @@ -0,0 +1,84 @@ +package tasks + +import ( + "testing" + + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestRecordDigestMetrics(t *testing.T) { + matchBucket := func(m bool) bucket { return bucket{Match: m} } + + t.Run("match increments runs only", func(t *testing.T) { + const norm = "match-norm" + runsBefore := testutil.ToFloat64(evmLogicalDigestRuns.WithLabelValues(norm, "match")) + + recordDigestMetrics(EndpointDigestRecord{ + Normalization: norm, + Match: true, + PerBucket: map[string]bucket{ + "account": matchBucket(true), + "code": matchBucket(true), + "storage": matchBucket(true), + "legacy": matchBucket(true), + }, + }) + + if got := testutil.ToFloat64(evmLogicalDigestRuns.WithLabelValues(norm, "match")) - runsBefore; got != 1 { + t.Errorf("runs{result=match} delta = %v, want 1", got) + } + }) + + t.Run("mismatch increments per diverging bucket", func(t *testing.T) { + const norm = "bucket-norm" + runsBefore := testutil.ToFloat64(evmLogicalDigestRuns.WithLabelValues(norm, "mismatch")) + storageBefore := testutil.ToFloat64(evmLogicalDigestMismatches.WithLabelValues(norm, "storage")) + codeBefore := testutil.ToFloat64(evmLogicalDigestMismatches.WithLabelValues(norm, "code")) + finalBefore := testutil.ToFloat64(evmLogicalDigestMismatches.WithLabelValues(norm, "final")) + + recordDigestMetrics(EndpointDigestRecord{ + Normalization: norm, + Match: false, + PerBucket: map[string]bucket{ + "account": matchBucket(true), + "code": matchBucket(false), + "storage": matchBucket(false), + "legacy": matchBucket(true), + }, + }) + + if got := testutil.ToFloat64(evmLogicalDigestRuns.WithLabelValues(norm, "mismatch")) - runsBefore; got != 1 { + t.Errorf("runs{result=mismatch} delta = %v, want 1", got) + } + if got := testutil.ToFloat64(evmLogicalDigestMismatches.WithLabelValues(norm, "storage")) - storageBefore; got != 1 { + t.Errorf("mismatches{reason=storage} delta = %v, want 1", got) + } + if got := testutil.ToFloat64(evmLogicalDigestMismatches.WithLabelValues(norm, "code")) - codeBefore; got != 1 { + t.Errorf("mismatches{reason=code} delta = %v, want 1", got) + } + if got := testutil.ToFloat64(evmLogicalDigestMismatches.WithLabelValues(norm, "final")) - finalBefore; got != 0 { + t.Errorf("mismatches{reason=final} delta = %v, want 0 (per-bucket attributed)", got) + } + }) + + t.Run("final-only mismatch attributes to final", func(t *testing.T) { + const norm = "final-norm" + finalBefore := testutil.ToFloat64(evmLogicalDigestMismatches.WithLabelValues(norm, "final")) + + // Combined digest differs but every per-bucket digest matches. + recordDigestMetrics(EndpointDigestRecord{ + Normalization: norm, + Match: false, + PerBucket: map[string]bucket{ + "account": matchBucket(true), + "code": matchBucket(true), + "storage": matchBucket(true), + "legacy": matchBucket(true), + }, + }) + + if got := testutil.ToFloat64(evmLogicalDigestMismatches.WithLabelValues(norm, "final")) - finalBefore; got != 1 { + t.Errorf("mismatches{reason=final} delta = %v, want 1", got) + } + }) +} diff --git a/sidecar/tasks/result_compare.go b/sidecar/tasks/result_compare.go index 8b5473f..ac82d5a 100644 --- a/sidecar/tasks/result_compare.go +++ b/sidecar/tasks/result_compare.go @@ -176,11 +176,13 @@ func (l *comparisonLoop) handleDivergence(ctx context.Context, result shadow.Com if result.DivergenceLayer != nil { layer = fmt.Sprintf("%d", *result.DivergenceLayer) } - shadow.Divergences.WithLabelValues(l.exporter.chainID, l.exporter.podName, layer).Inc() + reason := shadow.ReasonFor(&result) + shadow.Divergences.WithLabelValues(l.exporter.chainID, l.exporter.podName, layer, reason).Inc() exportLog.Info("app-hash divergence detected", "height", l.height, "divergence-layer", layer, + "reason", reason, "shadow-app-hash", result.Layer0.ShadowAppHash, "canonical-app-hash", result.Layer0.CanonicalAppHash) diff --git a/sidecar/tasks/result_export_test.go b/sidecar/tasks/result_export_test.go index 84b2795..dc0fb23 100644 --- a/sidecar/tasks/result_export_test.go +++ b/sidecar/tasks/result_export_test.go @@ -364,7 +364,7 @@ func TestExportAndCompare_DivergenceDetected(t *testing.T) { const testPodName = "shadow-test-0" e := NewResultExporter(tmpDir, "test-1", testPodName, mockResultUploaderFactory()) - divergenceBefore := testutil.ToFloat64(shadow.Divergences.WithLabelValues("test-1", testPodName, "0")) + divergenceBefore := testutil.ToFloat64(shadow.Divergences.WithLabelValues("test-1", testPodName, "0", "layer0-apphash")) err := e.ExportAndCompare(context.Background(), ResultExportRequest{ Bucket: "test-bucket", @@ -383,8 +383,8 @@ func TestExportAndCompare_DivergenceDetected(t *testing.T) { t.Errorf("LastExportedHeight = %d, want 1 (diverged at first block)", state.LastExportedHeight) } - if got := testutil.ToFloat64(shadow.Divergences.WithLabelValues("test-1", testPodName, "0")); got-divergenceBefore != 1 { - t.Errorf("seictl_shadow_divergences_total{chain_id=test-1,pod_name=%s,divergence_layer=0} delta = %v, want 1", testPodName, got-divergenceBefore) + if got := testutil.ToFloat64(shadow.Divergences.WithLabelValues("test-1", testPodName, "0", "layer0-apphash")); got-divergenceBefore != 1 { + t.Errorf("seictl_shadow_divergences_total{chain_id=test-1,pod_name=%s,divergence_layer=0,reason=layer0-apphash} delta = %v, want 1", testPodName, got-divergenceBefore) } if got := testutil.ToFloat64(shadow.BlocksCompared.WithLabelValues("test-1", testPodName)); got < 1 { t.Errorf("seictl_shadow_blocks_compared_total{chain_id=test-1,pod_name=%s} = %v, want >= 1", testPodName, got) From 55a7f2d4b4f30e12dffdcd73cd9932d22e8f5334 Mon Sep 17 00:00:00 2001 From: bdchatham Date: Sat, 20 Jun 2026 11:47:17 -0700 Subject: [PATCH 2/2] fix(sidecar): emit digest metrics only after a successful S3 publish (Bugbot) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit recordDigestMetrics ran before StreamGzipJSON, so a failed upload still bumped seictl_evm_logical_digest_runs_total — making the oracle-liveness signal look healthy while no artifact landed in S3. Move the emit after the publish succeeds so the counters count records that actually persisted. Co-Authored-By: Claude Opus 4.8 --- sidecar/tasks/evm_logical_digest.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sidecar/tasks/evm_logical_digest.go b/sidecar/tasks/evm_logical_digest.go index 0fa5a6d..57f172d 100644 --- a/sidecar/tasks/evm_logical_digest.go +++ b/sidecar/tasks/evm_logical_digest.go @@ -135,13 +135,16 @@ func (d *EvmLogicalDigester) run(ctx context.Context, req EvmLogicalDigestReques } record := buildEndpointDigest(req.Height, norm, flatkv, memiavl) - recordDigestMetrics(record) key := fmt.Sprintf("%sendpoint-digest-%d-%s.json.gz", prefix, req.Height, norm) emit, err := seis3.StreamGzipJSON(ctx, uploader, req.Bucket, key, record) if err != nil { return seis3.ClassifyS3Error("evm-logical-digest", req.Bucket, key, req.Region, err) } + // After the publish succeeds: the run/mismatch counters count records + // that actually landed in S3, so a failed upload can't leave the + // oracle-liveness signal looking healthy while no artifact was written. + recordDigestMetrics(record) evmDigestLog.Info("published endpoint digest", "height", req.Height,