From 0414834f645c3e73e3ea8fb5c8e4dd9adee15686 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ois=C3=ADn=20Kyne?= Date: Wed, 28 Oct 2026 11:13:16 +0000 Subject: [PATCH 1/2] app/log: redact credentials and API keys in URLs Redact passwords, query values and API-key-like path segments of URLs in all log messages and fields, and the full path of endpoint and URL flags in the startup config log. Keep integer and 0x hex query values, such as slots and roots. Drop the user info from the app_builder_config_url label. --- app/builderconfig.go | 16 +++- app/builderconfig_internal_test.go | 6 ++ app/log/log.go | 14 ++-- app/log/redact.go | 122 +++++++++++++++++++++++++++++ app/log/redact_internal_test.go | 77 ++++++++++++++++++ cmd/cmd.go | 16 ++-- cmd/cmd_internal_test.go | 24 ++++++ 7 files changed, 259 insertions(+), 16 deletions(-) create mode 100644 app/log/redact.go create mode 100644 app/log/redact_internal_test.go diff --git a/app/builderconfig.go b/app/builderconfig.go index 763572272..ecf536c52 100644 --- a/app/builderconfig.go +++ b/app/builderconfig.go @@ -5,6 +5,7 @@ package app import ( "crypto/sha256" "fmt" + "net/url" "slices" "github.com/attestantio/go-eth2-client/spec/gloas" @@ -91,10 +92,23 @@ func initBuilderConfigMetrics(conf Config) { } for _, u := range conf.BuilderURLs { - builderURLGauge.WithLabelValues(u).Set(1) + builderURLGauge.WithLabelValues(withoutUserInfo(u)).Set(1) } builderMinBidGauge.WithLabelValues().Set(float64(conf.BuilderMinBid)) builderBoostFactorGauge.WithLabelValues().Set(float64(conf.BuilderBoostFactor)) builderMaxExecutionPaymentGauge.WithLabelValues().Set(float64(conf.BuilderMaxExecutionPayment)) } + +// withoutUserInfo returns the URL without its user info. Relay URLs carry the relay's public key as the +// username, which only lengthens the label, and private builders may carry a password. +func withoutUserInfo(raw string) string { + u, err := url.Parse(raw) + if err != nil || u.User == nil { + return raw + } + + u.User = nil + + return u.String() +} diff --git a/app/builderconfig_internal_test.go b/app/builderconfig_internal_test.go index 01d90f2df..414204576 100644 --- a/app/builderconfig_internal_test.go +++ b/app/builderconfig_internal_test.go @@ -85,3 +85,9 @@ func TestBuilderHosts(t *testing.T) { require.NoError(t, err) require.Empty(t, hosts) } + +func TestWithoutUserInfo(t *testing.T) { + require.Equal(t, "https://relay.example.com", withoutUserInfo("https://0xabc@relay.example.com")) + require.Equal(t, "https://builder.example.com/path", withoutUserInfo("https://user:secret@builder.example.com/path")) + require.Equal(t, "http://mev-boost:18550", withoutUserInfo("http://mev-boost:18550")) +} diff --git a/app/log/log.go b/app/log/log.go index e4166fd5e..b1861d9cc 100644 --- a/app/log/log.go +++ b/app/log/log.go @@ -88,7 +88,7 @@ func Debug(ctx context.Context, msg string, fields ...z.Field) { } trace.SpanFromContext(ctx).AddEvent("log.Debug: "+msg, toAttributes(zfl)) - getLogger(ctx).Debug(msg, zfl...) + getLogger(ctx).Debug(redactURLs(msg), zfl...) } // Info logs the message and fields (incl fields in the context) at Info level. @@ -100,7 +100,7 @@ func Info(ctx context.Context, msg string, fields ...z.Field) { } trace.SpanFromContext(ctx).AddEvent("log.Info: "+msg, toAttributes(zfl)) - getLogger(ctx).Info(msg, zfl...) + getLogger(ctx).Info(redactURLs(msg), zfl...) } // Warn wraps err with msg and fields and logs it (incl fields in the context) at Warn level. @@ -116,7 +116,7 @@ func Warn(ctx context.Context, msg string, err error, fields ...z.Field) { } trace.SpanFromContext(ctx).AddEvent("log.Warn: "+msg, toAttributes(zfl)) - getLogger(ctx).Warn(msg, zfl...) + getLogger(ctx).Warn(redactURLs(msg), zfl...) return } @@ -129,7 +129,7 @@ func Warn(ctx context.Context, msg string, err error, fields ...z.Field) { } trace.SpanFromContext(ctx).RecordError(err, trace.WithStackTrace(true), toAttributes(zfl)) - getLogger(ctx).Warn(err.Error(), zfl...) + getLogger(ctx).Warn(redactURLs(err.Error()), zfl...) } // Error wraps err with msg and fields and logs it (incl fields in the context) at Error level. @@ -145,7 +145,7 @@ func Error(ctx context.Context, msg string, err error, fields ...z.Field) { } trace.SpanFromContext(ctx).AddEvent("log.Error: "+msg, toAttributes(zfl)) - getLogger(ctx).Error(msg, zfl...) + getLogger(ctx).Error(redactURLs(msg), zfl...) return } @@ -158,7 +158,7 @@ func Error(ctx context.Context, msg string, err error, fields ...z.Field) { } trace.SpanFromContext(ctx).RecordError(err, trace.WithStackTrace(true), toAttributes(zfl)) - getLogger(ctx).Error(err.Error(), zfl...) + getLogger(ctx).Error(redactURLs(err.Error()), zfl...) } // unwrapDedup returns true and the wrapped zap fields from the slice and from the context. Duplicate fields are dropped. @@ -181,7 +181,7 @@ func unwrapDedup(ctx context.Context, fields ...z.Field) ([]zap.Field, bool) { } dups[f.Key] = true - resp = append(resp, f) + resp = append(resp, redactField(f)) } for _, field := range fields { diff --git a/app/log/redact.go b/app/log/redact.go new file mode 100644 index 000000000..e92f6893f --- /dev/null +++ b/app/log/redact.go @@ -0,0 +1,122 @@ +// Copyright © 2022-2026 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +package log + +import ( + "net/url" + "regexp" + "strings" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +const redacted = "xxxxx" + +var ( + // urlPattern matches URLs embedded in free text, stopping at whitespace, quotes, brackets and commas. + urlPattern = regexp.MustCompile(`[a-zA-Z][a-zA-Z0-9+.-]*://[^\s"'<>\[\](),]+`) + // apiKeySegment matches path segments that look like API keys, e.g. "/v2/" of hosted node providers: + // at least 20 characters, mixing letters and digits. Beacon API names like "prepare_beacon_proposer" have + // no digits, and hex values with a 0x prefix (roots, pubkeys, lock hashes) are skipped by the caller. + apiKeySegment = regexp.MustCompile(`^[A-Za-z0-9_-]{20,}$`) + digit = regexp.MustCompile(`[0-9]`) + letter = regexp.MustCompile(`[A-Za-z]`) + // publicQueryValue matches query values kept for debugging: integers like slots and indices, and 0x hex values. + publicQueryValue = regexp.MustCompile(`^([0-9]{1,20}|0x[0-9a-fA-F]*)$`) +) + +// RedactURL returns raw with the URL's password and query values replaced with "xxxxx". +// Query values that are integers or 0x hex values, like slots and roots, are kept. +// If maskPath is true, a non-empty path is replaced too, since hosted beacon and execution +// providers put API keys in it. Otherwise only path segments that look like API keys are. +// Values that aren't absolute URLs are returned unchanged. +func RedactURL(raw string, maskPath bool) string { + u, err := url.Parse(raw) + if err != nil || u.Scheme == "" || u.Host == "" { + return raw + } + + changed := false + + if _, ok := u.User.Password(); ok { + u.User = url.UserPassword(u.User.Username(), redacted) + changed = true + } + + if u.RawQuery != "" { + q := u.Query() + for k, vals := range q { + for i, v := range vals { + if !publicQueryValue.MatchString(v) { + vals[i] = redacted + changed = true + } + } + + q[k] = vals + } + + if changed { + u.RawQuery = q.Encode() + } + } + + if maskPath && strings.Trim(u.Path, "/") != "" { + u.Path, u.RawPath = "/"+redacted, "" + changed = true + } else if !maskPath { + segments := strings.Split(u.Path, "/") + for i, s := range segments { + if looksLikeAPIKey(s) { + segments[i] = redacted + changed = true + } + } + + u.Path, u.RawPath = strings.Join(segments, "/"), "" + } + + if !changed { + return raw + } + + return u.String() +} + +func looksLikeAPIKey(segment string) bool { + return !strings.HasPrefix(segment, "0x") && apiKeySegment.MatchString(segment) && + digit.MatchString(segment) && letter.MatchString(segment) +} + +// redactURLs redacts the credentials, query values and API-key-like path segments of any URLs in s. +// Beacon node, execution, builder and other URLs may contain credentials, and logs may be shared. +func redactURLs(s string) string { + if !strings.Contains(s, "://") { + return s + } + + return urlPattern.ReplaceAllStringFunc(s, func(m string) string { + return RedactURL(m, false) + }) +} + +// redactField returns the field with any URLs in its string or error value redacted. +func redactField(f zap.Field) zap.Field { + switch f.Type { + case zapcore.StringType: + f.String = redactURLs(f.String) + case zapcore.ErrorType: + err, ok := f.Interface.(error) + if !ok { + return f + } + + if msg := redactURLs(err.Error()); msg != err.Error() { + return zap.String(f.Key, msg) + } + default: + } + + return f +} diff --git a/app/log/redact_internal_test.go b/app/log/redact_internal_test.go new file mode 100644 index 000000000..638adce07 --- /dev/null +++ b/app/log/redact_internal_test.go @@ -0,0 +1,77 @@ +// Copyright © 2022-2026 Obol Labs Inc. Licensed under the terms of a Business Source License 1.1 + +package log + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" + + "github.com/obolnetwork/charon/app/errors" + "github.com/obolnetwork/charon/app/z" +) + +func TestRedactURLs(t *testing.T) { + tests := []struct { + in string + want string + }{ + {"https://user:secret@bn.example.com:5052", "https://user:xxxxx@bn.example.com:5052"}, + {"http://:secret@lighthouse:5052/eth/v1?x=1", "http://:xxxxx@lighthouse:5052/eth/v1?x=1"}, + {"[http://a:b@one,https://c:d@two]", "[http://a:xxxxx@one,https://c:xxxxx@two]"}, + {`Get "https://user:secret@bn:5052/eth/v1/node/version": EOF`, `Get "https://user:xxxxx@bn:5052/eth/v1/node/version": EOF`}, + {"https://eth-mainnet.g.alchemy.com/v2/abcDEF123ghiJKL456mnoPQR789", "https://eth-mainnet.g.alchemy.com/v2/xxxxx"}, + {"https://bn.example.com/eth/v1/beacon?apikey=secret&x=1", "https://bn.example.com/eth/v1/beacon?apikey=xxxxx&x=1"}, + {"http://bn:5052/eth/v1/validator/attestation_data?slot=12345678&committee_index=3", "http://bn:5052/eth/v1/validator/attestation_data?slot=12345678&committee_index=3"}, + {"http://bn:5052/eth/v3/validator/blocks/123?randao_reveal=0xab12&graffiti=hi", "http://bn:5052/eth/v3/validator/blocks/123?graffiti=xxxxx&randao_reveal=0xab12"}, + {"http://bn:5052/eth/v1/validator/prepare_beacon_proposer", "http://bn:5052/eth/v1/validator/prepare_beacon_proposer"}, + {"https://api.obol.tech/v1/lock/0x0f481bbd06a596cb3ba569b9de0cbfcf822b209c2d6877c98173df986dd3c0ec", "https://api.obol.tech/v1/lock/0x0f481bbd06a596cb3ba569b9de0cbfcf822b209c2d6877c98173df986dd3c0ec"}, + {"https://user@bn.example.com", "https://user@bn.example.com"}, + {"http://lighthouse:5052", "http://lighthouse:5052"}, + {"user:secret@host", "user:secret@host"}, + {"operator@example.com", "operator@example.com"}, + } + for _, test := range tests { + t.Run(test.in, func(t *testing.T) { + require.Equal(t, test.want, redactURLs(test.in)) + }) + } +} + +func TestRedactURLMaskPath(t *testing.T) { + tests := []struct { + in string + want string + }{ + {"https://mainnet.infura.io/v3/0123456789abcdef", "https://mainnet.infura.io/xxxxx"}, + {"https://user:secret@bn.example.com/key?token=t", "https://user:xxxxx@bn.example.com/xxxxx?token=xxxxx"}, + {"http://lighthouse:5052", "http://lighthouse:5052"}, + {"http://lighthouse:5052/", "http://lighthouse:5052/"}, + {"lighthouse:5052", "lighthouse:5052"}, + {"127.0.0.1:3600", "127.0.0.1:3600"}, + } + for _, test := range tests { + t.Run(test.in, func(t *testing.T) { + require.Equal(t, test.want, RedactURL(test.in, true)) + }) + } +} + +func TestLogRedactsURLPasswords(t *testing.T) { + var buf zaptest.Buffer + + ctx := WithLogger(context.Background(), NewConsoleForT(t, &buf)) + ctx = WithCtx(ctx, z.Str("ctx_url", "https://ctx:secret@bn")) + url := "https://user:secret@bn.example.com" + + Debug(ctx, "debug "+url, z.Str("url", url)) + Info(ctx, "info", z.Any("urls", []string{url})) + Warn(ctx, "warn", errors.New("dial "+url, z.Str("addr", url))) + Error(ctx, "error", nil, z.Err(errors.New("dial "+url))) + + require.NotContains(t, buf.String(), "secret") + require.Contains(t, buf.String(), "https://user:xxxxx@bn.example.com") + require.Contains(t, buf.String(), "https://ctx:xxxxx@bn") +} diff --git a/cmd/cmd.go b/cmd/cmd.go index 17eb2cfbb..6c41cc065 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -6,7 +6,6 @@ package cmd import ( "context" "fmt" - "net/url" "strings" libp2plog "github.com/ipfs/go-log/v2" @@ -236,7 +235,9 @@ func flagsToLogFields(flags *pflag.FlagSet) []z.Field { // redact returns a redacted version of the given flag value. It supports: // - Full redaction of auth token flags (e.g., keymanager-auth-token) // - Value redaction of header flags (e.g., beacon-node-headers, otlp-headers) keeping keys visible -// - Password redaction in URLs for address flags (e.g., loki-addresses) +// - Password and query value redaction in URLs for address flags (e.g., loki-addresses) +// - Password, query value and path redaction in URLs for endpoint and URL flags (e.g., beacon-node-endpoints), +// since hosted beacon and execution providers put API keys in the path func redact(flag, val string) string { if strings.Contains(flag, "auth-token") { return "xxxxx" @@ -246,16 +247,15 @@ func redact(flag, val string) string { return redactHeaderValue(val) } - if !strings.Contains(flag, "address") { - return val + if strings.Contains(flag, "endpoint") || strings.Contains(flag, "url") { + return log.RedactURL(val, true) } - u, err := url.ParseRequestURI(val) - if err != nil { - return val + if strings.Contains(flag, "address") { + return log.RedactURL(val, false) } - return u.Redacted() + return val } // redactHeaderValue redacts the value portion of a "key=value" header string, diff --git a/cmd/cmd_internal_test.go b/cmd/cmd_internal_test.go index 551825eb0..4bac4ed24 100644 --- a/cmd/cmd_internal_test.go +++ b/cmd/cmd_internal_test.go @@ -267,6 +267,30 @@ func TestRedact(t *testing.T) { value: "https://user:password@example.com/foo/bar", expected: "https://user:xxxxx@example.com/foo/bar", }, + { + name: "redact API keys in endpoint paths", + flag: "beacon-node-endpoints", + value: "https://eth-mainnet.example.com/v2/secret-key", + expected: "https://eth-mainnet.example.com/xxxxx", + }, + { + name: "redact query values in execution endpoints", + flag: "execution-client-rpc-endpoint", + value: "https://el.example.com?apikey=secret", + expected: "https://el.example.com?apikey=xxxxx", + }, + { + name: "keep local endpoints", + flag: "beacon-node-endpoints", + value: "http://lighthouse:5052", + expected: "http://lighthouse:5052", + }, + { + name: "redact builder URL passwords", + flag: "builder-urls", + value: "https://0xabc:secret@relay.example.com", + expected: "https://0xabc:xxxxx@relay.example.com", + }, { name: "no redact", flag: "definition-file", From 3eea43d933c04725bc9c756e7e4eff0b07d952dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ois=C3=ADn=20Kyne?= Date: Wed, 28 Oct 2026 11:38:04 +0000 Subject: [PATCH 2/2] *: redact URLs where they are logged instead of in the logger Checking every field of every log line costs about 1.8us per line, mostly from formatting errors. Remove that from the logger and redact only where raw endpoint URLs reach logs or metrics: the SSE listener's logs and metric labels, the fetcher and scheduler bn_addr fields, eth2wrap's url.Error decomposition, eth1wrap errors and the exit commands. go-eth2-client already masks its own Address(). --- app/eth1wrap/runner.go | 6 +-- app/eth2wrap/eth2wrap.go | 2 +- app/log/log.go | 14 +++---- app/log/redact.go | 66 ++++++++++++++------------------- app/log/redact_internal_test.go | 34 +++++++---------- app/sse/client.go | 8 ++-- app/sse/listener.go | 61 ++++++++++++++++++------------ cmd/exit_broadcast.go | 2 +- cmd/exit_list.go | 2 +- cmd/exit_sign.go | 2 +- core/fetcher/fetcher.go | 10 ++--- core/scheduler/scheduler.go | 6 +-- 12 files changed, 105 insertions(+), 108 deletions(-) diff --git a/app/eth1wrap/runner.go b/app/eth1wrap/runner.go index 0704c365a..e37c1de97 100644 --- a/app/eth1wrap/runner.go +++ b/app/eth1wrap/runner.go @@ -83,7 +83,7 @@ func NewDefaultEthClientRunner(addr string) EthClientRunner { func(ctx context.Context, url string) (EthClient, error) { cl, err := ethclient.DialContext(ctx, url) if err != nil { - return nil, errors.Wrap(err, "connect to eth1 client") + return nil, errors.Wrap(log.RedactURLError(err), "connect to eth1 client") } return cl, nil @@ -168,7 +168,7 @@ func (cl *client) VerifySmartContractBasedSignature(contractAddress string, hash result, err := erc1271.IsValidSignature(nil, hash, sig) if err != nil { cl.maybeReconnect() - return false, err + return false, log.RedactURLError(err) } return result == erc1271MagicValue, nil @@ -202,7 +202,7 @@ func (cl *client) ClientVersion(ctx context.Context) (string, error) { var ver string if err := eth1client.Client().CallContext(ctx, &ver, "web3_clientVersion"); err != nil { cl.maybeReconnect() - return "", errors.Wrap(err, "get execution layer client version") + return "", errors.Wrap(log.RedactURLError(err), "get execution layer client version") } return ver, nil diff --git a/app/eth2wrap/eth2wrap.go b/app/eth2wrap/eth2wrap.go index b04271221..d0fcad1ab 100644 --- a/app/eth2wrap/eth2wrap.go +++ b/app/eth2wrap/eth2wrap.go @@ -361,7 +361,7 @@ func wrapError(ctx context.Context, err error, label string, fields ...z.Field) } err = errors.Wrap(uerr.Err, msg, - z.Str("url", uerr.URL), + z.Str("url", log.RedactURL(uerr.URL, false)), z.Str("method", uerr.Op), ) } diff --git a/app/log/log.go b/app/log/log.go index b1861d9cc..e4166fd5e 100644 --- a/app/log/log.go +++ b/app/log/log.go @@ -88,7 +88,7 @@ func Debug(ctx context.Context, msg string, fields ...z.Field) { } trace.SpanFromContext(ctx).AddEvent("log.Debug: "+msg, toAttributes(zfl)) - getLogger(ctx).Debug(redactURLs(msg), zfl...) + getLogger(ctx).Debug(msg, zfl...) } // Info logs the message and fields (incl fields in the context) at Info level. @@ -100,7 +100,7 @@ func Info(ctx context.Context, msg string, fields ...z.Field) { } trace.SpanFromContext(ctx).AddEvent("log.Info: "+msg, toAttributes(zfl)) - getLogger(ctx).Info(redactURLs(msg), zfl...) + getLogger(ctx).Info(msg, zfl...) } // Warn wraps err with msg and fields and logs it (incl fields in the context) at Warn level. @@ -116,7 +116,7 @@ func Warn(ctx context.Context, msg string, err error, fields ...z.Field) { } trace.SpanFromContext(ctx).AddEvent("log.Warn: "+msg, toAttributes(zfl)) - getLogger(ctx).Warn(redactURLs(msg), zfl...) + getLogger(ctx).Warn(msg, zfl...) return } @@ -129,7 +129,7 @@ func Warn(ctx context.Context, msg string, err error, fields ...z.Field) { } trace.SpanFromContext(ctx).RecordError(err, trace.WithStackTrace(true), toAttributes(zfl)) - getLogger(ctx).Warn(redactURLs(err.Error()), zfl...) + getLogger(ctx).Warn(err.Error(), zfl...) } // Error wraps err with msg and fields and logs it (incl fields in the context) at Error level. @@ -145,7 +145,7 @@ func Error(ctx context.Context, msg string, err error, fields ...z.Field) { } trace.SpanFromContext(ctx).AddEvent("log.Error: "+msg, toAttributes(zfl)) - getLogger(ctx).Error(redactURLs(msg), zfl...) + getLogger(ctx).Error(msg, zfl...) return } @@ -158,7 +158,7 @@ func Error(ctx context.Context, msg string, err error, fields ...z.Field) { } trace.SpanFromContext(ctx).RecordError(err, trace.WithStackTrace(true), toAttributes(zfl)) - getLogger(ctx).Error(redactURLs(err.Error()), zfl...) + getLogger(ctx).Error(err.Error(), zfl...) } // unwrapDedup returns true and the wrapped zap fields from the slice and from the context. Duplicate fields are dropped. @@ -181,7 +181,7 @@ func unwrapDedup(ctx context.Context, fields ...z.Field) ([]zap.Field, bool) { } dups[f.Key] = true - resp = append(resp, redactField(f)) + resp = append(resp, f) } for _, field := range fields { diff --git a/app/log/redact.go b/app/log/redact.go index e92f6893f..12a5beabf 100644 --- a/app/log/redact.go +++ b/app/log/redact.go @@ -7,15 +7,12 @@ import ( "regexp" "strings" - "go.uber.org/zap" - "go.uber.org/zap/zapcore" + "github.com/obolnetwork/charon/app/errors" ) const redacted = "xxxxx" var ( - // urlPattern matches URLs embedded in free text, stopping at whitespace, quotes, brackets and commas. - urlPattern = regexp.MustCompile(`[a-zA-Z][a-zA-Z0-9+.-]*://[^\s"'<>\[\](),]+`) // apiKeySegment matches path segments that look like API keys, e.g. "/v2/" of hosted node providers: // at least 20 characters, mixing letters and digits. Beacon API names like "prepare_beacon_proposer" have // no digits, and hex values with a 0x prefix (roots, pubkeys, lock hashes) are skipped by the caller. @@ -31,6 +28,9 @@ var ( // If maskPath is true, a non-empty path is replaced too, since hosted beacon and execution // providers put API keys in it. Otherwise only path segments that look like API keys are. // Values that aren't absolute URLs are returned unchanged. +// +// Beacon node, execution and other endpoint URLs may contain credentials, so redact them +// wherever they are logged or used as metric labels. func RedactURL(raw string, maskPath bool) string { u, err := url.Parse(raw) if err != nil || u.Scheme == "" || u.Host == "" { @@ -46,19 +46,20 @@ func RedactURL(raw string, maskPath bool) string { if u.RawQuery != "" { q := u.Query() - for k, vals := range q { + queryChanged := false + + for _, vals := range q { for i, v := range vals { if !publicQueryValue.MatchString(v) { vals[i] = redacted - changed = true + queryChanged = true } } - - q[k] = vals } - if changed { + if queryChanged { u.RawQuery = q.Encode() + changed = true } } @@ -84,39 +85,28 @@ func RedactURL(raw string, maskPath bool) string { return u.String() } -func looksLikeAPIKey(segment string) bool { - return !strings.HasPrefix(segment, "0x") && apiKeySegment.MatchString(segment) && - digit.MatchString(segment) && letter.MatchString(segment) -} - -// redactURLs redacts the credentials, query values and API-key-like path segments of any URLs in s. -// Beacon node, execution, builder and other URLs may contain credentials, and logs may be shared. -func redactURLs(s string) string { - if !strings.Contains(s, "://") { - return s +// RedactURLs returns the URLs redacted with RedactURL. +func RedactURLs(raws []string, maskPath bool) []string { + resp := make([]string, 0, len(raws)) + for _, raw := range raws { + resp = append(resp, RedactURL(raw, maskPath)) } - return urlPattern.ReplaceAllStringFunc(s, func(m string) string { - return RedactURL(m, false) - }) + return resp } -// redactField returns the field with any URLs in its string or error value redacted. -func redactField(f zap.Field) zap.Field { - switch f.Type { - case zapcore.StringType: - f.String = redactURLs(f.String) - case zapcore.ErrorType: - err, ok := f.Interface.(error) - if !ok { - return f - } - - if msg := redactURLs(err.Error()); msg != err.Error() { - return zap.String(f.Key, msg) - } - default: +// RedactURLError redacts the URL of the first *url.Error in err's chain in place, keeping its path +// for debugging, and returns err. Go's HTTP client strips passwords from it, but not API keys +// in the path or query. Call it before wrapping err, since wrapping captures the message. +func RedactURLError(err error) error { + if uerr := new(url.Error); errors.As(err, &uerr) { + uerr.URL = RedactURL(uerr.URL, false) } - return f + return err +} + +func looksLikeAPIKey(segment string) bool { + return !strings.HasPrefix(segment, "0x") && apiKeySegment.MatchString(segment) && + digit.MatchString(segment) && letter.MatchString(segment) } diff --git a/app/log/redact_internal_test.go b/app/log/redact_internal_test.go index 638adce07..26ba58473 100644 --- a/app/log/redact_internal_test.go +++ b/app/log/redact_internal_test.go @@ -3,25 +3,21 @@ package log import ( - "context" + "net/url" "testing" "github.com/stretchr/testify/require" - "go.uber.org/zap/zaptest" "github.com/obolnetwork/charon/app/errors" - "github.com/obolnetwork/charon/app/z" ) -func TestRedactURLs(t *testing.T) { +func TestRedactURL(t *testing.T) { tests := []struct { in string want string }{ {"https://user:secret@bn.example.com:5052", "https://user:xxxxx@bn.example.com:5052"}, {"http://:secret@lighthouse:5052/eth/v1?x=1", "http://:xxxxx@lighthouse:5052/eth/v1?x=1"}, - {"[http://a:b@one,https://c:d@two]", "[http://a:xxxxx@one,https://c:xxxxx@two]"}, - {`Get "https://user:secret@bn:5052/eth/v1/node/version": EOF`, `Get "https://user:xxxxx@bn:5052/eth/v1/node/version": EOF`}, {"https://eth-mainnet.g.alchemy.com/v2/abcDEF123ghiJKL456mnoPQR789", "https://eth-mainnet.g.alchemy.com/v2/xxxxx"}, {"https://bn.example.com/eth/v1/beacon?apikey=secret&x=1", "https://bn.example.com/eth/v1/beacon?apikey=xxxxx&x=1"}, {"http://bn:5052/eth/v1/validator/attestation_data?slot=12345678&committee_index=3", "http://bn:5052/eth/v1/validator/attestation_data?slot=12345678&committee_index=3"}, @@ -35,7 +31,7 @@ func TestRedactURLs(t *testing.T) { } for _, test := range tests { t.Run(test.in, func(t *testing.T) { - require.Equal(t, test.want, redactURLs(test.in)) + require.Equal(t, test.want, RedactURL(test.in, false)) }) } } @@ -57,21 +53,17 @@ func TestRedactURLMaskPath(t *testing.T) { require.Equal(t, test.want, RedactURL(test.in, true)) }) } -} - -func TestLogRedactsURLPasswords(t *testing.T) { - var buf zaptest.Buffer - ctx := WithLogger(context.Background(), NewConsoleForT(t, &buf)) - ctx = WithCtx(ctx, z.Str("ctx_url", "https://ctx:secret@bn")) - url := "https://user:secret@bn.example.com" + require.Equal(t, []string{"https://a.example.com/xxxxx", "http://lighthouse:5052"}, + RedactURLs([]string{"https://a.example.com/v2/key", "http://lighthouse:5052"}, true)) +} - Debug(ctx, "debug "+url, z.Str("url", url)) - Info(ctx, "info", z.Any("urls", []string{url})) - Warn(ctx, "warn", errors.New("dial "+url, z.Str("addr", url))) - Error(ctx, "error", nil, z.Err(errors.New("dial "+url))) +func TestRedactURLError(t *testing.T) { + uerr := &url.Error{Op: "Post", URL: "https://el.example.com/v2/abcDEF123ghiJKL456mnoPQR789", Err: errors.New("timeout")} + err := errors.Wrap(RedactURLError(uerr), "call") - require.NotContains(t, buf.String(), "secret") - require.Contains(t, buf.String(), "https://user:xxxxx@bn.example.com") - require.Contains(t, buf.String(), "https://ctx:xxxxx@bn") + require.NotContains(t, err.Error(), "abcDEF123") + require.Contains(t, err.Error(), "https://el.example.com/v2/xxxxx") + require.ErrorIs(t, err, uerr) + require.NoError(t, RedactURLError(nil)) } diff --git a/app/sse/client.go b/app/sse/client.go index 3241028e1..c0a797ceb 100644 --- a/app/sse/client.go +++ b/app/sse/client.go @@ -51,7 +51,7 @@ func newClient(addr string, header http.Header) (*client, error) { u, err := url.Parse(prefixedAddr) if err != nil { - return nil, errors.Wrap(err, "parse beacon node address", z.Str("addr", addr)) + return nil, errors.Wrap(err, "parse beacon node address", z.Str("addr", log.RedactURL(addr, true))) } u.Path = "/eth/v1/events" @@ -79,7 +79,7 @@ func newClientForT(addr, path string) (*client, error) { u, err := url.Parse(prefixedAddr) if err != nil { - return nil, errors.Wrap(err, "parse beacon node address", z.Str("addr", addr)) + return nil, errors.Wrap(err, "parse beacon node address", z.Str("addr", log.RedactURL(addr, true))) } u.Path = path @@ -115,7 +115,7 @@ func (c *client) start(ctx context.Context, eventFn EventHandler) error { default: // If error is not stream-related error, do not attempt retries and return the error. if !errors.Is(err, errStreamConn) { - return errors.Wrap(err, "handle SSE payload", z.Str("url", c.sseURL.String())) + return errors.Wrap(err, "handle SSE payload", z.Str("url", log.RedactURL(c.sseURL.String(), false))) } if !backoffSet { @@ -135,7 +135,7 @@ func (c *client) start(ctx context.Context, eventFn EventHandler) error { } func (c *client) connect(ctx context.Context, eventFn EventHandler) error { - log.Debug(ctx, "Connecting to SSE stream", z.Str("url", c.sseURL.String())) + log.Debug(ctx, "Connecting to SSE stream", z.Str("url", log.RedactURL(c.sseURL.String(), false))) req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.sseURL.String(), nil) if err != nil { diff --git a/app/sse/listener.go b/app/sse/listener.go index fd7f5f5f2..00d976c26 100644 --- a/app/sse/listener.go +++ b/app/sse/listener.go @@ -50,6 +50,7 @@ type listener struct { blockGossipTimes map[uint64]map[string]time.Time // immutable fields + labels map[string]string // Redacted beacon node addresses, used in logs and metric labels. genesisTime time.Time slotDuration time.Duration slotsPerEpoch uint64 @@ -80,6 +81,7 @@ func StartListener(ctx context.Context, eth2Cl eth2wrap.Client, forkSchedule fun chainReorgSubs: make([]ChainReorgEventHandlerFunc, 0), headSubs: make([]HeadEventHandlerFunc, 0), blockGossipTimes: make(map[uint64]map[string]time.Time), + labels: make(map[string]string), genesisTime: genesisTime, slotDuration: slotDuration, slotsPerEpoch: slotsPerEpoch, @@ -96,15 +98,19 @@ func StartListener(ctx context.Context, eth2Cl eth2wrap.Client, forkSchedule fun httpHeader.Add(k, v) } + for _, addr := range addresses { + l.labels[addr] = log.RedactURL(addr, true) + } + // Open connections for each beacon node. for _, addr := range addresses { go func(addr string) { client, err := newClient(addr, httpHeader) if err != nil { - log.Warn(ctx, "Failed to create SSE client", err, z.Str("addr", addr)) + log.Warn(ctx, "Failed to create SSE client", err, z.Str("addr", l.label(addr))) } else { if err := client.start(ctx, l.eventHandler); err != nil { - log.Warn(ctx, "Failed to start SSE client", err, z.Str("addr", addr)) + log.Warn(ctx, "Failed to start SSE client", err, z.Str("addr", l.label(addr))) } } }(addr) @@ -134,6 +140,15 @@ func (p *listener) SetDutiesDependentRootFunc(fn DutiesDependentRootFunc) { p.dutiesDependentRootFunc = fn } +// label returns the beacon node address without credentials, for logs and metric labels. +func (p *listener) label(addr string) string { + if label, ok := p.labels[addr]; ok { + return label + } + + return log.RedactURL(addr, true) +} + func (p *listener) eventHandler(ctx context.Context, event *event, addr string) error { switch event.Event { case sseHeadEvent: @@ -154,16 +169,16 @@ func (p *listener) handleHeadEvent(ctx context.Context, event *event, addr strin err := json.Unmarshal(event.Data, &head) if err != nil { - return errors.Wrap(err, "unmarshal SSE head event", z.Str("addr", addr)) + return errors.Wrap(err, "unmarshal SSE head event", z.Str("addr", p.label(addr))) } slot, err := strconv.ParseUint(head.Slot, 10, 64) if err != nil { - return errors.Wrap(err, "parse slot to uint64", z.Str("addr", addr)) + return errors.Wrap(err, "parse slot to uint64", z.Str("addr", p.label(addr))) } if slot > math.MaxInt64 { - return errors.New("slot value exceeds int64 range", z.Str("addr", addr), z.U64("slot", slot)) + return errors.New("slot value exceeds int64 range", z.Str("addr", p.label(addr)), z.U64("slot", slot)) } delay, ok := p.computeDelay(slot, event.Timestamp, func(delay time.Duration) bool { @@ -174,10 +189,10 @@ func (p *listener) handleHeadEvent(ctx context.Context, event *event, addr strin if !ok { log.Debug(ctx, "Beacon node received head event too late", z.U64("slot", slot), z.Str("delay", delay.String())) } else { - sseHeadDelayHistogram.WithLabelValues(addr).Observe(delay.Seconds()) + sseHeadDelayHistogram.WithLabelValues(p.label(addr)).Observe(delay.Seconds()) } - sseHeadSlotGauge.WithLabelValues(addr).Set(float64(slot)) + sseHeadSlotGauge.WithLabelValues(p.label(addr)).Set(float64(slot)) // Calculate and record block processing time p.recordBlockProcessingTime(slot, addr, event.Timestamp) @@ -191,7 +206,7 @@ func (p *listener) handleHeadEvent(ctx context.Context, event *event, addr strin blockRoot, err := parseRoot(head.Block) if err != nil { - return errors.Wrap(err, "parse head block root", z.Str("addr", addr)) + return errors.Wrap(err, "parse head block root", z.Str("addr", p.label(addr))) } // Invalidate stale duties before head subscribers start an early fetch from them. @@ -221,12 +236,12 @@ func (p *listener) checkDutiesDependentRoots(ctx context.Context, slot uint64, h previous, err := parseRoot(head.PreviousDutyDependentRoot) if err != nil { - return errors.Wrap(err, "parse head previous duty dependent root", z.Str("addr", addr)) + return errors.Wrap(err, "parse head previous duty dependent root", z.Str("addr", p.label(addr))) } current, err := parseRoot(head.CurrentDutyDependentRoot) if err != nil { - return errors.Wrap(err, "parse head current duty dependent root", z.Str("addr", addr)) + return errors.Wrap(err, "parse head current duty dependent root", z.Str("addr", p.label(addr))) } epoch := eth2p0.Epoch(slot / p.slotsPerEpoch) @@ -252,8 +267,8 @@ func (p *listener) checkDutiesDependentRoots(ctx context.Context, slot uint64, h z.Str("cached_dependent_root", cachedRoot.String()), z.Str("head_dependent_root", check.headRoot.String()), z.U64("head_slot", slot), - z.Str("addr", addr)) - sseDependentRootMismatchCounter.WithLabelValues(addr).Inc() + z.Str("addr", p.label(addr))) + sseDependentRootMismatchCounter.WithLabelValues(p.label(addr)).Inc() // Not deduplicated by epoch like chain_reorg: the refetch may be answered by the lagging node again, // in which case the next head event must trigger again. @@ -268,17 +283,17 @@ func (p *listener) handleChainReorgEvent(ctx context.Context, event *event, addr err := json.Unmarshal(event.Data, &chainReorg) if err != nil { - return errors.Wrap(err, "unmarshal SSE chain_reorg event", z.Str("addr", addr)) + return errors.Wrap(err, "unmarshal SSE chain_reorg event", z.Str("addr", p.label(addr))) } slot, err := strconv.ParseUint(chainReorg.Slot, 10, 64) if err != nil { - return errors.Wrap(err, "parse slot to uint64", z.Str("addr", addr)) + return errors.Wrap(err, "parse slot to uint64", z.Str("addr", p.label(addr))) } depth, err := strconv.ParseUint(chainReorg.Depth, 10, 64) if err != nil { - return errors.Wrap(err, "parse depth to uint64", z.Str("addr", addr)) + return errors.Wrap(err, "parse depth to uint64", z.Str("addr", p.label(addr))) } if slot < depth { @@ -297,7 +312,7 @@ func (p *listener) handleChainReorgEvent(ctx context.Context, event *event, addr z.Str("old_head_block", chainReorg.OldHeadBlock), z.Str("new_head_block", chainReorg.NewHeadBlock)) - sseChainReorgDepthHistogram.WithLabelValues(addr).Observe(float64(depth)) + sseChainReorgDepthHistogram.WithLabelValues(p.label(addr)).Observe(float64(depth)) return nil } @@ -307,12 +322,12 @@ func (p *listener) handleBlockGossipEvent(ctx context.Context, event *event, add err := json.Unmarshal(event.Data, &blockGossip) if err != nil { - return errors.Wrap(err, "unmarshal SSE block_gossip event", z.Str("addr", addr)) + return errors.Wrap(err, "unmarshal SSE block_gossip event", z.Str("addr", p.label(addr))) } slot, err := strconv.ParseUint(blockGossip.Slot, 10, 64) if err != nil { - return errors.Wrap(err, "parse slot to uint64", z.Str("addr", addr)) + return errors.Wrap(err, "parse slot to uint64", z.Str("addr", p.label(addr))) } // Beacon node should receive a block via P2P or API before the attestation due offset. @@ -326,7 +341,7 @@ func (p *listener) handleBlockGossipEvent(ctx context.Context, event *event, add z.Str("delay", delay.String()), z.Str("block", blockGossip.Block)) - sseBlockGossipHistogram.WithLabelValues(addr).Observe(delay.Seconds()) + sseBlockGossipHistogram.WithLabelValues(p.label(addr)).Observe(delay.Seconds()) p.storeBlockGossipTime(slot, addr, event.Timestamp) @@ -338,12 +353,12 @@ func (p *listener) handleBlockEvent(ctx context.Context, event *event, addr stri err := json.Unmarshal(event.Data, &block) if err != nil { - return errors.Wrap(err, "unmarshal SSE block event", z.Str("addr", addr)) + return errors.Wrap(err, "unmarshal SSE block event", z.Str("addr", p.label(addr))) } slot, err := strconv.ParseUint(block.Slot, 10, 64) if err != nil { - return errors.Wrap(err, "parse slot to uint64", z.Str("addr", addr)) + return errors.Wrap(err, "parse slot to uint64", z.Str("addr", p.label(addr))) } // Beacon node should import a block to its fork-choice before the attestation due offset. @@ -357,7 +372,7 @@ func (p *listener) handleBlockEvent(ctx context.Context, event *event, addr stri z.Str("delay", delay.String()), z.Str("block", block.Block)) - sseBlockHistogram.WithLabelValues(addr).Observe(delay.Seconds()) + sseBlockHistogram.WithLabelValues(p.label(addr)).Observe(delay.Seconds()) return nil } @@ -461,7 +476,7 @@ func (p *listener) recordBlockProcessingTime(slot uint64, addr string, headTimes processingTime := headTimestamp.Sub(gossipTime) if processingTime > 0 { - sseBlockProcessingTimeHistogram.WithLabelValues(addr).Observe(processingTime.Seconds()) + sseBlockProcessingTimeHistogram.WithLabelValues(p.label(addr)).Observe(processingTime.Seconds()) } // Clean up this entry as it's no longer needed diff --git a/cmd/exit_broadcast.go b/cmd/exit_broadcast.go index bae31cb76..19208d9d0 100644 --- a/cmd/exit_broadcast.go +++ b/cmd/exit_broadcast.go @@ -138,7 +138,7 @@ func runBcastFullExit(ctx context.Context, config exitConfig) error { eth2Cl, err := eth2Client(ctx, config.FallbackBeaconNodeAddrs, beaconNodeHeaders, config.BeaconNodeEndpoints, config.BeaconNodeTimeout, [4]byte(cl.ForkVersion)) if err != nil { - return errors.Wrap(err, "create eth2 client for specified beacon node(s)", z.Any("beacon_nodes_endpoints", config.BeaconNodeEndpoints)) + return errors.Wrap(err, "create eth2 client for specified beacon node(s)", z.Any("beacon_nodes_endpoints", log.RedactURLs(config.BeaconNodeEndpoints, true))) } fullExits := make(map[core.PubKey]eth2p0.SignedVoluntaryExit) diff --git a/cmd/exit_list.go b/cmd/exit_list.go index 3951da113..7b1d69e5d 100644 --- a/cmd/exit_list.go +++ b/cmd/exit_list.go @@ -102,7 +102,7 @@ func listActiveVals(ctx context.Context, config exitConfig) ([]string, error) { eth2Cl, err := eth2Client(ctx, config.FallbackBeaconNodeAddrs, beaconNodeHeaders, config.BeaconNodeEndpoints, config.BeaconNodeTimeout, [4]byte{}) // fine to avoid initializing a fork version, we're just querying the BN if err != nil { - return nil, errors.Wrap(err, "create eth2 client for specified beacon node(s)", z.Any("beacon_nodes_endpoints", config.BeaconNodeEndpoints)) + return nil, errors.Wrap(err, "create eth2 client for specified beacon node(s)", z.Any("beacon_nodes_endpoints", log.RedactURLs(config.BeaconNodeEndpoints, true))) } var allVals []eth2p0.BLSPubKey diff --git a/cmd/exit_sign.go b/cmd/exit_sign.go index c32ac8a6d..93c53b120 100644 --- a/cmd/exit_sign.go +++ b/cmd/exit_sign.go @@ -139,7 +139,7 @@ func runSignPartialExit(ctx context.Context, config exitConfig) error { eth2Cl, err := eth2Client(ctx, config.FallbackBeaconNodeAddrs, beaconNodeHeaders, config.BeaconNodeEndpoints, config.BeaconNodeTimeout, [4]byte(cl.ForkVersion)) if err != nil { - return errors.Wrap(err, "create eth2 client for specified beacon node(s)", z.Any("beacon_nodes_endpoints", config.BeaconNodeEndpoints)) + return errors.Wrap(err, "create eth2 client for specified beacon node(s)", z.Any("beacon_nodes_endpoints", log.RedactURLs(config.BeaconNodeEndpoints, true))) } if config.ValidatorIndexPresent { diff --git a/core/fetcher/fetcher.go b/core/fetcher/fetcher.go index ea2dca964..54857478c 100644 --- a/core/fetcher/fetcher.go +++ b/core/fetcher/fetcher.go @@ -156,7 +156,7 @@ func (f *Fetcher) FetchOnly(ctx context.Context, duty core.Duty, defSet core.Dut if attData.Data.BeaconBlockRoot != headBlockRoot { log.Debug(ctx, "Skipping early attestation cache: fetched head differs from head event", - z.U64("slot", duty.Slot), z.Str("bn_addr", bnAddr), + z.U64("slot", duty.Slot), z.Str("bn_addr", log.RedactURL(bnAddr, true)), z.Str("head_event_root", headBlockRoot.String()), z.Str("fetched_root", attData.Data.BeaconBlockRoot.String())) @@ -165,7 +165,7 @@ func (f *Fetcher) FetchOnly(ctx context.Context, duty core.Duty, defSet core.Dut } f.attDataCache.Store(duty.Slot, unsignedSet) - log.Debug(ctx, "Early attestation data fetched and cached", z.U64("slot", duty.Slot), z.Str("bn_addr", bnAddr)) + log.Debug(ctx, "Early attestation data fetched and cached", z.U64("slot", duty.Slot), z.Str("bn_addr", log.RedactURL(bnAddr, true))) return nil } @@ -193,7 +193,7 @@ func (f *Fetcher) HandleHeadEvent(ctx context.Context, slot eth2p0.Slot, root et select { case events <- headEvent{root: root, bnAddr: bnAddr}: default: - log.Debug(ctx, "Dropping head event, buffer full", z.U64("slot", uint64(slot)), z.Str("bn_addr", bnAddr)) + log.Debug(ctx, "Dropping head event, buffer full", z.U64("slot", uint64(slot)), z.Str("bn_addr", log.RedactURL(bnAddr, true))) } } @@ -236,14 +236,14 @@ func (f *Fetcher) fetchGloasAttesterData(ctx context.Context, slot uint64, defSe unsignedSet, err := f.fetchAttesterDataFrom(ctx, slot, defSet, event.bnAddr) if err != nil { log.Warn(ctx, "Attestation data fetch upon head event failed, waiting for next head event", err, - z.U64("slot", slot), z.Str("bn_addr", event.bnAddr)) + z.U64("slot", slot), z.Str("bn_addr", log.RedactURL(event.bnAddr, true))) continue } if !votesFor(unsignedSet, event.root) { log.Debug(ctx, "Attestation data fetched upon head event votes for another head, waiting for next head event", - z.U64("slot", slot), z.Str("bn_addr", event.bnAddr), z.Str("head_event_root", event.root.String())) + z.U64("slot", slot), z.Str("bn_addr", log.RedactURL(event.bnAddr, true)), z.Str("head_event_root", event.root.String())) continue } diff --git a/core/scheduler/scheduler.go b/core/scheduler/scheduler.go index 60656802b..5daaa81ef 100644 --- a/core/scheduler/scheduler.go +++ b/core/scheduler/scheduler.go @@ -224,7 +224,7 @@ func (s *Scheduler) HandleChainReorgEvent(ctx context.Context, epoch eth2p0.Epoc // networks then fetch attestation data at the deadline, while the gloas fork checks must remain. func (s *Scheduler) HandleHeadEvent(ctx context.Context, slot eth2p0.Slot, blockRoot eth2p0.Root, bnAddr string) { if s.fetcherFetchOnly == nil { - log.Warn(ctx, "Early attestation data fetch skipped, fetcher fetch-only function not registered", nil, z.U64("slot", uint64(slot)), z.Str("bn_addr", bnAddr)) + log.Warn(ctx, "Early attestation data fetch skipped, fetcher fetch-only function not registered", nil, z.U64("slot", uint64(slot)), z.Str("bn_addr", log.RedactURL(bnAddr, true))) return } @@ -267,7 +267,7 @@ func (s *Scheduler) HandleHeadEvent(ctx context.Context, slot eth2p0.Slot, block return } - log.Debug(ctx, "Early attestation data fetch triggered by SSE head event", z.U64("slot", uint64(slot)), z.Str("bn_addr", bnAddr)) + log.Debug(ctx, "Early attestation data fetch triggered by SSE head event", z.U64("slot", uint64(slot)), z.Str("bn_addr", log.RedactURL(bnAddr, true))) // Fetch attestation data early without triggering consensus // Use background context to prevent cancellation if SSE connection drops @@ -275,7 +275,7 @@ func (s *Scheduler) HandleHeadEvent(ctx context.Context, slot eth2p0.Slot, block go func() { fetchCtx := log.CopyFields(context.Background(), ctx) if err := s.fetcherFetchOnly(fetchCtx, duty, clonedDefSet, bnAddr, blockRoot); err != nil { - log.Warn(fetchCtx, "Early attestation data fetch failed", err, z.U64("slot", uint64(slot)), z.Str("bn_addr", bnAddr)) + log.Warn(fetchCtx, "Early attestation data fetch failed", err, z.U64("slot", uint64(slot)), z.Str("bn_addr", log.RedactURL(bnAddr, true))) } }() }