From 4f15fbc2f18ccb332e91dc6574da6f6c358e449c Mon Sep 17 00:00:00 2001 From: Matthew Jacobs Date: Thu, 19 Nov 2026 13:23:59 -0800 Subject: [PATCH 1/2] Recover verified transcripts when the daemon truncates a conversation --- docs/completeness.md | 22 +++ internal/cache/sidecar.go | 2 +- internal/cli/audit_qualify.go | 3 + internal/cli/audit_sweep.go | 4 + internal/completeness/attestation.go | 34 +++- .../completeness/projection_subagents148.go | 22 ++- .../projection_subagents148_test.go | 8 +- internal/completeness/schema.go | 13 +- internal/completeness/sqlite.go | 47 ++++- internal/completeness/tools158_test.go | 3 +- internal/completeness/transcript.go | 32 +++ internal/completeness/transcript_test.go | 31 +++ internal/daemon/client.go | 83 +++++--- internal/daemon/recovery.go | 187 ++++++++++++++++++ internal/daemon/recovery_test.go | 51 +++++ internal/doctor/doctor.go | 11 +- internal/session/session.go | 24 ++- internal/watcher/refusals.go | 2 +- internal/watcher/watcher.go | 2 +- 19 files changed, 517 insertions(+), 64 deletions(-) create mode 100644 internal/completeness/transcript.go create mode 100644 internal/completeness/transcript_test.go create mode 100644 internal/daemon/recovery.go create mode 100644 internal/daemon/recovery_test.go diff --git a/docs/completeness.md b/docs/completeness.md index acfe877..bd52e52 100644 --- a/docs/completeness.md +++ b/docs/completeness.md @@ -1756,3 +1756,25 @@ occur together in a stored command assessor configuration in executor metadata. The full and ordinary responses preserve all three, including explicit boolean presence. SQLite verification and offline JSON retain them under the same equality rules as the surrounding metadata. + +### Partial responses and verified transcripts + +When agy reports more records than it returned, the reader resumes each short +list at its returned length. Verification still compares the assembled full +and presented responses with SQLite. An empty page before the advertised end, +a count overrun, or a content mismatch prevents verified publication. + +For SQLite sessions whose executor metadata would exceed the cumulative +64 MiB source budget, sync can publish a smaller representation. A +`verified-transcript-v1` attestation covers the header, steps, generator +metadata, parent references, and battle-mode records. The sidecar omits +`executorMetadatas` and records the number excluded in +`agyReader.completeness.omittedExecutorMetadata`. Changes confined to those +omitted payloads are outside its digest. Both daemon envelopes must report the +source counts, and the reader checks the source again before publication. +Doctor counts these transcripts separately. Full-source exports and format +qualification still require full coverage. + +The `killall1512` profile verifies the observed single-child `kill_all` +result against the child's identity in both persisted result lists. The +presented action and both ID strings must match. diff --git a/internal/cache/sidecar.go b/internal/cache/sidecar.go index f5e08a1..86fdac2 100644 --- a/internal/cache/sidecar.go +++ b/internal/cache/sidecar.go @@ -88,7 +88,7 @@ func WriteVerified(sidecarPath string, t *daemon.Trajectory, verifiedAt time.Tim if sourcePath != "" { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - snapshot, err := completeness.ReadSnapshot(ctx, sourcePath) + snapshot, err := completeness.ReadSyncSnapshot(ctx, sourcePath) if err != nil { return fmt.Errorf("cache: completeness unknown: %w", err) } diff --git a/internal/cli/audit_qualify.go b/internal/cli/audit_qualify.go index fe62882..69dc564 100644 --- a/internal/cli/audit_qualify.go +++ b/internal/cli/audit_qualify.go @@ -201,6 +201,9 @@ func liveQualification(sessions []discovery.Session) func(string) (string, strin return string(completeness.Unknown), "no sidecar: ordinary sync has not verified and written this conversation" } r := session.Completeness(s) + if r.Reason == completeness.TranscriptReason { + return "transcript-only", r.Reason + } return string(r.State), r.Reason } } diff --git a/internal/cli/audit_sweep.go b/internal/cli/audit_sweep.go index 5d5dd62..f53b2a8 100644 --- a/internal/cli/audit_sweep.go +++ b/internal/cli/audit_sweep.go @@ -241,6 +241,10 @@ func qualifySweepSession(ctx context.Context, client *daemon.Client, s discovery entry.ReaderSyncReason = t.Verification.Reason entry.SourceDigest = t.Verification.SourceDigest entry.Profile = t.Verification.Profile + if t.Verification.OmittedExecutors > 0 { + entry.ReaderSync = "transcript-only" + entry.ReaderSyncReason = completeness.TranscriptReason + } } case errors.As(err, &contentErr): entry.ReaderSync = string(contentErr.Result.State) diff --git a/internal/completeness/attestation.go b/internal/completeness/attestation.go index cef5ec7..5c75a28 100644 --- a/internal/completeness/attestation.go +++ b/internal/completeness/attestation.go @@ -17,17 +17,27 @@ import ( // SourceOnlyReason and SourceOnly records that content; both must match the // snapshot on every recheck. type Attestation struct { - State string `json:"state,omitempty"` - Profile string `json:"profile"` - SourceDigest string `json:"sourceDigest"` - PayloadDigest string `json:"payloadDigest"` - Representation string `json:"representation"` - Reason string `json:"reason,omitempty"` - SourceOnly []SourceOnlyField `json:"sourceOnly,omitempty"` + OmittedExecutors int `json:"omittedExecutorMetadata,omitempty"` + State string `json:"state,omitempty"` + Profile string `json:"profile"` + SourceDigest string `json:"sourceDigest"` + PayloadDigest string `json:"payloadDigest"` + Representation string `json:"representation"` + Reason string `json:"reason,omitempty"` + SourceOnly []SourceOnlyField `json:"sourceOnly,omitempty"` } const representation = "supported-daemon-projection" +const TranscriptRepresentation = "verified-transcript-v1" + +func (s *Snapshot) representation() string { + if s.OmittedExecutors > 0 { + return TranscriptRepresentation + } + return representation +} + func PayloadDigest(raw []byte) (string, error) { var top map[string]any decoder := json.NewDecoder(bytes.NewReader(raw)) @@ -67,11 +77,14 @@ func (s *Snapshot) Verify(fullRaw, presentation []byte) (a *Attestation, r Resul if err != nil { return nil, Result{Unknown, "invalid trajectory JSON"}, false } - a = &Attestation{Profile: s.Profile, SourceDigest: s.Digest, PayloadDigest: digest, Representation: representation} + a = &Attestation{Profile: s.Profile, SourceDigest: s.Digest, PayloadDigest: digest, Representation: s.representation(), OmittedExecutors: s.OmittedExecutors} if len(checked.sourceOnly) > 0 { a.Reason, a.SourceOnly = SourceOnlyReason, checked.sourceOnly r = Result{Verified, SourceOnlyReason} } + if s.OmittedExecutors > 0 { + r.Reason = TranscriptReason + } return a, r, false } @@ -97,7 +110,7 @@ func (s *Snapshot) CheckAttestation(presentation []byte, a *Attestation) Result if s == nil { return Result{Unknown, "source snapshot unavailable"} } - if a == nil || a.State != "" || a.Profile != s.Profile || a.Representation != representation { + if a == nil || a.State != "" || a.Profile != s.Profile || a.Representation != s.representation() || a.OmittedExecutors != s.OmittedExecutors { return Result{Unknown, "missing or unsupported completeness attestation"} } digest, err := PayloadDigest(presentation) @@ -143,6 +156,9 @@ func (s *Snapshot) CheckAttestation(presentation []byte, a *Attestation) Result if r.State == Verified && len(checked.sourceOnly) > 0 { r.Reason = SourceOnlyReason } + if r.State == Verified && s.OmittedExecutors > 0 { + r.Reason = TranscriptReason + } return r } diff --git a/internal/completeness/projection_subagents148.go b/internal/completeness/projection_subagents148.go index 9fc53b7..29db409 100644 --- a/internal/completeness/projection_subagents148.go +++ b/internal/completeness/projection_subagents148.go @@ -9,7 +9,7 @@ import ( "google.golang.org/protobuf/reflect/protoreflect" ) -// projectManageSubagents148 covers list results and one- or two-child kills. State +// projectManageSubagents148 covers list, targeted kills, and single-child kill_all results. State // stays in the payload; only present step indices enter the metadata JSON. func projectManageSubagents148(step, generic, result, payload protoreflect.Message) (proto.Message, error) { unsupported := errors.New("unsupported manage_subagents 1.4.8 projection") @@ -45,6 +45,10 @@ func projectManageSubagents148(step, generic, result, payload protoreflect.Messa return nil, unsupported } indices[id] = state.Get(indexField).Int() + case "kill_all": + if ids.Len() != 1 || !genericArgsAllowed(generic, "Action") || state.Has(indexField) { + return nil, unsupported + } case "kill": if !genericArgsAllowed(generic, "Action", "ConversationIds") || state.Has(indexField) { return nil, unsupported @@ -61,15 +65,17 @@ func projectManageSubagents148(step, generic, result, payload protoreflect.Messa } metadata["subagent_step_indices"] = string(b) } else { - arg, ok := genericArg(generic, "ConversationIds") - var requested []string - if !ok || json.Unmarshal([]byte(arg), &requested) != nil || len(requested) != len(names) { - return nil, unsupported - } - for i := range names { - if requested[i] != names[i] { + if action == "kill" { + arg, ok := genericArg(generic, "ConversationIds") + var requested []string + if !ok || json.Unmarshal([]byte(arg), &requested) != nil || len(requested) != len(names) { return nil, unsupported } + for i := range names { + if requested[i] != names[i] { + return nil, unsupported + } + } } b, err := json.Marshal(names) if err != nil { diff --git a/internal/completeness/projection_subagents148_test.go b/internal/completeness/projection_subagents148_test.go index 19d08cf..48038dc 100644 --- a/internal/completeness/projection_subagents148_test.go +++ b/internal/completeness/projection_subagents148_test.go @@ -18,8 +18,10 @@ func subagents148Pair(t *testing.T, action string) (string, string) { state["stepIndex"] = -1 meta["subagent_step_indices"] = `{"fixture-child":-1}` } else { - args["ConversationIds"] = []string{"fixture-child"} - genericArgs["ConversationIds"] = `["fixture-child"]` + if action == "kill" { + args["ConversationIds"] = []string{"fixture-child"} + genericArgs["ConversationIds"] = `["fixture-child"]` + } meta["killed_conversation_ids"] = `["fixture-child"]` } call, _ := json.Marshal(args) @@ -33,7 +35,7 @@ func subagents148Pair(t *testing.T, action string) (string, string) { } func TestSubagents148(t *testing.T) { - for _, action := range []string{"list", "kill"} { + for _, action := range []string{"list", "kill", "kill_all"} { t.Run(action, func(t *testing.T) { raw, shown := subagents148Pair(t, action) s := snapshot1216(t, raw) diff --git a/internal/completeness/schema.go b/internal/completeness/schema.go index 2480f02..bbc7a22 100644 --- a/internal/completeness/schema.go +++ b/internal/completeness/schema.go @@ -141,7 +141,7 @@ import ( // policy1510 retains the three PolicyGuardianConfig fields observed in executor metadata. // refs1511 retains the observed agy 1.5.11 render-reference fields. // tools1512 projects the observed skill, file, command, timer and list results. -const Profile = "sqlite-v1-format0-agy-1.3.4-5b93565c5f04-planner50-permission2-prompt23-tools143-permission1-tools144-skill1-history2-ide225-options145-compaction146-skill146-finish147-replay147-failedread146-subagents148-media149-tools149-sendmessage149-files1411-media1410-parentmsgs1410-paths1410-denied1412-cwd1412-unrun1412-config150-skill150-frontmatter150-bgcancel150-readlines150-config152-tools153-tools154-tools155-images155-tools156-retained156-legacychoices156-runtime157-planexecute157-tools158-workflow159-schedule1510-policy1510-refs1511-tools1512" +const Profile = "sqlite-v1-format0-agy-1.3.4-5b93565c5f04-planner50-permission2-prompt23-tools143-permission1-tools144-skill1-history2-ide225-options145-compaction146-skill146-finish147-replay147-failedread146-subagents148-media149-tools149-sendmessage149-files1411-media1410-parentmsgs1410-paths1410-denied1412-cwd1412-unrun1412-config150-skill150-frontmatter150-bgcancel150-readlines150-config152-tools153-tools154-tools155-images155-tools156-retained156-legacychoices156-runtime157-planexecute157-tools158-workflow159-schedule1510-policy1510-refs1511-tools1512-killall1512" const maxBytes = 64 << 20 const maxDepth = 100 @@ -356,11 +356,12 @@ func normalizeIn(m protoreflect.Message, depth int, scope sourceOnlyScope) error // removed and recorded in sourceOnly. Legacy model choices keep their original // fields until a complete RPC match selects the source-only alternative. type Snapshot struct { - Profile string - Digest string - StepCount int - trajectory proto.Message - sourceOnly []SourceOnlyField + Profile string + Digest string + StepCount int + OmittedExecutors int + trajectory proto.Message + sourceOnly []SourceOnlyField } type State string diff --git a/internal/completeness/sqlite.go b/internal/completeness/sqlite.go index 3bb0a8a..d4133ac 100644 --- a/internal/completeness/sqlite.go +++ b/internal/completeness/sqlite.go @@ -54,14 +54,27 @@ func ReadSnapshot(ctx context.Context, path string) (*Snapshot, error) { if filepath.Ext(path) == ".pb" { return readLegacySnapshot(ctx, path) } - s, err := readSnapshot(ctx, path) + s, err := readSnapshot(ctx, path, false) if err != nil && notADatabase(err) { return nil, refuse(err) } return s, err } -func readSnapshot(ctx context.Context, path string) (_ *Snapshot, resultErr error) { +// ReadSyncSnapshot permits a transcript-only snapshot when executor metadata +// would exhaust the source budget. Offline exports retain their full-source contract. +func ReadSyncSnapshot(ctx context.Context, path string) (*Snapshot, error) { + if filepath.Ext(path) != ".db" { + return ReadSnapshot(ctx, path) + } + s, err := readSnapshot(ctx, path, true) + if err != nil && notADatabase(err) { + return nil, refuse(err) + } + return s, err +} + +func readSnapshot(ctx context.Context, path string, allowTranscript bool) (_ *Snapshot, resultErr error) { path, err := filepath.Abs(path) if err != nil { return nil, err @@ -145,12 +158,12 @@ func readSnapshot(ctx context.Context, path string) (_ *Snapshot, resultErr erro if err = readSteps(ctx, tx, out, &budget); err != nil { return nil, err } + omitted := 0 for _, list := range []struct { table string field protoreflect.Name }{ {"gen_metadata", "generator_metadata"}, - {"executor_metadata", "executor_metadatas"}, {"parent_references", "parent_references"}, {"battle_mode_infos", "battle_mode_infos"}, } { @@ -195,14 +208,40 @@ func readSnapshot(ctx context.Context, path string) (_ *Snapshot, resultErr erro if count != 1 { return nil, refusal("source must contain one trajectory metadata record") } + // Measure executor records without materializing their blobs. Check them + // last so the rest of the transcript has already paid its cumulative cost. + if allowTranscript { + var count, first, last int + var size int64 + err = tx.QueryRowContext(ctx, "SELECT count(*), coalesce(min(idx),0), coalesce(max(idx),-1), coalesce(sum(length(data)),0) FROM executor_metadata").Scan(&count, &first, &last, &size) + if err != nil { + return nil, err + } + if size > int64(budget) { + if count < 1 || count > 100000 || first != 0 || last != count-1 { + return nil, refusal("unsupported metadata indexing") + } + omitted = count + } + } + if omitted == 0 { + if err = readList(ctx, tx, "executor_metadata", out, "executor_metadatas", &budget); err != nil { + return nil, err + } + } if err = tx.Commit(); err != nil { return nil, err } // finishSnapshot reads nothing but the content already read. - snapshot, err := finishSnapshot(m) + profile := Profile + if omitted > 0 { + profile = Profile + "/" + TranscriptRepresentation + } + snapshot, err := finishSnapshotProfile(m, profile) if err != nil { return nil, refuse(err) } + snapshot.OmittedExecutors = omitted return snapshot, nil } diff --git a/internal/completeness/tools158_test.go b/internal/completeness/tools158_test.go index 9bb9027..3eb4512 100644 --- a/internal/completeness/tools158_test.go +++ b/internal/completeness/tools158_test.go @@ -161,7 +161,8 @@ func TestCurrentProfileInvalidatesPreviousAttestations(t *testing.T) { strings.Split(Profile, "-schedule1510")[0], strings.Split(Profile, "-policy1510")[0], strings.Split(Profile, "-refs1511")[0], - strings.TrimSuffix(Profile, "-tools1512"), + strings.Split(Profile, "-tools1512")[0], + strings.TrimSuffix(Profile, "-killall1512"), } { a.Profile = previous if a.Profile == Profile { diff --git a/internal/completeness/transcript.go b/internal/completeness/transcript.go new file mode 100644 index 0000000..f48e8c3 --- /dev/null +++ b/internal/completeness/transcript.go @@ -0,0 +1,32 @@ +package completeness + +import ( + "encoding/json" + + "google.golang.org/protobuf/encoding/protojson" +) + +const TranscriptReason = "verified transcript only; executor metadata omitted" + +const MaxResponseBytes = maxBytes + +func (s *Snapshot) GeneratorCount() int { + m := s.trajectory.ProtoReflect() + return m.Get(m.Descriptor().Fields().ByName("generator_metadata")).List().Len() +} + +// TranscriptHeaderJSON uses the pinned trajectory descriptor to preserve the +// header independently of the daemon's truncated repeated record fields. +func TranscriptHeaderJSON(header []byte) (map[string]json.RawMessage, error) { + m, err := decode(trajectoryName, header) + if err != nil { + return nil, err + } + raw, err := (protojson.MarshalOptions{Resolver: schemaTypes}).Marshal(m) + if err != nil { + return nil, err + } + var fields map[string]json.RawMessage + err = json.Unmarshal(raw, &fields) + return fields, err +} diff --git a/internal/completeness/transcript_test.go b/internal/completeness/transcript_test.go new file mode 100644 index 0000000..42b5052 --- /dev/null +++ b/internal/completeness/transcript_test.go @@ -0,0 +1,31 @@ +package completeness + +import ( + "github.com/mjacobs/agy-reader/internal/testfixture" + "path/filepath" + "testing" +) + +func TestTranscriptScope(t *testing.T) { + p := filepath.Join(t.TempDir(), "source.db") + db := testfixture.Source(t, p, []byte(syntheticRaw)) + testfixture.Exec(t, db, "INSERT INTO executor_metadata(idx,data) VALUES (0, zeroblob(67108865))") + s, err := ReadSyncSnapshot(t.Context(), p) + if err != nil { + t.Fatal(err) + } + if s.OmittedExecutors != 1 { + t.Fatal(s.OmittedExecutors) + } + a, r := s.VerifyResponses([]byte(syntheticRaw), []byte(syntheticPresentation)) + if r.State != Verified || r.Reason != TranscriptReason || a.Representation != TranscriptRepresentation { + t.Fatal(r) + } + if s.CheckAttestation([]byte(syntheticPresentation), a).Reason != TranscriptReason { + t.Fatal("scope lost on recheck") + } + a.OmittedExecutors = 2 + if s.CheckAttestation([]byte(syntheticPresentation), a).State == Verified { + t.Fatal("accepted changed omission count") + } +} diff --git a/internal/daemon/client.go b/internal/daemon/client.go index eeecab2..5e0b3cc 100644 --- a/internal/daemon/client.go +++ b/internal/daemon/client.go @@ -82,25 +82,26 @@ func (c *Client) LoadTrajectory(ctx context.Context, cascadeID string) error { // GetCascadeTrajectory fetches the decrypted trajectory. func (c *Client) GetCascadeTrajectory(ctx context.Context, cascadeID string) (*Trajectory, error) { - return c.getTrajectory(ctx, GetCascadeTrajectoryRequest{CascadeID: cascadeID}) + return c.getTrajectory(ctx, cascadeID, false, GetCascadeTrajectoryRequest{CascadeID: cascadeID}) } // GetFullRawTrajectory exposes the persisted representation for comparison; // callers retain the ordinary response for the public sidecar contract. func (c *Client) GetFullRawTrajectory(ctx context.Context, cascadeID string) (*Trajectory, error) { - return c.getTrajectory(ctx, struct { + return c.getTrajectory(ctx, cascadeID, true, struct { CascadeID string `json:"cascadeId"` Verbosity string `json:"trajectoryVerbosity"` DisableRehydration bool `json:"disableRehydration"` }{cascadeID, "CLIENT_TRAJECTORY_VERBOSITY_FULL", true}) } -func (c *Client) getTrajectory(ctx context.Context, request any) (*Trajectory, error) { +func (c *Client) getTrajectory(ctx context.Context, id string, full bool, request any) (*Trajectory, error) { // Decode the envelope only. The trajectory object stays raw until we have // retained an owned copy, otherwise typed unmarshalling would discard new // daemon fields before cache.Write ever sees them. var resp struct { Trajectory json.RawMessage `json:"trajectory"` + recordCounts } if err := c.call(ctx, "GetCascadeTrajectory", request, &resp); err != nil { return nil, err @@ -111,11 +112,38 @@ func (c *Client) getTrajectory(ctx context.Context, request any) (*Trajectory, e // It decodes to the same zero Trajectory the old typed envelope produced. resp.Trajectory = json.RawMessage(`{}`) } + if resp.Steps != nil || resp.Generators != nil || resp.Executors != nil { + var fields map[string]json.RawMessage + if err := json.Unmarshal(resp.Trajectory, &fields); err != nil { + return nil, err + } + if fields == nil { + return nil, errors.New("missing trajectory") + } + changed, err := c.completeLists(ctx, id, full, fields, resp.recordCounts) + if err != nil { + return nil, err + } + if !changed { + return trajectoryFromJSON(resp.Trajectory) + } + resp.Trajectory, err = json.Marshal(fields) + if len(resp.Trajectory) > responseLimit { + return nil, errors.New("recovered trajectory exceeds size limit") + } + if err != nil { + return nil, err + } + } + return trajectoryFromJSON(resp.Trajectory) +} + +func trajectoryFromJSON(data json.RawMessage) (*Trajectory, error) { var traj Trajectory - if err := json.Unmarshal(resp.Trajectory, &traj); err != nil { + if err := json.Unmarshal(data, &traj); err != nil { return nil, fmt.Errorf("unmarshal GetCascadeTrajectory trajectory: %w", err) } - traj.RawJSON = append(json.RawMessage(nil), resp.Trajectory...) + traj.RawJSON = data return &traj, nil } @@ -184,19 +212,33 @@ func (c *Client) CheckResponds(ctx context.Context) error { } func (c *Client) call(ctx context.Context, method string, body, out any) error { - endpoint, err := url.JoinPath(c.BaseURL, rpcPathPrefix+method) + payload, err := json.Marshal(body) if err != nil { - return fmt.Errorf("build url: %w", err) + return err } - payload, err := json.Marshal(body) + data, err := c.request(ctx, method, "application/json", payload, responseLimit) if err != nil { - return fmt.Errorf("marshal request: %w", err) + return err + } + if out == nil { + return nil + } + if err = json.Unmarshal(data, out); err != nil { + return fmt.Errorf("unmarshal %s response: %w", method, err) + } + return nil +} + +func (c *Client) request(ctx context.Context, method, media string, payload []byte, limit int) ([]byte, error) { + endpoint, err := url.JoinPath(c.BaseURL, rpcPathPrefix+method) + if err != nil { + return nil, err } req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(payload)) if err != nil { - return fmt.Errorf("new request: %w", err) + return nil, fmt.Errorf("new request: %w", err) } - req.Header.Set("Content-Type", "application/json") + req.Header.Set("Content-Type", media) if c.CSRFToken != "" { req.Header.Set(CSRFHeader, c.CSRFToken) } @@ -207,11 +249,11 @@ func (c *Client) call(ctx context.Context, method string, body, out any) error { } resp, err := httpClient.Do(req) if err != nil { - return fmt.Errorf("call %s: %w (is `agy` running?)", method, err) + return nil, fmt.Errorf("call %s: %w (is `agy` running?)", method, err) } defer resp.Body.Close() - respBody, readErr := io.ReadAll(resp.Body) + respBody, readErr := io.ReadAll(io.LimitReader(resp.Body, int64(limit)+1)) if resp.StatusCode >= 300 { // The status line is the answer, so it is reported even when the body // that follows is cut short. @@ -219,7 +261,7 @@ func (c *Client) call(ctx context.Context, method string, body, out any) error { if statusErr.authentication() { // Don't echo an authentication response body: it could contain a // credential supplied by the caller. - return statusErr + return nil, statusErr } statusErr.Message = string(respBody) var errEnv struct { @@ -230,16 +272,13 @@ func (c *Client) call(ctx context.Context, method string, body, out any) error { } else if json.Unmarshal(respBody, &errEnv) == nil && errEnv.Message != "" { statusErr.Message = errEnv.Message } - return statusErr + return nil, statusErr } if readErr != nil { - return fmt.Errorf("read response: %w", readErr) + return nil, fmt.Errorf("read response: %w", readErr) } - if out == nil { - return nil + if len(respBody) > limit { + return nil, fmt.Errorf("%s response exceeds size limit", method) } - if err := json.Unmarshal(respBody, out); err != nil { - return fmt.Errorf("unmarshal %s response: %w", method, err) - } - return nil + return respBody, nil } diff --git a/internal/daemon/recovery.go b/internal/daemon/recovery.go new file mode 100644 index 0000000..5bf5bcd --- /dev/null +++ b/internal/daemon/recovery.go @@ -0,0 +1,187 @@ +package daemon + +import ( + "context" + "encoding/json" + "errors" + "fmt" + + "github.com/mjacobs/agy-reader/internal/completeness" + "google.golang.org/protobuf/encoding/protowire" +) + +const responseLimit = completeness.MaxResponseBytes + +type recordCounts struct { + Steps *int `json:"numTotalSteps"` + Generators *int `json:"numTotalGeneratorMetadata"` + Executors *int `json:"numTotalExecutorMetadata"` +} + +// completeLists resumes a truncated list at its exact offset. The caller checks +// the assembled raw and presented records against one persisted snapshot. +func (c *Client) completeLists(ctx context.Context, id string, full bool, fields map[string]json.RawMessage, counts recordCounts) (bool, error) { + changed := false + budget := responseLimit + for k, v := range fields { + budget -= len(k) + len(v) + 4 + } + for _, spec := range []struct { + field, method, offset, result string + total *int + }{ + {"steps", "GetCascadeTrajectorySteps", "stepOffset", "steps", counts.Steps}, + {"generatorMetadata", "GetCascadeTrajectoryGeneratorMetadata", "generatorMetadataOffset", "generatorMetadata", counts.Generators}, + {"executorMetadatas", "GetCascadeTrajectoryExecutorMetadatas", "executorMetadataOffset", "executorMetadata", counts.Executors}, + } { + if spec.total == nil { + continue + } + target := *spec.total + if target < 0 || target > 100000 { + return false, errors.New("invalid trajectory record count") + } + var rows []json.RawMessage + if data, ok := fields[spec.field]; ok { + if err := json.Unmarshal(data, &rows); err != nil { + return false, err + } + } + if len(rows) > target { + return false, errors.New("trajectory exceeds advertised record count") + } + if len(rows) == target { + continue + } + for len(rows) < target { + args := map[string]any{"cascadeId": id, spec.offset: len(rows)} + if spec.field == "steps" && full { + args["trajectoryVerbosity"] = "CLIENT_TRAJECTORY_VERBOSITY_FULL" + args["disableRehydration"] = true + } + if spec.field == "generatorMetadata" { + args["includeMessages"] = true + } + payload, err := json.Marshal(args) + if err != nil { + return false, err + } + if budget <= 0 { + return false, errors.New("recovered trajectory exceeds size limit") + } + page, err := c.request(ctx, spec.method, "application/json", payload, budget) + if err != nil { + return false, err + } + var envelope map[string][]json.RawMessage + if err = json.Unmarshal(page, &envelope); err != nil { + return false, err + } + next := envelope[spec.result] + if len(next) == 0 || len(next) > target-len(rows) { + return false, fmt.Errorf("%s returned invalid page length", spec.method) + } + for _, row := range next { + budget -= len(row) + 1 + } + rows = append(rows, next...) + } + data, err := json.Marshal(rows) + if err != nil { + return false, err + } + fields[spec.field] = data + changed = true + } + return changed, nil +} + +// GetTranscript uses the compact binary envelope because its JSON rendering +// can exceed the response bound even when the daemon capped the protobuf. +func (c *Client) GetTranscript(ctx context.Context, id string, full bool, steps, generators, executors int) (*Trajectory, error) { + req := protowire.AppendString(protowire.AppendTag(nil, 1, protowire.BytesType), id) + if full { + req = protowire.AppendVarint(protowire.AppendTag(req, 3, protowire.VarintType), 3) + req = protowire.AppendVarint(protowire.AppendTag(req, 4, protowire.VarintType), 1) + } + data, err := c.request(ctx, "GetCascadeTrajectory", "application/proto", req, responseLimit) + if err != nil { + return nil, err + } + header, err := transcriptHeader(data, [3]int{steps, generators, executors}) + if err != nil { + return nil, err + } + fields, err := completeness.TranscriptHeaderJSON(header) + if err != nil { + return nil, err + } + if _, err = c.completeLists(ctx, id, full, fields, recordCounts{Steps: &steps, Generators: &generators}); err != nil { + return nil, err + } + raw, err := json.Marshal(fields) + if err != nil { + return nil, err + } + if len(raw) > responseLimit { + return nil, errors.New("transcript exceeds size limit") + } + return trajectoryFromJSON(raw) +} + +func transcriptHeader(data []byte, expected [3]int) ([]byte, error) { + var trajectory []byte + var totals [3]int + bad := errors.New("invalid trajectory protobuf envelope") + seen := map[protowire.Number]bool{} + for len(data) > 0 { + n, t, k := protowire.ConsumeTag(data) + if k < 0 || seen[n] { + return nil, bad + } + seen[n] = true + data = data[k:] + switch { + case n == 1 && t == protowire.BytesType: + v, k := protowire.ConsumeBytes(data) + if k < 0 { + return nil, bad + } + trajectory = v + data = data[k:] + case n >= 2 && n <= 5 && t == protowire.VarintType: + v, k := protowire.ConsumeVarint(data) + if k < 0 || v > 1<<32-1 { + return nil, bad + } + if n >= 3 { + totals[n-3] = int(v) + } + data = data[k:] + default: + return nil, bad + } + } + if trajectory == nil || totals != expected { + return nil, errors.New("daemon trajectory totals differ from source") + } + // The dedicated RPCs supply complete step and generator lists. Executor + // records are outside the transcript representation. + var header []byte + for len(trajectory) > 0 { + n, t, k := protowire.ConsumeField(trajectory) + if k < 0 { + return nil, bad + } + switch n { + case 2, 3, 9: + if t != protowire.BytesType { + return nil, bad + } + default: + header = append(header, trajectory[:k]...) + } + trajectory = trajectory[k:] + } + return header, nil +} diff --git a/internal/daemon/recovery_test.go b/internal/daemon/recovery_test.go new file mode 100644 index 0000000..c198026 --- /dev/null +++ b/internal/daemon/recovery_test.go @@ -0,0 +1,51 @@ +package daemon_test + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/mjacobs/agy-reader/internal/daemon" +) + +func TestRecoverPartialSteps(t *testing.T) { + for _, tc := range []struct { + name, page string + ok bool + }{ + {"success", `{"steps":[{"status":"next"}]}`, true}, + {"early end", `{}`, false}, {"overrun", `{"steps":[{},{}]}`, false}, + } { + t.Run(tc.name, func(t *testing.T) { + calls := 0 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/GetCascadeTrajectory") { + fmt.Fprint(w, `{"trajectory":{"cascadeId":"fixture","steps":[{}],"unknown":1234567890123456789},"numTotalSteps":2}`) + return + } + calls++ + var req struct { + CascadeID string + StepOffset int + DisableRehydration bool + TrajectoryVerbosity string + } + if !strings.HasSuffix(r.URL.Path, "/GetCascadeTrajectorySteps") || json.NewDecoder(r.Body).Decode(&req) != nil || req.CascadeID != "fixture" || req.StepOffset != 1 || !req.DisableRehydration || req.TrajectoryVerbosity != "CLIENT_TRAJECTORY_VERBOSITY_FULL" { + t.Error("wrong continuation") + } + fmt.Fprint(w, tc.page) + })) + defer srv.Close() + got, err := daemon.NewClient(srv.URL).GetFullRawTrajectory(t.Context(), "fixture") + if (err == nil) != tc.ok || calls != 1 { + t.Fatalf("calls=%d err=%v", calls, err) + } + if tc.ok && (len(got.Steps) != 2 || !strings.Contains(string(got.RawJSON), "1234567890123456789")) { + t.Fatal("incomplete assembly") + } + }) + } +} diff --git a/internal/doctor/doctor.go b/internal/doctor/doctor.go index 59bdfa6..59ead2a 100644 --- a/internal/doctor/doctor.go +++ b/internal/doctor/doctor.go @@ -75,8 +75,9 @@ type Report struct { UnsupportedSessions []string // up to 5 unsupported sessions (" ()") Unreadable int // Sources that could not be read now (lock, access) UnreadableReason string // the first unreadable source's error - VerifiedSourceOnly int // verified with declared source-only fields; fresh, not in verified - CoverageErr error // non-nil when sidecar coverage could not be read + VerifiedTranscript int + VerifiedSourceOnly int // verified with declared source-only fields; fresh, not in verified + CoverageErr error // non-nil when sidecar coverage could not be read WatchRunning bool WatchKnown bool } @@ -237,6 +238,9 @@ func WriteReportBody(w io.Writer, r Report) int { if r.VerifiedSourceOnly > 0 { sourceOnly = fmt.Sprintf(" %d verified with source-only fields,", r.VerifiedSourceOnly) } + if r.VerifiedTranscript > 0 { + sourceOnly += fmt.Sprintf(" %d verified transcripts,", r.VerifiedTranscript) + } label := "SQLite" if r.Total > r.SQLite { label = fmt.Sprintf("native (%d SQLite, %d legacy .pb)", r.SQLite, r.Total-r.SQLite) @@ -595,6 +599,9 @@ func PopulateSidecarCoverage(r *Report, root string) { case session.SourceOnlyVerified(string(result.State), result.Reason): r.VerifiedSourceOnly++ fresh = true + case result.State == completeness.Verified && result.Reason == completeness.TranscriptReason: + r.VerifiedTranscript++ + fresh = true case result.State == completeness.Verified: r.Verified++ fresh = true diff --git a/internal/session/session.go b/internal/session/session.go index d3ba621..1a903f2 100644 --- a/internal/session/session.go +++ b/internal/session/session.go @@ -52,7 +52,7 @@ func NewDaemonClient(root string, conn discovery.Connection) *daemon.Client { // Fetch uses one authenticated endpoint and persisted identity for both // representations. A second source snapshot rejects same-step writes in flight. func Fetch(ctx context.Context, client *daemon.Client, s discovery.Session) (*daemon.Trajectory, error) { - before, err := completeness.ReadSnapshot(ctx, s.PBPath) + before, err := completeness.ReadSyncSnapshot(ctx, s.PBPath) if err != nil { refusal := &CompletenessError{Result: completeness.Result{State: completeness.Unknown, Reason: err.Error()}, BeforeRPC: true} var shape *completeness.UnknownShape @@ -69,11 +69,20 @@ func Fetch(ctx context.Context, client *daemon.Client, s discovery.Session) (*da if id == "" { return nil, &CompletenessError{Result: completeness.Result{State: completeness.Unknown, Reason: "persisted cascade identity is empty"}, SourceDigest: before.Digest} } - presentation, err := client.GetCascadeTrajectory(ctx, id) + fetch := func(raw bool) (*daemon.Trajectory, error) { + if before.OmittedExecutors > 0 { + return client.GetTranscript(ctx, id, raw, before.StepCount, before.GeneratorCount(), before.OmittedExecutors) + } + if raw { + return client.GetFullRawTrajectory(ctx, id) + } + return client.GetCascadeTrajectory(ctx, id) + } + presentation, err := fetch(false) if err != nil { return nil, err } - raw, err := client.GetFullRawTrajectory(ctx, id) + raw, err := fetch(true) if err != nil { return nil, err } @@ -91,11 +100,11 @@ func Fetch(ctx context.Context, client *daemon.Client, s discovery.Session) (*da } return nil, refusal } - after, err := completeness.ReadSnapshot(ctx, s.PBPath) + after, err := completeness.ReadSyncSnapshot(ctx, s.PBPath) if err != nil { return nil, Unverified(completeness.Unknown, err.Error()) } - if after.Digest != before.Digest { + if after.Digest != before.Digest || after.OmittedExecutors != before.OmittedExecutors { return nil, Unverified(completeness.Incomplete, "persisted source changed during fetch") } presentation.Verification = attestation @@ -150,6 +159,9 @@ func SourceOnlyVerified(state, reason string) bool { // VerifiedNote is the suffix a sync message adds for a source-only // verification, so that message never reads as a plain verified write. func VerifiedNote(t *daemon.Trajectory) string { + if t != nil && t.Verification != nil && t.Verification.OmittedExecutors > 0 { + return fmt.Sprintf("; verified transcript, %d executor metadata records omitted", t.Verification.OmittedExecutors) + } if t == nil || t.Verification == nil || t.Verification.Reason != completeness.SourceOnlyReason { return "" } @@ -187,7 +199,7 @@ const ( func CompletenessClassified(s discovery.Session) (res completeness.Result, class SourceClass) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - snapshot, err := completeness.ReadSnapshot(ctx, s.PBPath) + snapshot, err := completeness.ReadSyncSnapshot(ctx, s.PBPath) if err != nil { res = completeness.Result{State: completeness.Unknown, Reason: err.Error()} if completeness.IsUnsupported(err) { diff --git a/internal/watcher/refusals.go b/internal/watcher/refusals.go index d32bb38..6adcbc1 100644 --- a/internal/watcher/refusals.go +++ b/internal/watcher/refusals.go @@ -69,7 +69,7 @@ func HeldSourceDigest(path string) string { func heldSourceDigest(path string) (string, error) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() - snapshot, err := completeness.ReadSnapshot(ctx, path) + snapshot, err := completeness.ReadSyncSnapshot(ctx, path) if err != nil { return "", err } diff --git a/internal/watcher/watcher.go b/internal/watcher/watcher.go index 64db1c5..9664b1f 100644 --- a/internal/watcher/watcher.go +++ b/internal/watcher/watcher.go @@ -466,7 +466,7 @@ func captureRefusal(ctx context.Context, client *daemon.Client, s discovery.Sess return err } } - after, err := completeness.ReadSnapshot(ctx, s.PBPath) + after, err := completeness.ReadSyncSnapshot(ctx, s.PBPath) var shape *completeness.UnknownShape if refusal.BeforeRPC && (t.CascadeID != refusal.CaptureID || len(t.Steps) != refusal.CaptureSteps) { return session.Unverified(completeness.Incomplete, "daemon presentation identity or step count differs from source") From b5321089173d010a79689bd3773551ae72b4daf3 Mon Sep 17 00:00:00 2001 From: Matthew Jacobs Date: Thu, 19 Nov 2026 13:33:02 -0800 Subject: [PATCH 2/2] Keep oversized transcripts available to audits and back off failed retries --- internal/cli/audit_sweep.go | 19 +++++++++++-- internal/cli/audit_sweep_test.go | 33 +++++++++++++++++++++ internal/daemon/client.go | 6 ++-- internal/daemon/recovery.go | 4 +++ internal/session/session.go | 6 +++- internal/watcher/watch_refusals_test.go | 38 +++++++++++++++++++++++++ internal/watcher/watcher.go | 2 +- 7 files changed, 101 insertions(+), 7 deletions(-) diff --git a/internal/cli/audit_sweep.go b/internal/cli/audit_sweep.go index f53b2a8..2215e54 100644 --- a/internal/cli/audit_sweep.go +++ b/internal/cli/audit_sweep.go @@ -148,7 +148,7 @@ func RunAuditSweepTo(ctx context.Context, args []string, out io.Writer) error { returned := map[string]string{} for _, s := range sessions { fetchCtx, cancel := context.WithTimeout(ctx, *timeout) - t, fetchErr := client.FetchTrajectoryWithFallback(fetchCtx, s.CascadeID, s.InternalCascadeID) + t, fetchErr := fetchSweepPresentation(fetchCtx, client, s) cancel() if fetchErr == nil { fetchErr = checkSweepIdentity(ctx, s, t.CascadeID, returned) @@ -195,6 +195,21 @@ func RunAuditSweepTo(ctx context.Context, args []string, out io.Writer) error { return printSweepReaderSync(out, m) } +// Keep corpus rendering independent of qualification. If full JSON or a +// continuation exceeds the budget, use the same bounded transcript as sync +// when SQLite confirms executor records are outside the transcript scope. +func fetchSweepPresentation(ctx context.Context, c *daemon.Client, s discovery.Session) (*daemon.Trajectory, error) { + t, err := c.FetchTrajectoryWithFallback(ctx, s.CascadeID, s.InternalCascadeID) + if !errors.Is(err, daemon.ErrRecovery) || errors.Is(err, daemon.ErrAuthentication) { + return t, err + } + snap, sourceErr := completeness.ReadSyncSnapshot(ctx, s.PBPath) + if sourceErr != nil || snap.OmittedExecutors == 0 { + return nil, err + } + return c.GetTranscript(ctx, snap.CascadeID(), false, snap.StepCount, snap.GeneratorCount(), snap.OmittedExecutors) +} + func checkSweepIdentity(ctx context.Context, s discovery.Session, got string, returned map[string]string) error { id := daemon.CanonicalCascadeID(got) if id == "" { @@ -217,7 +232,7 @@ func selectedSweepIdentity(ctx context.Context, s discovery.Session, id string) if internal := daemon.CanonicalCascadeID(s.InternalCascadeID); internal != "" && id == internal { return true } - snapshot, err := completeness.ReadSnapshot(ctx, s.PBPath) + snapshot, err := completeness.ReadSyncSnapshot(ctx, s.PBPath) return err == nil && id == daemon.CanonicalCascadeID(snapshot.CascadeID()) } diff --git a/internal/cli/audit_sweep_test.go b/internal/cli/audit_sweep_test.go index 9c2f000..ce4fe59 100644 --- a/internal/cli/audit_sweep_test.go +++ b/internal/cli/audit_sweep_test.go @@ -15,6 +15,7 @@ import ( "github.com/mjacobs/agy-reader/internal/discovery" "github.com/mjacobs/agy-reader/internal/testfixture" + "google.golang.org/protobuf/encoding/protowire" ) const sweepTestID = "11111111-1111-1111-1111-111111111111" @@ -298,3 +299,35 @@ func TestAuditSweepAcceptsStoredIdentityDiscoveryCannotRead(t *testing.T) { t.Fatalf("bad manifest: %+v", m) } } + +func TestSweepPublishesOversizedTranscriptWithoutQualifyingFullSource(t *testing.T) { + root, dest, _ := sweepFixture(t) + raw := "{\"cascadeId\":\"" + sweepTestID + "\",\"metadata\":{}}" + db := testfixture.Source(t, filepath.Join(root, "conversations", sweepTestID+".db"), []byte(raw)) + testfixture.Exec(t, db, "INSERT INTO executor_metadata VALUES(0,zeroblob(67108865))") + header := protowire.AppendString(protowire.AppendTag(nil, 6, protowire.BytesType), sweepTestID) + header = protowire.AppendBytes(protowire.AppendTag(header, 7, protowire.BytesType), nil) + envelope := protowire.AppendBytes(protowire.AppendTag(nil, 1, protowire.BytesType), header) + envelope = protowire.AppendVarint(protowire.AppendTag(envelope, 5, protowire.VarintType), 1) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/GetCascadeTrajectory") { + if r.Header.Get("Content-Type") == "application/proto" { + _, _ = w.Write(envelope) + return + } + _, _ = w.Write([]byte(`{"trajectory":` + raw + `,"numTotalExecutorMetadata":1}`)) + return + } + _, _ = w.Write([]byte("{}")) + })) + defer srv.Close() + t.Setenv("ANTIGRAVITY_DAEMON_URL", srv.URL) + var out bytes.Buffer + if err := RunAuditSweepTo(t.Context(), []string{"--root", root, "--out", dest}, &out); err != nil { + t.Fatal(err) + } + m := readSweepManifest(t, dest) + if !m.Complete || len(m.Sessions) != 1 || m.Sessions[0].ReaderSync != "transcript-only" || m.ReaderSync.Verified != 0 { + t.Fatal(m) + } +} diff --git a/internal/daemon/client.go b/internal/daemon/client.go index 5e0b3cc..e8dbd74 100644 --- a/internal/daemon/client.go +++ b/internal/daemon/client.go @@ -122,14 +122,14 @@ func (c *Client) getTrajectory(ctx context.Context, id string, full bool, reques } changed, err := c.completeLists(ctx, id, full, fields, resp.recordCounts) if err != nil { - return nil, err + return nil, fmt.Errorf("%w: %w", ErrRecovery, err) } if !changed { return trajectoryFromJSON(resp.Trajectory) } resp.Trajectory, err = json.Marshal(fields) if len(resp.Trajectory) > responseLimit { - return nil, errors.New("recovered trajectory exceeds size limit") + return nil, fmt.Errorf("%w: recovered trajectory exceeds size limit", ErrRecovery) } if err != nil { return nil, err @@ -278,7 +278,7 @@ func (c *Client) request(ctx context.Context, method, media string, payload []by return nil, fmt.Errorf("read response: %w", readErr) } if len(respBody) > limit { - return nil, fmt.Errorf("%s response exceeds size limit", method) + return nil, fmt.Errorf("%w: %s response exceeds size limit", ErrRecovery, method) } return respBody, nil } diff --git a/internal/daemon/recovery.go b/internal/daemon/recovery.go index 5bf5bcd..44109a6 100644 --- a/internal/daemon/recovery.go +++ b/internal/daemon/recovery.go @@ -12,6 +12,10 @@ import ( const responseLimit = completeness.MaxResponseBytes +// ErrRecovery marks a response that could not be assembled within the reader's +// budget. Callers may try a source-qualified transcript or back off the retry. +var ErrRecovery = errors.New("trajectory recovery failed") + type recordCounts struct { Steps *int `json:"numTotalSteps"` Generators *int `json:"numTotalGeneratorMetadata"` diff --git a/internal/session/session.go b/internal/session/session.go index 1a903f2..b11171e 100644 --- a/internal/session/session.go +++ b/internal/session/session.go @@ -71,7 +71,11 @@ func Fetch(ctx context.Context, client *daemon.Client, s discovery.Session) (*da } fetch := func(raw bool) (*daemon.Trajectory, error) { if before.OmittedExecutors > 0 { - return client.GetTranscript(ctx, id, raw, before.StepCount, before.GeneratorCount(), before.OmittedExecutors) + t, err := client.GetTranscript(ctx, id, raw, before.StepCount, before.GeneratorCount(), before.OmittedExecutors) + if err != nil { + return nil, fmt.Errorf("%w: %w", daemon.ErrRecovery, err) + } + return t, nil } if raw { return client.GetFullRawTrajectory(ctx, id) diff --git a/internal/watcher/watch_refusals_test.go b/internal/watcher/watch_refusals_test.go index fe0d4bc..c46bb58 100644 --- a/internal/watcher/watch_refusals_test.go +++ b/internal/watcher/watch_refusals_test.go @@ -2,10 +2,13 @@ package watcher import ( "bytes" + "net/http" + "net/http/httptest" "os" "path/filepath" "runtime" "strings" + "sync/atomic" "testing" "time" @@ -436,3 +439,38 @@ func TestFailedFirstCaptureBacksOffAndRecovers(t *testing.T) { t.Fatal(err, logs.String()) } } + +func TestOversizedTranscriptRecoveryBacksOff(t *testing.T) { + root, _, db := unsupportedProjectionFixture(t) + testfixture.Exec(t, db, "INSERT INTO executor_metadata VALUES(0,zeroblob(67108865))") + var calls atomic.Int64 + d := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Content-Type") == "application/proto" { + calls.Add(1) + return + } + _, _ = w.Write([]byte("{}")) + })) + defer d.Close() + var logs bytes.Buffer + w := heldWatcher(t, root, d.URL, &logs) + now := time.Now() + old := refusalNow + refusalNow = func() time.Time { return now } + t.Cleanup(func() { refusalNow = old }) + w.Tick() + first := calls.Load() + if first == 0 { + t.Fatal("first recovery did not contact daemon") + } + now = now.Add(time.Minute) + w.Tick() + if calls.Load() != first { + t.Fatal("retried recovery after one minute", logs.String()) + } + now = now.Add(captureRetryDelay) + w.Tick() + if calls.Load() == first { + t.Fatal("recovery never retried", logs.String()) + } +} diff --git a/internal/watcher/watcher.go b/internal/watcher/watcher.go index 9664b1f..dd199dc 100644 --- a/internal/watcher/watcher.go +++ b/internal/watcher/watcher.go @@ -416,7 +416,7 @@ func WatchTickCachedPolicy(ctx context.Context, client *daemon.Client, root stri } else if mismatchedResponse(err) && held != nil && ctx.Err() == nil { held.HoldMismatch(s, endpoint) logger.Printf("watch: skip %s (%v); retrying in %s unless its source or daemon endpoint changes", s.CascadeID, err, mismatchRetryDelay) - } else if captureFailed && held != nil && ctx.Err() == nil { + } else if (captureFailed || errors.Is(err, daemon.ErrRecovery)) && held != nil && ctx.Err() == nil { held.holdUntilRetry(s, endpoint, captureRetryDelay) logger.Printf("watch: skip %s (%v); retrying capture in %s unless its inputs change", s.CascadeID, err, captureRetryDelay) } else {