Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions apps/computer/local-daemon/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,14 @@ model.
Daemon rereads the status, which carries what the phase reserved or freed
(pool use, placement, models unloaded first), and applies the job's line on
top; later lines of the phase update that snapshot without another read.
- Error causes. `library status` keeps only a failed row's `code` and
`detail`; the `cause` and `path` of a `GATEWAY_REGISTRATION_FAILED` reach
the Daemon only on the job's error line (`GATEWAY_HOME_NOT_PRIVATE`: the
gateway's private folder at `path` has another owner). The Daemon keeps them
per model and puts them back on that row in every read while it still fails
with that code; the model's next accepted job, or a row that moved on,
drops them. They live in memory, so after a Daemon restart the row shows
only its code.
- Pi. When the loaded models change, the Daemon asks the Runtime monitor for a
full discovery, so TeamHub sees Pi's new model list. Each model's
`pi_model` (`<provider>/<id>`, as an Agent's `runtime_model` stores it) is
Expand Down
33 changes: 30 additions & 3 deletions apps/computer/local-daemon/internal/localmodels/library.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"log/slog"
"os"
"path/filepath"
"slices"
"sort"
"strings"
"sync"
Expand Down Expand Up @@ -92,8 +93,12 @@ type Library struct {
current *wire.LocalModelsSnapshot
// loaded identifies the loaded models of the last status read; nil
// before the first one.
loaded *string
jobs map[string]*job
loaded *string
jobs map[string]*job
// causes holds, by model, the cause the last job's error line gave. The
// tool's status keeps only the code, so each read puts the cause back on
// a row that still fails with that code.
causes map[string]*wire.LocalModelError
lastProgress time.Time
progressTimer *time.Timer
// progressTimerID tells a timer callback that already fired whether it
Expand All @@ -113,7 +118,7 @@ func New(options Options) *Library {
tool: options.Tool, environment: options.Environment, jobDirectory: options.JobDirectory,
publish: options.Publish, runtimeChanged: options.RuntimeChanged,
progressInterval: options.ProgressInterval, statusTimeout: options.StatusTimeout, actionTimeout: options.ActionTimeout,
now: time.Now, ctx: ctx, stop: stop, jobs: make(map[string]*job),
now: time.Now, ctx: ctx, stop: stop, jobs: make(map[string]*job), causes: make(map[string]*wire.LocalModelError),
}
if library.progressInterval <= 0 {
library.progressInterval = defaultProgressInterval
Expand Down Expand Up @@ -190,9 +195,25 @@ func (l *Library) read(ctx context.Context) wire.LocalModelsSnapshot {
slog.Warn("local_models.status_invalid", "error", err)
return unavailable(CodeToolFailed, "localmodels reported a library status outside its contract", l.now())
}
l.restoreCauses(&snapshot)
return snapshot
}

// restoreCauses puts each kept cause back on its failed row, and forgets one
// whose row no longer fails with the code it came with.
func (l *Library) restoreCauses(snapshot *wire.LocalModelsSnapshot) {
l.mu.Lock()
defer l.mu.Unlock()
for id, cause := range l.causes {
index := slices.IndexFunc(snapshot.Models, func(model wire.LocalModel) bool { return model.ID == id })
if index < 0 || snapshot.Models[index].Error == nil || snapshot.Models[index].Error.Code != cause.Code {
delete(l.causes, id)
continue
}
snapshot.Models[index].Error.Cause, snapshot.Models[index].Error.Path = cause.Cause, cause.Path
}
}

// loadedKey names the loaded models; a change means Pi's model list changed.
func loadedKey(snapshot wire.LocalModelsSnapshot) string {
var loaded []string
Expand Down Expand Up @@ -482,6 +503,7 @@ func (l *Library) superviseJob(action Action, events <-chan libraryEvent, ended
if accepted {
l.mu.Lock()
l.jobs[action.Model] = &job{action: action}
delete(l.causes, action.Model)
l.mu.Unlock()
}
}
Expand Down Expand Up @@ -532,6 +554,11 @@ func (l *Library) superviseJob(action Action, events <-chan libraryEvent, ended
}
l.mu.Lock()
delete(l.jobs, action.Model)
if final != nil && final.Event == "error" {
if cause := final.cause(); cause != nil {
l.causes[action.Model] = cause
}
}
l.mu.Unlock()
l.Refresh(l.ctx)
if action.Name == ActionLoad {
Expand Down
44 changes: 44 additions & 0 deletions apps/computer/local-daemon/internal/localmodels/library_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -622,3 +622,47 @@ func TestRefreshMapsRichModelCardMetadataAndKeepsItAcrossProgress(t *testing.T)
t.Fatalf("progress dropped model card fields: %+v", after)
}
}

// localmodels names why a load could not register with its gateway only on
// the job's error line; the snapshot's failed row carries it until another
// job for that model ends.
func TestAFailedLoadKeepsTheGatewayCauseOnItsRow(t *testing.T) {
fake := newFakeTool(t)
fake.write("status.json", status(model(testModel, "idle")))
failed := model(testModel, "failed")
failed["error"] = map[string]any{"code": "GATEWAY_REGISTRATION_FAILED", "detail": "gateway-home is not private"}
encodedFailed, _ := json.Marshal(status(failed))
const home = `C:\Users\ana\AppData\Local\localmodels\tools\gateway-home`
causeLine := errorLine(testModel, "GATEWAY_REGISTRATION_FAILED", `,"cause":"GATEWAY_HOME_NOT_PRIVATE","path":"C:\\Users\\ana\\AppData\\Local\\localmodels\\tools\\gateway-home"`)
fake.script("load", fakeScript{
Lines: []fakeLine{{Line: progressLine(testModel, "load", 0, 2)}, {Line: causeLine, DelayMS: 20}},
Exit: 1, StatusAfter: encodedFailed,
})
record := &recorder{}
library := newLibrary(t, fake, record, 10*time.Millisecond)
library.Refresh(context.Background())

if refusal, _ := library.Act(context.Background(), Action{Name: ActionLoad, Model: testModel}, soon()); refusal != nil {
t.Fatalf("load refused: %+v", refusal)
}
waitFor(t, "the failed row with its cause", func() bool {
model := modelState(record.last(), testModel)
return model.State == "failed" && model.Error != nil && model.Error.Cause == "GATEWAY_HOME_NOT_PRIVATE" && model.Error.Path == home
})
if again := modelState(library.Refresh(context.Background()), testModel); again.Error == nil || again.Error.Cause != "GATEWAY_HOME_NOT_PRIVATE" {
t.Fatalf("a later read dropped the cause: %+v", again.Error)
}

// A retry that fails for another reason carries no stale cause.
fake.script("load", fakeScript{
Lines: []fakeLine{{Line: progressLine(testModel, "load", 0, 2)}, {Line: errorLine(testModel, "GATEWAY_REGISTRATION_FAILED", ""), DelayMS: 20}},
Exit: 1, StatusAfter: encodedFailed,
})
if refusal, _ := library.Act(context.Background(), Action{Name: ActionLoad, Model: testModel}, soon()); refusal != nil {
t.Fatalf("retry refused: %+v", refusal)
}
waitFor(t, "the cause to clear", func() bool {
model := modelState(library.Refresh(context.Background()), testModel)
return model.Error != nil && model.Error.Cause == "" && model.Error.Path == ""
})
}
15 changes: 15 additions & 0 deletions apps/computer/local-daemon/internal/localmodels/status.go
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,8 @@ type libraryEvent struct {
NeedBytes *float64 `json:"need_bytes"`
FreeBytes *float64 `json:"free_bytes"`
HaveBytes *float64 `json:"have_bytes"`
Cause string `json:"cause"`
Path string `json:"path"`
}

func parseEvent(line []byte) (libraryEvent, bool) {
Expand Down Expand Up @@ -306,6 +308,19 @@ func bytesValue(value float64) uint64 {
return uint64(value)
}

// errorCauses is the tool's closed set of error causes; a GATEWAY_REGISTRATION_FAILED
// line carries one with the path it names.
var errorCauses = []string{"GATEWAY_HOME_NOT_PRIVATE"}

// cause is the error line's cause as the wire carries it on a model's error,
// or nil when the line names none the contract knows.
func (e libraryEvent) cause() *wire.LocalModelError {
if !slices.Contains(errorCodes, e.Code) || !slices.Contains(errorCauses, e.Cause) || e.Path == "" {
return nil
}
return &wire.LocalModelError{Code: e.Code, Cause: e.Cause, Path: clip(e.Path, 4096)}
}

// clip bounds display text to the schema's length in characters.
func clip(text string, limit int) string {
if utf8.RuneCountInString(text) <= limit {
Expand Down
6 changes: 6 additions & 0 deletions apps/node-services/src/agent/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,12 @@ take precedence.
cancellation, and the only Runtime execution path. Do not add a second
scheduler, execution store, or Runtime executor beside it, and do not
restore TeamHub Run admission.
- A Room Agent speaks only through TeamHub message writes, which the Computer
classifies as `message` activities (`reply-delivery.ts`). A Room Run whose
every write failed finishes failed with `reply_undelivered`. One that never
tried to write while a Human's Message that mentioned the Agent (a direct
chat always does) waits finishes failed with `reply_not_sent`. Its final
text is private scratch and is never posted for it.
- Ordinary Room/Meta execution reads effective Agent configuration before a new
Local Run. Skill saves do not activate idle Contexts. If versions changed,
reuse graceful activation to resume the same native history; running/steered
Expand Down
1 change: 1 addition & 0 deletions apps/node-services/src/agent/executor/provider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -459,6 +459,7 @@ export interface RuntimeExecutionFailure {
| "execution_failed"
| "execution_outcome_unknown"
| "reply_undelivered"
| "reply_not_sent"
| "interrupted"
| "computer_unavailable"
| "cleanup_unconfirmed"
Expand Down
42 changes: 41 additions & 1 deletion apps/node-services/src/agent/executor/reply-delivery.ts
Original file line number Diff line number Diff line change
@@ -1,19 +1,30 @@
import { isRecord } from "@linger/service-kit";
import type { RunEvent } from "./provider.js";
import type { Message } from "./types.js";

/**
* The Run's TeamHub message calls, as the Computer classifies them: a
* `message` activity is a message write (`teamhub message send` and its
* siblings), `message_read` any other `teamhub message` call, and a command
* that exits non-zero finishes as failed.
* that exits non-zero finishes as failed. An emoji Reaction is a message
* write too.
*/
export class ReplyDelivery {
readonly #started = new Map<string, "message" | "message_read">();
#awaitingHuman = false;
#attemptedWrites = 0;
#failedWrites = 0;
#deliveredWrites = 0;
#failedReads = 0;

/** Records the inputs the Run took up, at its start or as steers. */
observeInputs(messages: readonly Message[]): void {
if (messages.some(awaitsReply)) this.#awaitingHuman = true;
}

observe(event: RunEvent): void {
if (event.type === "activity.started") {
if (event.kind === "message") this.#attemptedWrites += 1;
if (event.kind === "message" || event.kind === "message_read")
this.#started.set(event.activityId, event.kind);
return;
Expand All @@ -39,6 +50,15 @@ export class ReplyDelivery {
return this.#failedWrites > 0 && this.#deliveredWrites === 0;
}

/**
* A Human who mentioned the Agent is waiting and the Agent never tried to
* post. Its Runtime ended the turn in text nobody sees; small models do
* this when they answer without calling the message command.
*/
get replyNotSent(): boolean {
return this.#awaitingHuman && this.#attemptedWrites === 0;
}

get hasFailures(): boolean {
return this.#failedWrites > 0 || this.#failedReads > 0;
}
Expand All @@ -55,3 +75,23 @@ export class ReplyDelivery {
};
}
}

/**
* Whether TeamHub queued this input for a Human's Message that mentioned this
* Agent. A direct chat mentions its Agent too. Room talk the Agent only reads
* along with, a Routine wake, a Task callback and another Agent's Message may
* rightly go unanswered.
*/
function awaitsReply(message: Message): boolean {
const notification = (message as unknown as Record<string, unknown>)
.acceptedNotification;
if (!isRecord(notification) || !Array.isArray(notification.messages))
return false;
return notification.messages.some(
(queued) =>
isRecord(queued) &&
queued.mentioned === true &&
queued.authorType === "user" &&
queued.messageKind === "conversation",
);
}
52 changes: 37 additions & 15 deletions apps/node-services/src/agent/executor/run-loop.ts
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,7 @@ interface ActiveExecution {
readonly runId: string;
readonly sourceInboxIds: Set<string>;
readonly steerOperations: Set<Promise<void>>;
readonly replyDelivery: ReplyDelivery;
acceptingSteers: boolean;
}

Expand Down Expand Up @@ -1005,6 +1006,7 @@ export class RunLoop {
);
}
const replyDelivery = new ReplyDelivery();
replyDelivery.observeInputs(execution.messages);
let resumeStartRetries = 0;
for (;;) {
controller.signal.throwIfAborted();
Expand Down Expand Up @@ -1052,7 +1054,12 @@ export class RunLoop {
execution.run = run;
const providerReadyAt = performance.now();
this.#logProviderReady(execution, run, providerStartAt, providerReadyAt);
const active = await this.#activateExecution(execution, run, input);
const active = await this.#activateExecution(
execution,
run,
input,
replyDelivery,
);

execution.runtimeStatus ??= await this.#openRuntimeStatus(
runId,
Expand Down Expand Up @@ -1291,6 +1298,7 @@ export class RunLoop {
execution: LocalRunExecution,
run: AgentRun,
input: AgentInput,
replyDelivery: ReplyDelivery,
): Promise<ActiveExecution> {
const { runId, controller } = execution;
if ((await this.#state.getLocalRun(runId))?.cancellationRequested) {
Expand Down Expand Up @@ -1322,6 +1330,7 @@ export class RunLoop {
this.#activeExecution?.sourceInboxIds ??
new Set(execution.provenance.sourceInboxIds),
steerOperations: new Set(),
replyDelivery,
};
this.#activeExecution = active;
execution.intent.resolveProviderReady(active);
Expand Down Expand Up @@ -1717,19 +1726,19 @@ export class RunLoop {
}

/**
* A Room Agent speaks only through TeamHub message writes. When it tried to
* post and none landed, the Room has nothing to show, so a finished Provider
* turn is not a succeeded Run. Task Runs report through their final
* response and keep it.
* A Room Agent speaks only through TeamHub message writes, so a finished
* Provider turn is not a succeeded Run when the Room has nothing to show:
* the Agent tried to post and none landed, or a Human who mentioned it is
* waiting and it never tried. Its final text is private scratch and is not
* posted for it. Task Runs report through their final response and keep it.
*/
#completedTerminal(
runId: string,
finalResponse: string,
replyDelivery: ReplyDelivery,
): LocalRunTerminal {
const undelivered =
this.#executionContextRef?.kind === "meta" &&
replyDelivery.replyUndelivered;
const roomRun = this.#executionContextRef?.kind === "meta";
const undelivered = roomRun && replyDelivery.replyUndelivered;
if (replyDelivery.hasFailures) {
this.#logger.log("warn", "local_run.teamhub_messages_failed", {
runId,
Expand All @@ -1738,14 +1747,26 @@ export class RunLoop {
finalResponseLength: finalResponse.length,
});
}
if (!undelivered) {
return { type: "succeeded", finalResponse };
if (undelivered) {
return {
type: "failed",
error: "Agent reply was not delivered to the Room",
failure: runFailure("reply_undelivered", this.#runtimeType),
};
}
return {
type: "failed",
error: "Agent reply was not delivered to the Room",
failure: runFailure("reply_undelivered", this.#runtimeType),
};
if (roomRun && replyDelivery.replyNotSent) {
this.#logger.log("warn", "local_run.reply_not_sent", {
runId,
...replyDelivery.counts(),
finalResponseLength: finalResponse.length,
});
return {
type: "failed",
error: "Agent ended its turn without sending a reply to the Room",
failure: runFailure("reply_not_sent", this.#runtimeType),
};
}
return { type: "succeeded", finalResponse };
}

#reportTerminal(runId: string, terminal: LocalTurnTerminal): void {
Expand Down Expand Up @@ -1839,6 +1860,7 @@ export class RunLoop {
target.sourceInboxIds.add(inboxId);
}
target.inputs.push(input);
target.replyDelivery.observeInputs(messages);
this.#reportActiveRunContext({
runId: target.runId,
sourceInboxIds: Object.freeze([...target.sourceInboxIds]),
Expand Down
Loading
Loading