Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions apps/computer/local-daemon/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,14 @@ model.
Daemon rereads the status, which carries what the phase reserved or freed
(pool use, placement, models unloaded first), and applies the job's line on
top; later lines of the phase update that snapshot without another read.
- Error causes. `library status` keeps only a failed row's `code` and
`detail`; the `cause` and `path` of a `GATEWAY_REGISTRATION_FAILED` reach
the Daemon only on the job's error line (`GATEWAY_HOME_NOT_PRIVATE`: the
gateway's private folder at `path` has another owner). The Daemon keeps them
per model and puts them back on that row in every read while it still fails
with that code; the model's next accepted job, or a row that moved on,
drops them. They live in memory, so after a Daemon restart the row shows
only its code.
- Pi. When the loaded models change, the Daemon asks the Runtime monitor for a
full discovery, so TeamHub sees Pi's new model list. Each model's
`pi_model` (`<provider>/<id>`, as an Agent's `runtime_model` stores it) is
Expand Down
33 changes: 30 additions & 3 deletions apps/computer/local-daemon/internal/localmodels/library.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"log/slog"
"os"
"path/filepath"
"slices"
"sort"
"strings"
"sync"
Expand Down Expand Up @@ -92,8 +93,12 @@ type Library struct {
current *wire.LocalModelsSnapshot
// loaded identifies the loaded models of the last status read; nil
// before the first one.
loaded *string
jobs map[string]*job
loaded *string
jobs map[string]*job
// causes holds, by model, the cause the last job's error line gave. The
// tool's status keeps only the code, so each read puts the cause back on
// a row that still fails with that code.
causes map[string]*wire.LocalModelError
lastProgress time.Time
progressTimer *time.Timer
// progressTimerID tells a timer callback that already fired whether it
Expand All @@ -113,7 +118,7 @@ func New(options Options) *Library {
tool: options.Tool, environment: options.Environment, jobDirectory: options.JobDirectory,
publish: options.Publish, runtimeChanged: options.RuntimeChanged,
progressInterval: options.ProgressInterval, statusTimeout: options.StatusTimeout, actionTimeout: options.ActionTimeout,
now: time.Now, ctx: ctx, stop: stop, jobs: make(map[string]*job),
now: time.Now, ctx: ctx, stop: stop, jobs: make(map[string]*job), causes: make(map[string]*wire.LocalModelError),
}
if library.progressInterval <= 0 {
library.progressInterval = defaultProgressInterval
Expand Down Expand Up @@ -190,9 +195,25 @@ func (l *Library) read(ctx context.Context) wire.LocalModelsSnapshot {
slog.Warn("local_models.status_invalid", "error", err)
return unavailable(CodeToolFailed, "localmodels reported a library status outside its contract", l.now())
}
l.restoreCauses(&snapshot)
return snapshot
}

// restoreCauses puts each kept cause back on its failed row, and forgets one
// whose row no longer fails with the code it came with.
func (l *Library) restoreCauses(snapshot *wire.LocalModelsSnapshot) {
l.mu.Lock()
defer l.mu.Unlock()
for id, cause := range l.causes {
index := slices.IndexFunc(snapshot.Models, func(model wire.LocalModel) bool { return model.ID == id })
if index < 0 || snapshot.Models[index].Error == nil || snapshot.Models[index].Error.Code != cause.Code {
delete(l.causes, id)
continue
}
snapshot.Models[index].Error.Cause, snapshot.Models[index].Error.Path = cause.Cause, cause.Path
}
}

// loadedKey names the loaded models; a change means Pi's model list changed.
func loadedKey(snapshot wire.LocalModelsSnapshot) string {
var loaded []string
Expand Down Expand Up @@ -482,6 +503,7 @@ func (l *Library) superviseJob(action Action, events <-chan libraryEvent, ended
if accepted {
l.mu.Lock()
l.jobs[action.Model] = &job{action: action}
delete(l.causes, action.Model)
l.mu.Unlock()
}
}
Expand Down Expand Up @@ -532,6 +554,11 @@ func (l *Library) superviseJob(action Action, events <-chan libraryEvent, ended
}
l.mu.Lock()
delete(l.jobs, action.Model)
if final != nil && final.Event == "error" {
if cause := final.cause(); cause != nil {
l.causes[action.Model] = cause
}
}
l.mu.Unlock()
l.Refresh(l.ctx)
if action.Name == ActionLoad {
Expand Down
44 changes: 44 additions & 0 deletions apps/computer/local-daemon/internal/localmodels/library_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -622,3 +622,47 @@ func TestRefreshMapsRichModelCardMetadataAndKeepsItAcrossProgress(t *testing.T)
t.Fatalf("progress dropped model card fields: %+v", after)
}
}

// localmodels names why a load could not register with its gateway only on
// the job's error line; the snapshot's failed row carries it until another
// job for that model ends.
func TestAFailedLoadKeepsTheGatewayCauseOnItsRow(t *testing.T) {
fake := newFakeTool(t)
fake.write("status.json", status(model(testModel, "idle")))
failed := model(testModel, "failed")
failed["error"] = map[string]any{"code": "GATEWAY_REGISTRATION_FAILED", "detail": "gateway-home is not private"}
encodedFailed, _ := json.Marshal(status(failed))
const home = `C:\Users\ana\AppData\Local\localmodels\tools\gateway-home`
causeLine := errorLine(testModel, "GATEWAY_REGISTRATION_FAILED", `,"cause":"GATEWAY_HOME_NOT_PRIVATE","path":"C:\\Users\\ana\\AppData\\Local\\localmodels\\tools\\gateway-home"`)
fake.script("load", fakeScript{
Lines: []fakeLine{{Line: progressLine(testModel, "load", 0, 2)}, {Line: causeLine, DelayMS: 20}},
Exit: 1, StatusAfter: encodedFailed,
})
record := &recorder{}
library := newLibrary(t, fake, record, 10*time.Millisecond)
library.Refresh(context.Background())

if refusal, _ := library.Act(context.Background(), Action{Name: ActionLoad, Model: testModel}, soon()); refusal != nil {
t.Fatalf("load refused: %+v", refusal)
}
waitFor(t, "the failed row with its cause", func() bool {
model := modelState(record.last(), testModel)
return model.State == "failed" && model.Error != nil && model.Error.Cause == "GATEWAY_HOME_NOT_PRIVATE" && model.Error.Path == home
})
if again := modelState(library.Refresh(context.Background()), testModel); again.Error == nil || again.Error.Cause != "GATEWAY_HOME_NOT_PRIVATE" {
t.Fatalf("a later read dropped the cause: %+v", again.Error)
}

// A retry that fails for another reason carries no stale cause.
fake.script("load", fakeScript{
Lines: []fakeLine{{Line: progressLine(testModel, "load", 0, 2)}, {Line: errorLine(testModel, "GATEWAY_REGISTRATION_FAILED", ""), DelayMS: 20}},
Exit: 1, StatusAfter: encodedFailed,
})
if refusal, _ := library.Act(context.Background(), Action{Name: ActionLoad, Model: testModel}, soon()); refusal != nil {
t.Fatalf("retry refused: %+v", refusal)
}
waitFor(t, "the cause to clear", func() bool {
model := modelState(library.Refresh(context.Background()), testModel)
return model.Error != nil && model.Error.Cause == "" && model.Error.Path == ""
})
}
15 changes: 15 additions & 0 deletions apps/computer/local-daemon/internal/localmodels/status.go
Original file line number Diff line number Diff line change
Expand Up @@ -246,6 +246,8 @@ type libraryEvent struct {
NeedBytes *float64 `json:"need_bytes"`
FreeBytes *float64 `json:"free_bytes"`
HaveBytes *float64 `json:"have_bytes"`
Cause string `json:"cause"`
Path string `json:"path"`
}

func parseEvent(line []byte) (libraryEvent, bool) {
Expand Down Expand Up @@ -306,6 +308,19 @@ func bytesValue(value float64) uint64 {
return uint64(value)
}

// errorCauses is the tool's closed set of error causes; a GATEWAY_REGISTRATION_FAILED
// line carries one with the path it names.
var errorCauses = []string{"GATEWAY_HOME_NOT_PRIVATE"}

// cause is the error line's cause as the wire carries it on a model's error,
// or nil when the line names none the contract knows.
func (e libraryEvent) cause() *wire.LocalModelError {
if !slices.Contains(errorCodes, e.Code) || !slices.Contains(errorCauses, e.Cause) || e.Path == "" {
return nil
}
return &wire.LocalModelError{Code: e.Code, Cause: e.Cause, Path: clip(e.Path, 4096)}
}

// clip bounds display text to the schema's length in characters.
func clip(text string, limit int) string {
if utf8.RuneCountInString(text) <= limit {
Expand Down
7 changes: 7 additions & 0 deletions apps/teamhub/internal/app/handlers_local_models.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,6 +147,10 @@ func localModelsView(record agentruntimestate.LocalModelsRecord, viewerID string
}
}
}
// A cause's path is a folder on the owner's machine; only they can act on it.
if model.Error != nil && !view.ViewerIsOwner {
model.Error.Path = nil
}
view.Models = append(view.Models, httpapi.LocalModelView{Model: model, UsedBy: users})
}
return view, nil
Expand Down Expand Up @@ -324,6 +328,9 @@ func validLocalModelsSnapshot(snapshot teamhubinternal.LocalModelsSnapshot) bool
if model.PromptTokensPerSecond != nil && *model.PromptTokensPerSecond < 0 {
return false
}
if model.Error != nil && model.Error.Cause != nil && !model.Error.Cause.Valid() {
return false
}
seen[model.Id] = true
}
return true
Expand Down
29 changes: 29 additions & 0 deletions apps/teamhub/internal/app/handlers_local_models_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package app
import (
"encoding/json"
"slices"
"strings"
"testing"
"time"

Expand Down Expand Up @@ -97,6 +98,29 @@ func TestLocalModelsViewMatchesAgentsToModelsOnTheComputer(t *testing.T) {
}
}

// The folder a gateway cause names is a path on the owner's machine: only
// the owner, who can rename it, reads it.
func TestLocalModelsViewShowsTheCausePathOnlyToTheOwner(t *testing.T) {
snapshot := strings.Replace(storedLocalModelsSnapshotJSON, `"state":"idle"`,
`"state":"failed","error":{"code":"GATEWAY_REGISTRATION_FAILED","cause":"GATEWAY_HOME_NOT_PRIVATE","path":"/home/ana/.local/share/localmodels/tools/gateway-home"}`, 1)
record := agentruntimestate.LocalModelsRecord{
ComputerID: "computer_1", DisplayName: "Studio", Status: "offline",
Owner: agentruntimestate.LocalComputerOwner{TenantID: "tenant_1", OwnerActorID: "actor_owner"},
Snapshot: json.RawMessage(snapshot),
}
for viewer, wantPath := range map[string]bool{"actor_owner": true, "actor_member": false} {
view, err := localModelsView(record, viewer, time.Now())
if err != nil {
t.Fatal(err)
}
failed := view.Models[2].Model.Error
if failed == nil || failed.Cause == nil || *failed.Cause != httpapi.LocalModelErrorCauseGatewayHomeNotPrivate ||
(failed.Path != nil) != wantPath {
t.Fatalf("%s: error=%+v", viewer, failed)
}
}
}

func TestLocalModelsViewWithoutSnapshotIsNotReported(t *testing.T) {
view, err := localModelsView(agentruntimestate.LocalModelsRecord{ComputerID: "computer_1", DisplayName: "Studio", Status: "offline"}, "actor_owner", time.Now())
if err != nil || view.Available || view.Reason == nil || view.Reason.Code != httpapi.LocalModelsViewNotReported ||
Expand Down Expand Up @@ -179,6 +203,11 @@ func TestValidLocalModelsSnapshot(t *testing.T) {
u := "http://huggingface.co/Qwen/Qwen3-4B"
s.Models[0].SourceUrl = &u
},
"unknown error cause": func(s *teamhubinternal.LocalModelsSnapshot) {
c := teamhubinternal.LocalModelErrorCause("DISK_LOCKED")
s.Models[0].State = teamhubinternal.LocalModelStateFailed
s.Models[0].Error = &teamhubinternal.LocalModelError{Code: "GATEWAY_REGISTRATION_FAILED", Cause: &c}
},
"negative prompt_tokens_per_second": func(s *teamhubinternal.LocalModelsSnapshot) {
p := float32(-1)
s.Models[0].PromptTokensPerSecond = &p
Expand Down
5 changes: 4 additions & 1 deletion apps/teamhub/internal/app/handlers_message_attachment.go
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import (
"github.com/simplexai-labs/linger/apps/teamhub/internal/collaboration"
"github.com/simplexai-labs/linger/apps/teamhub/internal/idempotency"
"github.com/simplexai-labs/linger/apps/teamhub/internal/secretbox"
httpapi "github.com/simplexai-labs/linger/contracts-go/teamhubpublic"
)

func (a *App) uploadMessageAttachment(c *gin.Context) {
Expand All @@ -21,7 +22,9 @@ func (a *App) uploadMessageAttachment(c *gin.Context) {
upload, err := a.parseMessageAttachmentUpload(c)
if err != nil {
if errors.Is(err, artifactstore.ErrTooLarge) || isMaxBytesError(err) {
a.writeError(c, http.StatusRequestEntityTooLarge, "PAYLOAD_TOO_LARGE", "message attachment exceeds the configured limit")
code, limit := "too_large", a.cfg.ArtifactMaxBytes
a.writeErrorWithDetails(c, http.StatusRequestEntityTooLarge, "PAYLOAD_TOO_LARGE", "message attachment exceeds the configured limit",
[]httpapi.ErrorDetail{{Field: "content", Code: &code, Reason: "exceeds the attachment size limit", LimitBytes: &limit}})
} else {
a.writeError(c, http.StatusBadRequest, "INVALID_REQUEST", "message attachment upload is invalid")
}
Expand Down
54 changes: 54 additions & 0 deletions apps/teamhub/internal/app/handlers_upload_too_large_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package app

import (
"bytes"
"encoding/json"
"mime/multipart"
"net/http"
"net/http/httptest"
Expand All @@ -11,6 +12,7 @@ import (
"github.com/gin-gonic/gin"

"github.com/simplexai-labs/linger/apps/teamhub/internal/config"
httpapi "github.com/simplexai-labs/linger/contracts-go/teamhubpublic"
)

// The multipart parser must surface *http.MaxBytesError so handlers map an
Expand Down Expand Up @@ -56,3 +58,55 @@ func TestMultipartUploadsMapOversizedBodyToPayloadTooLarge(t *testing.T) {
})
}
}

// A refused message attachment names the limit as a structured detail, so a
// client can tell the person how large a file may be.
func TestMessageAttachmentTooLargeNamesTheLimit(t *testing.T) {
t.Parallel()
const maxBytes = 16
for name, size := range map[string]int{
"file over the limit": maxBytes + 1,
"body over the limit": maxBytes + (1 << 20) + 1,
} {
t.Run(name, func(t *testing.T) {
var body bytes.Buffer
writer := multipart.NewWriter(&body)
if err := writer.WriteField("filename", "big.zip"); err != nil {
t.Fatal(err)
}
part, err := writer.CreateFormFile("content", "big.zip")
if err != nil {
t.Fatal(err)
}
if _, err := part.Write(bytes.Repeat([]byte("x"), size)); err != nil {
t.Fatal(err)
}
if err := writer.Close(); err != nil {
t.Fatal(err)
}
recorder := httptest.NewRecorder()
context, _ := gin.CreateTestContext(recorder)
context.Set("request_id", "request-attachment-too-large")
context.Request = httptest.NewRequest(http.MethodPost, "/upload", &body)
context.Request.Header.Set("Content-Type", writer.FormDataContentType())
context.Request.Header.Set("Idempotency-Key", "attachment-too-large-key")
(&App{cfg: config.ForTest(func(c *config.Config) {
c.ArtifactMaxBytes = maxBytes
})}).uploadMessageAttachment(context)

var response httpapi.ErrorResponse
if err := json.Unmarshal(recorder.Body.Bytes(), &response); err != nil {
t.Fatal(err)
}
if recorder.Code != http.StatusRequestEntityTooLarge || response.Error.Code != "PAYLOAD_TOO_LARGE" ||
response.Error.Details == nil || len(*response.Error.Details) != 1 {
t.Fatalf("status=%d body=%s", recorder.Code, recorder.Body.String())
}
detail := (*response.Error.Details)[0]
if detail.Field != "content" || detail.Code == nil || *detail.Code != "too_large" ||
detail.LimitBytes == nil || *detail.LimitBytes != maxBytes {
t.Fatalf("detail = %+v", detail)
}
})
}
}
9 changes: 7 additions & 2 deletions apps/web/AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,13 @@ build:assets` to check the initial JS budget.
its own Computer. Only the Computer's owner can act; actions return at once
and progress arrives as `local-models` invalidation hints. From the click
until a snapshot with another status or observation time, or a refusal, the
row shows a requested load or download as started and spins the button of
any other action, with the row's actions off. A Computer's
row shows a requested load or download as started, labels a cancel, unload
or remove under way (Cancelling, Unloading, Removing), and spins that
action's button, with the row's actions off; a download the Computer
accepted can be cancelled before its first progress. Make room lists what
else holds the pool (components, and other apps from `used_bytes` no holder
accounts for) when unloading models cannot free enough, and says a model
cannot fit only when it needs more than the pool's total. A Computer's
detail on the Computers page links here and reads no model state itself.
- `src/test`: unit-test support, including the in-process TeamHub test server.
- `e2e`: Playwright product workflows; every spec runs on the real TeamHub and
Expand Down
11 changes: 3 additions & 8 deletions apps/web/src/features/local-models/local-model-details.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import {
isModelOnComputer,
modelCapabilities,
modelSourceLabel,
otherAppsBytes,
parametersLabel,
placeName,
poolFreeBytes,
Expand Down Expand Up @@ -118,7 +119,7 @@ export function ModelFacts({
label={stopped ? "Last download" : "Last load"}
>
<span className="text-app-danger">
{modelErrorCopy(model.error.code)}
{modelErrorCopy(model.error)}
</span>
</Fact>
) : null}
Expand Down Expand Up @@ -222,13 +223,7 @@ function PoolBar({
view: LocalModelsView;
}) {
const label = poolLabel(view, pool);
// used_bytes is the device's real use; on a shared card other processes
// hold memory that no holder accounts for.
const otherBytes = Math.max(
0,
pool.used_bytes -
pool.holders.reduce((sum, holder) => sum + holder.bytes, 0),
);
const otherBytes = otherAppsBytes(pool);
const width = (bytes: number) =>
`${pool.total_bytes ? Math.min(100, (bytes / pool.total_bytes) * 100) : 0}%`;
return (
Expand Down
Loading
Loading